summaryrefslogtreecommitdiffstats
path: root/drivers/staging/zcache/zcache-main.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/staging/zcache/zcache-main.c')
-rw-r--r--drivers/staging/zcache/zcache-main.c178
1 files changed, 86 insertions, 92 deletions
diff --git a/drivers/staging/zcache/zcache-main.c b/drivers/staging/zcache/zcache-main.c
index 784c796b9848..c214977b4ab4 100644
--- a/drivers/staging/zcache/zcache-main.c
+++ b/drivers/staging/zcache/zcache-main.c
@@ -31,13 +31,11 @@
#include <linux/math64.h>
#include <linux/crypto.h>
#include <linux/string.h>
+#include <linux/idr.h>
#include "tmem.h"
#include "../zsmalloc/zsmalloc.h"
-#if (!defined(CONFIG_CLEANCACHE) && !defined(CONFIG_FRONTSWAP))
-#error "zcache is useless without CONFIG_CLEANCACHE or CONFIG_FRONTSWAP"
-#endif
#ifdef CONFIG_CLEANCACHE
#include <linux/cleancache.h>
#endif
@@ -53,15 +51,13 @@
(__GFP_FS | __GFP_NORETRY | __GFP_NOWARN | __GFP_NOMEMALLOC)
#endif
-#define MAX_POOLS_PER_CLIENT 16
-
#define MAX_CLIENTS 16
#define LOCAL_CLIENT ((uint16_t)-1)
MODULE_LICENSE("GPL");
struct zcache_client {
- struct tmem_pool *tmem_pools[MAX_POOLS_PER_CLIENT];
+ struct idr tmem_pools;
struct zs_pool *zspool;
bool allocated;
atomic_t refcount;
@@ -78,6 +74,17 @@ static inline uint16_t get_client_id_from_client(struct zcache_client *cli)
return cli - &zcache_clients[0];
}
+static struct zcache_client *get_zcache_client(uint16_t cli_id)
+{
+ if (cli_id == LOCAL_CLIENT)
+ return &zcache_host;
+
+ if ((unsigned int)cli_id < MAX_CLIENTS)
+ return &zcache_clients[cli_id];
+
+ return NULL;
+}
+
static inline bool is_local_client(struct zcache_client *cli)
{
return cli == &zcache_host;
@@ -110,6 +117,8 @@ static inline int zcache_comp_op(enum comp_op op,
case ZCACHE_COMPOP_DECOMPRESS:
ret = crypto_comp_decompress(tfm, src, slen, dst, dlen);
break;
+ default:
+ ret = -EINVAL;
}
put_cpu();
return ret;
@@ -602,16 +611,14 @@ out:
return;
}
-static void zbud_init(void)
+static void __init zbud_init(void)
{
int i;
INIT_LIST_HEAD(&zbud_buddied_list);
- zcache_zbud_buddied_count = 0;
- for (i = 0; i < NCHUNKS; i++) {
+
+ for (i = 0; i < NCHUNKS; i++)
INIT_LIST_HEAD(&zbud_unbuddied[i].list);
- zbud_unbuddied[i].count = 0;
- }
}
#ifdef CONFIG_SYSFS
@@ -693,14 +700,14 @@ static unsigned int zv_max_mean_zsize = (PAGE_SIZE / 8) * 5;
static atomic_t zv_curr_dist_counts[NCHUNKS];
static atomic_t zv_cumul_dist_counts[NCHUNKS];
-static struct zv_hdr *zv_create(struct zs_pool *pool, uint32_t pool_id,
+static unsigned long zv_create(struct zs_pool *pool, uint32_t pool_id,
struct tmem_oid *oid, uint32_t index,
void *cdata, unsigned clen)
{
struct zv_hdr *zv;
u32 size = clen + sizeof(struct zv_hdr);
int chunks = (size + (CHUNK_SIZE - 1)) >> CHUNK_SHIFT;
- void *handle = NULL;
+ unsigned long handle = 0;
BUG_ON(!irqs_disabled());
BUG_ON(chunks >= NCHUNKS);
@@ -709,7 +716,7 @@ static struct zv_hdr *zv_create(struct zs_pool *pool, uint32_t pool_id,
goto out;
atomic_inc(&zv_curr_dist_counts[chunks]);
atomic_inc(&zv_cumul_dist_counts[chunks]);
- zv = zs_map_object(pool, handle);
+ zv = zs_map_object(pool, handle, ZS_MM_WO);
zv->index = index;
zv->oid = *oid;
zv->pool_id = pool_id;
@@ -721,14 +728,14 @@ out:
return handle;
}
-static void zv_free(struct zs_pool *pool, void *handle)
+static void zv_free(struct zs_pool *pool, unsigned long handle)
{
unsigned long flags;
struct zv_hdr *zv;
uint16_t size;
int chunks;
- zv = zs_map_object(pool, handle);
+ zv = zs_map_object(pool, handle, ZS_MM_RW);
ASSERT_SENTINEL(zv, ZVH);
size = zv->size + sizeof(struct zv_hdr);
INVERT_SENTINEL(zv, ZVH);
@@ -743,14 +750,14 @@ static void zv_free(struct zs_pool *pool, void *handle)
local_irq_restore(flags);
}
-static void zv_decompress(struct page *page, void *handle)
+static void zv_decompress(struct page *page, unsigned long handle)
{
unsigned int clen = PAGE_SIZE;
char *to_va;
int ret;
struct zv_hdr *zv;
- zv = zs_map_object(zcache_host.zspool, handle);
+ zv = zs_map_object(zcache_host.zspool, handle, ZS_MM_RO);
BUG_ON(zv->size == 0);
ASSERT_SENTINEL(zv, ZVH);
to_va = kmap_atomic(page);
@@ -939,21 +946,14 @@ static struct tmem_pool *zcache_get_pool_by_id(uint16_t cli_id, uint16_t poolid)
struct tmem_pool *pool = NULL;
struct zcache_client *cli = NULL;
- if (cli_id == LOCAL_CLIENT)
- cli = &zcache_host;
- else {
- if (cli_id >= MAX_CLIENTS)
- goto out;
- cli = &zcache_clients[cli_id];
- if (cli == NULL)
- goto out;
- atomic_inc(&cli->refcount);
- }
- if (poolid < MAX_POOLS_PER_CLIENT) {
- pool = cli->tmem_pools[poolid];
- if (pool != NULL)
- atomic_inc(&pool->refcount);
- }
+ cli = get_zcache_client(cli_id);
+ if (!cli)
+ goto out;
+
+ atomic_inc(&cli->refcount);
+ pool = idr_find(&cli->tmem_pools, poolid);
+ if (pool != NULL)
+ atomic_inc(&pool->refcount);
out:
return pool;
}
@@ -971,13 +971,11 @@ static void zcache_put_pool(struct tmem_pool *pool)
int zcache_new_client(uint16_t cli_id)
{
- struct zcache_client *cli = NULL;
+ struct zcache_client *cli;
int ret = -1;
- if (cli_id == LOCAL_CLIENT)
- cli = &zcache_host;
- else if ((unsigned int)cli_id < MAX_CLIENTS)
- cli = &zcache_clients[cli_id];
+ cli = get_zcache_client(cli_id);
+
if (cli == NULL)
goto out;
if (cli->allocated)
@@ -987,6 +985,7 @@ int zcache_new_client(uint16_t cli_id)
cli->zspool = zs_create_pool("zcache", ZCACHE_GFP_MASK);
if (cli->zspool == NULL)
goto out;
+ idr_init(&cli->tmem_pools);
#endif
ret = 0;
out:
@@ -1035,45 +1034,38 @@ static int zcache_do_preload(struct tmem_pool *pool)
goto out;
if (unlikely(zcache_obj_cache == NULL))
goto out;
- preempt_disable();
+
+ /* IRQ has already been disabled. */
kp = &__get_cpu_var(zcache_preloads);
while (kp->nr < ARRAY_SIZE(kp->objnodes)) {
- preempt_enable_no_resched();
objnode = kmem_cache_alloc(zcache_objnode_cache,
ZCACHE_GFP_MASK);
if (unlikely(objnode == NULL)) {
zcache_failed_alloc++;
goto out;
}
- preempt_disable();
- kp = &__get_cpu_var(zcache_preloads);
- if (kp->nr < ARRAY_SIZE(kp->objnodes))
- kp->objnodes[kp->nr++] = objnode;
- else
- kmem_cache_free(zcache_objnode_cache, objnode);
+
+ kp->objnodes[kp->nr++] = objnode;
}
- preempt_enable_no_resched();
- obj = kmem_cache_alloc(zcache_obj_cache, ZCACHE_GFP_MASK);
- if (unlikely(obj == NULL)) {
- zcache_failed_alloc++;
- goto out;
+
+ if (!kp->obj) {
+ obj = kmem_cache_alloc(zcache_obj_cache, ZCACHE_GFP_MASK);
+ if (unlikely(obj == NULL)) {
+ zcache_failed_alloc++;
+ goto out;
+ }
+ kp->obj = obj;
}
- page = (void *)__get_free_page(ZCACHE_GFP_MASK);
- if (unlikely(page == NULL)) {
- zcache_failed_get_free_pages++;
- kmem_cache_free(zcache_obj_cache, obj);
- goto out;
+
+ if (!kp->page) {
+ page = (void *)__get_free_page(ZCACHE_GFP_MASK);
+ if (unlikely(page == NULL)) {
+ zcache_failed_get_free_pages++;
+ goto out;
+ }
+ kp->page = page;
}
- preempt_disable();
- kp = &__get_cpu_var(zcache_preloads);
- if (kp->obj == NULL)
- kp->obj = obj;
- else
- kmem_cache_free(zcache_obj_cache, obj);
- if (kp->page == NULL)
- kp->page = page;
- else
- free_page((unsigned long)page);
+
ret = 0;
out:
return ret;
@@ -1247,7 +1239,7 @@ static int zcache_pampd_get_data(char *data, size_t *bufsize, bool raw,
int ret = 0;
BUG_ON(is_ephemeral(pool));
- zv_decompress((struct page *)(data), pampd);
+ zv_decompress((struct page *)(data), (unsigned long)pampd);
return ret;
}
@@ -1282,7 +1274,7 @@ static void zcache_pampd_free(void *pampd, struct tmem_pool *pool,
atomic_dec(&zcache_curr_eph_pampd_count);
BUG_ON(atomic_read(&zcache_curr_eph_pampd_count) < 0);
} else {
- zv_free(cli->zspool, pampd);
+ zv_free(cli->zspool, (unsigned long)pampd);
atomic_dec(&zcache_curr_pers_pampd_count);
BUG_ON(atomic_read(&zcache_curr_pers_pampd_count) < 0);
}
@@ -1583,15 +1575,14 @@ static int zcache_put_page(int cli_id, int pool_id, struct tmem_oid *oidp,
else
zcache_failed_pers_puts++;
}
- zcache_put_pool(pool);
- preempt_enable_no_resched();
} else {
zcache_put_to_flush++;
if (atomic_read(&pool->obj_count) > 0)
/* the put fails whether the flush succeeds or not */
(void)tmem_flush_page(pool, oidp, index);
- zcache_put_pool(pool);
}
+
+ zcache_put_pool(pool);
out:
return ret;
}
@@ -1661,22 +1652,21 @@ static int zcache_flush_object(int cli_id, int pool_id,
static int zcache_destroy_pool(int cli_id, int pool_id)
{
struct tmem_pool *pool = NULL;
- struct zcache_client *cli = NULL;
+ struct zcache_client *cli;
int ret = -1;
if (pool_id < 0)
goto out;
- if (cli_id == LOCAL_CLIENT)
- cli = &zcache_host;
- else if ((unsigned int)cli_id < MAX_CLIENTS)
- cli = &zcache_clients[cli_id];
+
+ cli = get_zcache_client(cli_id);
if (cli == NULL)
goto out;
+
atomic_inc(&cli->refcount);
- pool = cli->tmem_pools[pool_id];
+ pool = idr_find(&cli->tmem_pools, pool_id);
if (pool == NULL)
goto out;
- cli->tmem_pools[pool_id] = NULL;
+ idr_remove(&cli->tmem_pools, pool_id);
/* wait for pool activity on other cpus to quiesce */
while (atomic_read(&pool->refcount) != 0)
;
@@ -1696,13 +1686,12 @@ static int zcache_new_pool(uint16_t cli_id, uint32_t flags)
int poolid = -1;
struct tmem_pool *pool;
struct zcache_client *cli = NULL;
+ int r;
- if (cli_id == LOCAL_CLIENT)
- cli = &zcache_host;
- else if ((unsigned int)cli_id < MAX_CLIENTS)
- cli = &zcache_clients[cli_id];
+ cli = get_zcache_client(cli_id);
if (cli == NULL)
goto out;
+
atomic_inc(&cli->refcount);
pool = kmalloc(sizeof(struct tmem_pool), GFP_ATOMIC);
if (pool == NULL) {
@@ -1710,20 +1699,25 @@ static int zcache_new_pool(uint16_t cli_id, uint32_t flags)
goto out;
}
- for (poolid = 0; poolid < MAX_POOLS_PER_CLIENT; poolid++)
- if (cli->tmem_pools[poolid] == NULL)
- break;
- if (poolid >= MAX_POOLS_PER_CLIENT) {
- pr_info("zcache: pool creation failed: max exceeded\n");
+ do {
+ r = idr_pre_get(&cli->tmem_pools, GFP_ATOMIC);
+ if (r != 1) {
+ kfree(pool);
+ pr_info("zcache: pool creation failed: out of memory\n");
+ goto out;
+ }
+ r = idr_get_new(&cli->tmem_pools, pool, &poolid);
+ } while (r == -EAGAIN);
+ if (r) {
+ pr_info("zcache: pool creation failed: error %d\n", r);
kfree(pool);
- poolid = -1;
goto out;
}
+
atomic_set(&pool->refcount, 0);
pool->client = cli;
pool->pool_id = poolid;
tmem_new_pool(pool, flags);
- cli->tmem_pools[poolid] = pool;
pr_info("zcache: created %s tmem pool, id=%d, client=%d\n",
flags & TMEM_POOL_PERSIST ? "persistent" : "ephemeral",
poolid, cli_id);
@@ -1981,7 +1975,7 @@ static int __init enable_zcache_compressor(char *s)
__setup("zcache=", enable_zcache_compressor);
-static int zcache_comp_init(void)
+static int __init zcache_comp_init(void)
{
int ret = 0;
@@ -2021,7 +2015,7 @@ static int __init zcache_init(void)
goto out;
}
#endif /* CONFIG_SYSFS */
-#if defined(CONFIG_CLEANCACHE) || defined(CONFIG_FRONTSWAP)
+
if (zcache_enabled) {
unsigned int cpu;
@@ -2052,7 +2046,7 @@ static int __init zcache_init(void)
pr_err("zcache: can't create client\n");
goto out;
}
-#endif
+
#ifdef CONFIG_CLEANCACHE
if (zcache_enabled && use_cleancache) {
struct cleancache_ops old_ops;
generated by cgit 1.2.3-korg (git 2.39.0) at 2024-05-01 14:16:39 +0000