Commit 66cdef66 authored by Ganesh Mahendran's avatar Ganesh Mahendran Committed by Linus Torvalds

mm/zsmalloc: adjust order of functions

Currently functions in zsmalloc.c does not arranged in a readable and
reasonable sequence.  With the more and more functions added, we may
meet below inconvenience.  For example:

Current functions:

    void zs_init()
    {
    }

    static void get_maxobj_per_zspage()
    {
    }

Then I want to add a func_1() which is called from zs_init(), and this
new added function func_1() will used get_maxobj_per_zspage() which is
defined below zs_init().

    void func_1()
    {
        get_maxobj_per_zspage()
    }

    void zs_init()
    {
        func_1()
    }

    static void get_maxobj_per_zspage()
    {
    }

This will cause compiling issue. So we must add a declaration:

    static void get_maxobj_per_zspage();

before func_1() if we do not put get_maxobj_per_zspage() before
func_1().

In addition, puting module_[init|exit] functions at the bottom of the
file conforms to our habit.

So, this patch ajusts function sequence as:

    /* helper functions */
    ...
    obj_location_to_handle()
    ...

    /* Some exported functions */
    ...

    zs_map_object()
    zs_unmap_object()

    zs_malloc()
    zs_free()

    zs_init()
    zs_exit()
Signed-off-by: default avatarGanesh Mahendran <opensource.ganesh@gmail.com>
Cc: Nitin Gupta <ngupta@vflare.org>
Acked-by: default avatarMinchan Kim <minchan@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 136f49b9
...@@ -884,19 +884,6 @@ static struct notifier_block zs_cpu_nb = { ...@@ -884,19 +884,6 @@ static struct notifier_block zs_cpu_nb = {
.notifier_call = zs_cpu_notifier .notifier_call = zs_cpu_notifier
}; };
static void zs_unregister_cpu_notifier(void)
{
int cpu;
cpu_notifier_register_begin();
for_each_online_cpu(cpu)
zs_cpu_notifier(NULL, CPU_DEAD, (void *)(long)cpu);
__unregister_cpu_notifier(&zs_cpu_nb);
cpu_notifier_register_done();
}
static int zs_register_cpu_notifier(void) static int zs_register_cpu_notifier(void)
{ {
int cpu, uninitialized_var(ret); int cpu, uninitialized_var(ret);
...@@ -914,40 +901,28 @@ static int zs_register_cpu_notifier(void) ...@@ -914,40 +901,28 @@ static int zs_register_cpu_notifier(void)
return notifier_to_errno(ret); return notifier_to_errno(ret);
} }
static void init_zs_size_classes(void) static void zs_unregister_cpu_notifier(void)
{ {
int nr; int cpu;
nr = (ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE) / ZS_SIZE_CLASS_DELTA + 1; cpu_notifier_register_begin();
if ((ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE) % ZS_SIZE_CLASS_DELTA)
nr += 1;
zs_size_classes = nr; for_each_online_cpu(cpu)
} zs_cpu_notifier(NULL, CPU_DEAD, (void *)(long)cpu);
__unregister_cpu_notifier(&zs_cpu_nb);
static void __exit zs_exit(void) cpu_notifier_register_done();
{
#ifdef CONFIG_ZPOOL
zpool_unregister_driver(&zs_zpool_driver);
#endif
zs_unregister_cpu_notifier();
} }
static int __init zs_init(void) static void init_zs_size_classes(void)
{ {
int ret = zs_register_cpu_notifier(); int nr;
if (ret) {
zs_unregister_cpu_notifier();
return ret;
}
init_zs_size_classes(); nr = (ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE) / ZS_SIZE_CLASS_DELTA + 1;
if ((ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE) % ZS_SIZE_CLASS_DELTA)
nr += 1;
#ifdef CONFIG_ZPOOL zs_size_classes = nr;
zpool_register_driver(&zs_zpool_driver);
#endif
return 0;
} }
static unsigned int get_maxobj_per_zspage(int size, int pages_per_zspage) static unsigned int get_maxobj_per_zspage(int size, int pages_per_zspage)
...@@ -967,113 +942,101 @@ static bool can_merge(struct size_class *prev, int size, int pages_per_zspage) ...@@ -967,113 +942,101 @@ static bool can_merge(struct size_class *prev, int size, int pages_per_zspage)
return true; return true;
} }
unsigned long zs_get_total_pages(struct zs_pool *pool)
{
return atomic_long_read(&pool->pages_allocated);
}
EXPORT_SYMBOL_GPL(zs_get_total_pages);
/** /**
* zs_create_pool - Creates an allocation pool to work from. * zs_map_object - get address of allocated object from handle.
* @flags: allocation flags used to allocate pool metadata * @pool: pool from which the object was allocated
* @handle: handle returned from zs_malloc
* *
* This function must be called before anything when using * Before using an object allocated from zs_malloc, it must be mapped using
* the zsmalloc allocator. * this function. When done with the object, it must be unmapped using
* zs_unmap_object.
* *
* On success, a pointer to the newly created pool is returned, * Only one object can be mapped per cpu at a time. There is no protection
* otherwise NULL. * against nested mappings.
*
* This function returns with preemption and page faults disabled.
*/ */
struct zs_pool *zs_create_pool(gfp_t flags) void *zs_map_object(struct zs_pool *pool, unsigned long handle,
enum zs_mapmode mm)
{ {
int i; struct page *page;
struct zs_pool *pool; unsigned long obj_idx, off;
struct size_class *prev_class = NULL;
pool = kzalloc(sizeof(*pool), GFP_KERNEL);
if (!pool)
return NULL;
pool->size_class = kcalloc(zs_size_classes, sizeof(struct size_class *),
GFP_KERNEL);
if (!pool->size_class) {
kfree(pool);
return NULL;
}
/* unsigned int class_idx;
* Iterate reversly, because, size of size_class that we want to use enum fullness_group fg;
* for merging should be larger or equal to current size.
*/
for (i = zs_size_classes - 1; i >= 0; i--) {
int size;
int pages_per_zspage;
struct size_class *class; struct size_class *class;
struct mapping_area *area;
struct page *pages[2];
size = ZS_MIN_ALLOC_SIZE + i * ZS_SIZE_CLASS_DELTA; BUG_ON(!handle);
if (size > ZS_MAX_ALLOC_SIZE)
size = ZS_MAX_ALLOC_SIZE;
pages_per_zspage = get_pages_per_zspage(size);
/* /*
* size_class is used for normal zsmalloc operation such * Because we use per-cpu mapping areas shared among the
* as alloc/free for that size. Although it is natural that we * pools/users, we can't allow mapping in interrupt context
* have one size_class for each size, there is a chance that we * because it can corrupt another users mappings.
* can get more memory utilization if we use one size_class for
* many different sizes whose size_class have same
* characteristics. So, we makes size_class point to
* previous size_class if possible.
*/ */
if (prev_class) { BUG_ON(in_interrupt());
if (can_merge(prev_class, size, pages_per_zspage)) {
pool->size_class[i] = prev_class;
continue;
}
}
class = kzalloc(sizeof(struct size_class), GFP_KERNEL);
if (!class)
goto err;
class->size = size; obj_handle_to_location(handle, &page, &obj_idx);
class->index = i; get_zspage_mapping(get_first_page(page), &class_idx, &fg);
class->pages_per_zspage = pages_per_zspage; class = pool->size_class[class_idx];
spin_lock_init(&class->lock); off = obj_idx_to_offset(page, obj_idx, class->size);
pool->size_class[i] = class;
prev_class = class; area = &get_cpu_var(zs_map_area);
area->vm_mm = mm;
if (off + class->size <= PAGE_SIZE) {
/* this object is contained entirely within a page */
area->vm_addr = kmap_atomic(page);
return area->vm_addr + off;
} }
pool->flags = flags; /* this object spans two pages */
pages[0] = page;
return pool; pages[1] = get_next_page(page);
BUG_ON(!pages[1]);
err: return __zs_map_object(area, pages, off, class->size);
zs_destroy_pool(pool);
return NULL;
} }
EXPORT_SYMBOL_GPL(zs_create_pool); EXPORT_SYMBOL_GPL(zs_map_object);
void zs_destroy_pool(struct zs_pool *pool) void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
{ {
int i; struct page *page;
unsigned long obj_idx, off;
for (i = 0; i < zs_size_classes; i++) { unsigned int class_idx;
int fg; enum fullness_group fg;
struct size_class *class = pool->size_class[i]; struct size_class *class;
struct mapping_area *area;
if (!class) BUG_ON(!handle);
continue;
if (class->index != i) obj_handle_to_location(handle, &page, &obj_idx);
continue; get_zspage_mapping(get_first_page(page), &class_idx, &fg);
class = pool->size_class[class_idx];
off = obj_idx_to_offset(page, obj_idx, class->size);
for (fg = 0; fg < _ZS_NR_FULLNESS_GROUPS; fg++) { area = this_cpu_ptr(&zs_map_area);
if (class->fullness_list[fg]) { if (off + class->size <= PAGE_SIZE)
pr_info("Freeing non-empty class with size %db, fullness group %d\n", kunmap_atomic(area->vm_addr);
class->size, fg); else {
} struct page *pages[2];
}
kfree(class);
}
kfree(pool->size_class); pages[0] = page;
kfree(pool); pages[1] = get_next_page(page);
BUG_ON(!pages[1]);
__zs_unmap_object(area, pages, off, class->size);
}
put_cpu_var(zs_map_area);
} }
EXPORT_SYMBOL_GPL(zs_destroy_pool); EXPORT_SYMBOL_GPL(zs_unmap_object);
/** /**
* zs_malloc - Allocate block of given size from pool. * zs_malloc - Allocate block of given size from pool.
...@@ -1176,100 +1139,137 @@ void zs_free(struct zs_pool *pool, unsigned long obj) ...@@ -1176,100 +1139,137 @@ void zs_free(struct zs_pool *pool, unsigned long obj)
EXPORT_SYMBOL_GPL(zs_free); EXPORT_SYMBOL_GPL(zs_free);
/** /**
* zs_map_object - get address of allocated object from handle. * zs_create_pool - Creates an allocation pool to work from.
* @pool: pool from which the object was allocated * @flags: allocation flags used to allocate pool metadata
* @handle: handle returned from zs_malloc
*
* Before using an object allocated from zs_malloc, it must be mapped using
* this function. When done with the object, it must be unmapped using
* zs_unmap_object.
* *
* Only one object can be mapped per cpu at a time. There is no protection * This function must be called before anything when using
* against nested mappings. * the zsmalloc allocator.
* *
* This function returns with preemption and page faults disabled. * On success, a pointer to the newly created pool is returned,
* otherwise NULL.
*/ */
void *zs_map_object(struct zs_pool *pool, unsigned long handle, struct zs_pool *zs_create_pool(gfp_t flags)
enum zs_mapmode mm)
{ {
struct page *page; int i;
unsigned long obj_idx, off; struct zs_pool *pool;
struct size_class *prev_class = NULL;
unsigned int class_idx; pool = kzalloc(sizeof(*pool), GFP_KERNEL);
enum fullness_group fg; if (!pool)
return NULL;
pool->size_class = kcalloc(zs_size_classes, sizeof(struct size_class *),
GFP_KERNEL);
if (!pool->size_class) {
kfree(pool);
return NULL;
}
/*
* Iterate reversly, because, size of size_class that we want to use
* for merging should be larger or equal to current size.
*/
for (i = zs_size_classes - 1; i >= 0; i--) {
int size;
int pages_per_zspage;
struct size_class *class; struct size_class *class;
struct mapping_area *area;
struct page *pages[2];
BUG_ON(!handle); size = ZS_MIN_ALLOC_SIZE + i * ZS_SIZE_CLASS_DELTA;
if (size > ZS_MAX_ALLOC_SIZE)
size = ZS_MAX_ALLOC_SIZE;
pages_per_zspage = get_pages_per_zspage(size);
/* /*
* Because we use per-cpu mapping areas shared among the * size_class is used for normal zsmalloc operation such
* pools/users, we can't allow mapping in interrupt context * as alloc/free for that size. Although it is natural that we
* because it can corrupt another users mappings. * have one size_class for each size, there is a chance that we
* can get more memory utilization if we use one size_class for
* many different sizes whose size_class have same
* characteristics. So, we makes size_class point to
* previous size_class if possible.
*/ */
BUG_ON(in_interrupt()); if (prev_class) {
if (can_merge(prev_class, size, pages_per_zspage)) {
pool->size_class[i] = prev_class;
continue;
}
}
obj_handle_to_location(handle, &page, &obj_idx); class = kzalloc(sizeof(struct size_class), GFP_KERNEL);
get_zspage_mapping(get_first_page(page), &class_idx, &fg); if (!class)
class = pool->size_class[class_idx]; goto err;
off = obj_idx_to_offset(page, obj_idx, class->size);
area = &get_cpu_var(zs_map_area); class->size = size;
area->vm_mm = mm; class->index = i;
if (off + class->size <= PAGE_SIZE) { class->pages_per_zspage = pages_per_zspage;
/* this object is contained entirely within a page */ spin_lock_init(&class->lock);
area->vm_addr = kmap_atomic(page); pool->size_class[i] = class;
return area->vm_addr + off;
prev_class = class;
} }
/* this object spans two pages */ pool->flags = flags;
pages[0] = page;
pages[1] = get_next_page(page);
BUG_ON(!pages[1]);
return __zs_map_object(area, pages, off, class->size); return pool;
err:
zs_destroy_pool(pool);
return NULL;
} }
EXPORT_SYMBOL_GPL(zs_map_object); EXPORT_SYMBOL_GPL(zs_create_pool);
void zs_unmap_object(struct zs_pool *pool, unsigned long handle) void zs_destroy_pool(struct zs_pool *pool)
{ {
struct page *page; int i;
unsigned long obj_idx, off;
unsigned int class_idx; for (i = 0; i < zs_size_classes; i++) {
enum fullness_group fg; int fg;
struct size_class *class; struct size_class *class = pool->size_class[i];
struct mapping_area *area;
BUG_ON(!handle); if (!class)
continue;
obj_handle_to_location(handle, &page, &obj_idx); if (class->index != i)
get_zspage_mapping(get_first_page(page), &class_idx, &fg); continue;
class = pool->size_class[class_idx];
off = obj_idx_to_offset(page, obj_idx, class->size);
area = this_cpu_ptr(&zs_map_area); for (fg = 0; fg < _ZS_NR_FULLNESS_GROUPS; fg++) {
if (off + class->size <= PAGE_SIZE) if (class->fullness_list[fg]) {
kunmap_atomic(area->vm_addr); pr_info("Freeing non-empty class with size %db, fullness group %d\n",
else { class->size, fg);
struct page *pages[2]; }
}
kfree(class);
}
pages[0] = page; kfree(pool->size_class);
pages[1] = get_next_page(page); kfree(pool);
BUG_ON(!pages[1]); }
EXPORT_SYMBOL_GPL(zs_destroy_pool);
__zs_unmap_object(area, pages, off, class->size); static int __init zs_init(void)
{
int ret = zs_register_cpu_notifier();
if (ret) {
zs_unregister_cpu_notifier();
return ret;
} }
put_cpu_var(zs_map_area);
init_zs_size_classes();
#ifdef CONFIG_ZPOOL
zpool_register_driver(&zs_zpool_driver);
#endif
return 0;
} }
EXPORT_SYMBOL_GPL(zs_unmap_object);
unsigned long zs_get_total_pages(struct zs_pool *pool) static void __exit zs_exit(void)
{ {
return atomic_long_read(&pool->pages_allocated); #ifdef CONFIG_ZPOOL
zpool_unregister_driver(&zs_zpool_driver);
#endif
zs_unregister_cpu_notifier();
} }
EXPORT_SYMBOL_GPL(zs_get_total_pages);
module_init(zs_init); module_init(zs_init);
module_exit(zs_exit); module_exit(zs_exit);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment