Commit ede93213 authored by Vitaly Wool's avatar Vitaly Wool Committed by Linus Torvalds

z3fold: fix header size related issues

Currently the whole kernel build will be stopped if the size of struct
z3fold_header is greater than the size of one chunk, which is 64 bytes
by default.  This patch instead defines the offset for z3fold objects as
the size of the z3fold header in chunks.

Fixed also are the calculation of num_free_chunks() and the address to
move the middle chunk to in case of in-page compaction in
z3fold_compact_page().

Link: http://lkml.kernel.org/r/20170131214057.d98677032bc7b1c6c59a80c9@gmail.comSigned-off-by: default avatarVitaly Wool <vitalywool@gmail.com>
Reviewed-by: default avatarDan Streetman <ddstreet@ieee.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 12d59ae6
...@@ -34,29 +34,58 @@ ...@@ -34,29 +34,58 @@
/***************** /*****************
* Structures * Structures
*****************/ *****************/
struct z3fold_pool;
struct z3fold_ops {
int (*evict)(struct z3fold_pool *pool, unsigned long handle);
};
enum buddy {
HEADLESS = 0,
FIRST,
MIDDLE,
LAST,
BUDDIES_MAX
};
/*
* struct z3fold_header - z3fold page metadata occupying the first chunk of each
* z3fold page, except for HEADLESS pages
* @buddy: links the z3fold page into the relevant list in the pool
* @first_chunks: the size of the first buddy in chunks, 0 if free
* @middle_chunks: the size of the middle buddy in chunks, 0 if free
* @last_chunks: the size of the last buddy in chunks, 0 if free
* @first_num: the starting number (for the first handle)
*/
struct z3fold_header {
struct list_head buddy;
unsigned short first_chunks;
unsigned short middle_chunks;
unsigned short last_chunks;
unsigned short start_middle;
unsigned short first_num:2;
};
/* /*
* NCHUNKS_ORDER determines the internal allocation granularity, effectively * NCHUNKS_ORDER determines the internal allocation granularity, effectively
* adjusting internal fragmentation. It also determines the number of * adjusting internal fragmentation. It also determines the number of
* freelists maintained in each pool. NCHUNKS_ORDER of 6 means that the * freelists maintained in each pool. NCHUNKS_ORDER of 6 means that the
* allocation granularity will be in chunks of size PAGE_SIZE/64. As one chunk * allocation granularity will be in chunks of size PAGE_SIZE/64. Some chunks
* in allocated page is occupied by z3fold header, NCHUNKS will be calculated * in the beginning of an allocated page are occupied by z3fold header, so
* to 63 which shows the max number of free chunks in z3fold page, also there * NCHUNKS will be calculated to 63 (or 62 in case CONFIG_DEBUG_SPINLOCK=y),
* will be 63 freelists per pool. * which shows the max number of free chunks in z3fold page, also there will
* be 63, or 62, respectively, freelists per pool.
*/ */
#define NCHUNKS_ORDER 6 #define NCHUNKS_ORDER 6
#define CHUNK_SHIFT (PAGE_SHIFT - NCHUNKS_ORDER) #define CHUNK_SHIFT (PAGE_SHIFT - NCHUNKS_ORDER)
#define CHUNK_SIZE (1 << CHUNK_SHIFT) #define CHUNK_SIZE (1 << CHUNK_SHIFT)
#define ZHDR_SIZE_ALIGNED CHUNK_SIZE #define ZHDR_SIZE_ALIGNED round_up(sizeof(struct z3fold_header), CHUNK_SIZE)
#define ZHDR_CHUNKS (ZHDR_SIZE_ALIGNED >> CHUNK_SHIFT)
#define TOTAL_CHUNKS (PAGE_SIZE >> CHUNK_SHIFT)
#define NCHUNKS ((PAGE_SIZE - ZHDR_SIZE_ALIGNED) >> CHUNK_SHIFT) #define NCHUNKS ((PAGE_SIZE - ZHDR_SIZE_ALIGNED) >> CHUNK_SHIFT)
#define BUDDY_MASK (0x3) #define BUDDY_MASK (0x3)
struct z3fold_pool;
struct z3fold_ops {
int (*evict)(struct z3fold_pool *pool, unsigned long handle);
};
/** /**
* struct z3fold_pool - stores metadata for each z3fold pool * struct z3fold_pool - stores metadata for each z3fold pool
* @lock: protects all pool fields and first|last_chunk fields of any * @lock: protects all pool fields and first|last_chunk fields of any
...@@ -86,32 +115,6 @@ struct z3fold_pool { ...@@ -86,32 +115,6 @@ struct z3fold_pool {
const struct zpool_ops *zpool_ops; const struct zpool_ops *zpool_ops;
}; };
enum buddy {
HEADLESS = 0,
FIRST,
MIDDLE,
LAST,
BUDDIES_MAX
};
/*
* struct z3fold_header - z3fold page metadata occupying the first chunk of each
* z3fold page, except for HEADLESS pages
* @buddy: links the z3fold page into the relevant list in the pool
* @first_chunks: the size of the first buddy in chunks, 0 if free
* @middle_chunks: the size of the middle buddy in chunks, 0 if free
* @last_chunks: the size of the last buddy in chunks, 0 if free
* @first_num: the starting number (for the first handle)
*/
struct z3fold_header {
struct list_head buddy;
unsigned short first_chunks;
unsigned short middle_chunks;
unsigned short last_chunks;
unsigned short start_middle;
unsigned short first_num:2;
};
/* /*
* Internal z3fold page flags * Internal z3fold page flags
*/ */
...@@ -121,6 +124,7 @@ enum z3fold_page_flags { ...@@ -121,6 +124,7 @@ enum z3fold_page_flags {
MIDDLE_CHUNK_MAPPED, MIDDLE_CHUNK_MAPPED,
}; };
/***************** /*****************
* Helpers * Helpers
*****************/ *****************/
...@@ -204,9 +208,10 @@ static int num_free_chunks(struct z3fold_header *zhdr) ...@@ -204,9 +208,10 @@ static int num_free_chunks(struct z3fold_header *zhdr)
*/ */
if (zhdr->middle_chunks != 0) { if (zhdr->middle_chunks != 0) {
int nfree_before = zhdr->first_chunks ? int nfree_before = zhdr->first_chunks ?
0 : zhdr->start_middle - 1; 0 : zhdr->start_middle - ZHDR_CHUNKS;
int nfree_after = zhdr->last_chunks ? int nfree_after = zhdr->last_chunks ?
0 : NCHUNKS - zhdr->start_middle - zhdr->middle_chunks; 0 : TOTAL_CHUNKS -
(zhdr->start_middle + zhdr->middle_chunks);
nfree = max(nfree_before, nfree_after); nfree = max(nfree_before, nfree_after);
} else } else
nfree = NCHUNKS - zhdr->first_chunks - zhdr->last_chunks; nfree = NCHUNKS - zhdr->first_chunks - zhdr->last_chunks;
...@@ -254,26 +259,35 @@ static void z3fold_destroy_pool(struct z3fold_pool *pool) ...@@ -254,26 +259,35 @@ static void z3fold_destroy_pool(struct z3fold_pool *pool)
kfree(pool); kfree(pool);
} }
static inline void *mchunk_memmove(struct z3fold_header *zhdr,
unsigned short dst_chunk)
{
void *beg = zhdr;
return memmove(beg + (dst_chunk << CHUNK_SHIFT),
beg + (zhdr->start_middle << CHUNK_SHIFT),
zhdr->middle_chunks << CHUNK_SHIFT);
}
/* Has to be called with lock held */ /* Has to be called with lock held */
static int z3fold_compact_page(struct z3fold_header *zhdr) static int z3fold_compact_page(struct z3fold_header *zhdr)
{ {
struct page *page = virt_to_page(zhdr); struct page *page = virt_to_page(zhdr);
void *beg = zhdr;
if (test_bit(MIDDLE_CHUNK_MAPPED, &page->private))
return 0; /* can't move middle chunk, it's used */
if (!test_bit(MIDDLE_CHUNK_MAPPED, &page->private) && if (zhdr->middle_chunks == 0)
zhdr->middle_chunks != 0 && return 0; /* nothing to compact */
zhdr->first_chunks == 0 && zhdr->last_chunks == 0) {
memmove(beg + ZHDR_SIZE_ALIGNED, if (zhdr->first_chunks == 0 && zhdr->last_chunks == 0) {
beg + (zhdr->start_middle << CHUNK_SHIFT), /* move to the beginning */
zhdr->middle_chunks << CHUNK_SHIFT); mchunk_memmove(zhdr, ZHDR_CHUNKS);
zhdr->first_chunks = zhdr->middle_chunks; zhdr->first_chunks = zhdr->middle_chunks;
zhdr->middle_chunks = 0; zhdr->middle_chunks = 0;
zhdr->start_middle = 0; zhdr->start_middle = 0;
zhdr->first_num++; zhdr->first_num++;
return 1;
} }
return 0; return 1;
} }
/** /**
...@@ -365,7 +379,7 @@ static int z3fold_alloc(struct z3fold_pool *pool, size_t size, gfp_t gfp, ...@@ -365,7 +379,7 @@ static int z3fold_alloc(struct z3fold_pool *pool, size_t size, gfp_t gfp,
zhdr->last_chunks = chunks; zhdr->last_chunks = chunks;
else { else {
zhdr->middle_chunks = chunks; zhdr->middle_chunks = chunks;
zhdr->start_middle = zhdr->first_chunks + 1; zhdr->start_middle = zhdr->first_chunks + ZHDR_CHUNKS;
} }
if (zhdr->first_chunks == 0 || zhdr->last_chunks == 0 || if (zhdr->first_chunks == 0 || zhdr->last_chunks == 0 ||
...@@ -778,8 +792,8 @@ MODULE_ALIAS("zpool-z3fold"); ...@@ -778,8 +792,8 @@ MODULE_ALIAS("zpool-z3fold");
static int __init init_z3fold(void) static int __init init_z3fold(void)
{ {
/* Make sure the z3fold header will fit in one chunk */ /* Make sure the z3fold header is not larger than the page size */
BUILD_BUG_ON(sizeof(struct z3fold_header) > ZHDR_SIZE_ALIGNED); BUILD_BUG_ON(ZHDR_SIZE_ALIGNED > PAGE_SIZE);
zpool_register_driver(&z3fold_zpool_driver); zpool_register_driver(&z3fold_zpool_driver);
return 0; return 0;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment