Skip to content

L
linux
Commit 97e86a85 authored by Gao Xiang's avatar Gao Xiang Committed by Greg Kroah-Hartman
Browse files
Options

staging: erofs: tidy up decompression frontend 

Although this patch has an amount of changes, it is hard to
separate into smaller patches.

Most changes are due to structure renaming for better understand
and straightforward,
 z_erofs_vle_workgroup to z_erofs_pcluster
             since it represents a physical cluster;
 z_erofs_vle_work to z_erofs_collection
             since it represents a collection of logical pages;
 z_erofs_vle_work_builder to z_erofs_collector
             since it's used to fill z_erofs_{pcluster,collection}.

struct z_erofs_vle_work_finder has no extra use compared with
struct z_erofs_collector, delete it.

FULL_LENGTH bit is integrated into .length of pcluster so that it
can be updated with the corresponding length change in atomic.

Minor, add comments for better description.
Reviewed-by: default avatarChao Yu <yuchao0@huawei.com>
Signed-off-by: default avatarGao Xiang <gaoxiang25@huawei.com>
Link: https://lore.kernel.org/r/20190731155752.210602-18-gaoxiang25@huawei.comSigned-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent febde7b3
Hide whitespace changes
Inline Side-by-side
Showing with 465 additions and 614 deletions
drivers/staging/erofs/zdata.c
View file @ 97e86a85
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
*/ */
#define PAGE_UNALLOCATED ((void *)0x5F0E4B1D) #define PAGE_UNALLOCATED ((void *)0x5F0E4B1D)
/* how to allocate cached pages for a workgroup */ /* how to allocate cached pages for a pcluster */
enum z_erofs_cache_alloctype { enum z_erofs_cache_alloctype {
DONTALLOC, /* don't allocate any cached pages */ DONTALLOC, /* don't allocate any cached pages */
DELAYEDALLOC, /* delayed allocation (at the time of submitting io) */ DELAYEDALLOC, /* delayed allocation (at the time of submitting io) */
...@@ -34,156 +34,158 @@ typedef tagptr1_t compressed_page_t; ...@@ -34,156 +34,158 @@ typedef tagptr1_t compressed_page_t;
tagptr_fold(compressed_page_t, page, 1) tagptr_fold(compressed_page_t, page, 1)
static struct workqueue_struct *z_erofs_workqueue __read_mostly; static struct workqueue_struct *z_erofs_workqueue __read_mostly;
static struct kmem_cache *z_erofs_workgroup_cachep __read_mostly; static struct kmem_cache *pcluster_cachep __read_mostly;
void z_erofs_exit_zip_subsystem(void) void z_erofs_exit_zip_subsystem(void)
{ {
destroy_workqueue(z_erofs_workqueue); destroy_workqueue(z_erofs_workqueue);
kmem_cache_destroy(z_erofs_workgroup_cachep); kmem_cache_destroy(pcluster_cachep);
} }
static inline int init_unzip_workqueue(void) static inline int init_unzip_workqueue(void)
{ {
const unsigned int onlinecpus = num_possible_cpus(); const unsigned int onlinecpus = num_possible_cpus();
const unsigned int flags = WQ_UNBOUND | WQ_HIGHPRI | WQ_CPU_INTENSIVE;
/* /*
* we don't need too many threads, limiting threads * no need to spawn too many threads, limiting threads could minimum
* could improve scheduling performance. * scheduling overhead, perhaps per-CPU threads should be better?
*/ */
z_erofs_workqueue = z_erofs_workqueue = alloc_workqueue("erofs_unzipd", flags,
alloc_workqueue("erofs_unzipd", onlinecpus + onlinecpus / 4);
WQ_UNBOUND | WQ_HIGHPRI | WQ_CPU_INTENSIVE,
onlinecpus + onlinecpus / 4);
return z_erofs_workqueue ? 0 : -ENOMEM; return z_erofs_workqueue ? 0 : -ENOMEM;
} }
static void init_once(void *ptr) static void init_once(void *ptr)
{ {
struct z_erofs_vle_workgroup *grp = ptr; struct z_erofs_pcluster *pcl = ptr;
struct z_erofs_vle_work *const work = struct z_erofs_collection *cl = z_erofs_primarycollection(pcl);
z_erofs_vle_grab_primary_work(grp);
unsigned int i; unsigned int i;
mutex_init(&work->lock); mutex_init(&cl->lock);
work->nr_pages = 0; cl->nr_pages = 0;
work->vcnt = 0; cl->vcnt = 0;
for (i = 0; i < Z_EROFS_CLUSTER_MAX_PAGES; ++i) for (i = 0; i < Z_EROFS_CLUSTER_MAX_PAGES; ++i)
grp->compressed_pages[i] = NULL; pcl->compressed_pages[i] = NULL;
} }
static void init_always(struct z_erofs_vle_workgroup *grp) static void init_always(struct z_erofs_pcluster *pcl)
{ {
struct z_erofs_vle_work *const work = struct z_erofs_collection *cl = z_erofs_primarycollection(pcl);
z_erofs_vle_grab_primary_work(grp);
atomic_set(&grp->obj.refcount, 1); atomic_set(&pcl->obj.refcount, 1);
grp->flags = 0;
DBG_BUGON(work->nr_pages); DBG_BUGON(cl->nr_pages);
DBG_BUGON(work->vcnt); DBG_BUGON(cl->vcnt);
} }
int __init z_erofs_init_zip_subsystem(void) int __init z_erofs_init_zip_subsystem(void)
{ {
z_erofs_workgroup_cachep = pcluster_cachep = kmem_cache_create("erofs_compress",
kmem_cache_create("erofs_compress", Z_EROFS_WORKGROUP_SIZE, 0,
Z_EROFS_WORKGROUP_SIZE, 0, SLAB_RECLAIM_ACCOUNT, init_once);
SLAB_RECLAIM_ACCOUNT, init_once); if (pcluster_cachep) {
if (z_erofs_workgroup_cachep) {
if (!init_unzip_workqueue()) if (!init_unzip_workqueue())
return 0; return 0;
kmem_cache_destroy(z_erofs_workgroup_cachep); kmem_cache_destroy(pcluster_cachep);
} }
return -ENOMEM; return -ENOMEM;
} }
enum z_erofs_vle_work_role { enum z_erofs_collectmode {
Z_EROFS_VLE_WORK_SECONDARY, COLLECT_SECONDARY,
Z_EROFS_VLE_WORK_PRIMARY, COLLECT_PRIMARY,
/* /*
* The current work was the tail of an exist chain, and the previous * The current collection was the tail of an exist chain, in addition
* processed chained works are all decided to be hooked up to it. * that the previous processed chained collections are all decided to
* A new chain should be created for the remaining unprocessed works, * be hooked up to it.
* therefore different from Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED, * A new chain will be created for the remaining collections which are
* the next work cannot reuse the whole page in the following scenario: * not processed yet, therefore different from COLLECT_PRIMARY_FOLLOWED,
* the next collection cannot reuse the whole page safely in
* the following scenario:
* ________________________________________________________________ * ________________________________________________________________
* | tail (partial) page | head (partial) page | * | tail (partial) page | head (partial) page |
* | (belongs to the next work) | (belongs to the current work) | * | (belongs to the next cl) | (belongs to the current cl) |
* |_______PRIMARY_FOLLOWED_______|________PRIMARY_HOOKED___________| * |_______PRIMARY_FOLLOWED_______|________PRIMARY_HOOKED___________|
*/ */
Z_EROFS_VLE_WORK_PRIMARY_HOOKED, COLLECT_PRIMARY_HOOKED,
COLLECT_PRIMARY_FOLLOWED_NOINPLACE,
/* /*
* The current work has been linked with the processed chained works, * The current collection has been linked with the owned chain, and
* and could be also linked with the potential remaining works, which * could also be linked with the remaining collections, which means
* means if the processing page is the tail partial page of the work, * if the processing page is the tail page of the collection, thus
* the current work can safely use the whole page (since the next work * the current collection can safely use the whole page (since
* is under control) for in-place decompression, as illustrated below: * the previous collection is under control) for in-place I/O, as
* illustrated below:
* ________________________________________________________________ * ________________________________________________________________
* | tail (partial) page | head (partial) page | * | tail (partial) page | head (partial) page |
* | (of the current work) | (of the previous work) | * | (of the current cl) | (of the previous collection) |
* | PRIMARY_FOLLOWED or | | * | PRIMARY_FOLLOWED or | |
* |_____PRIMARY_HOOKED____|____________PRIMARY_FOLLOWED____________| * |_____PRIMARY_HOOKED___|____________PRIMARY_FOLLOWED____________|
* *
* [ (*) the above page can be used for the current work itself. ] * [ (*) the above page can be used as inplace I/O. ]
*/ */
Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED, COLLECT_PRIMARY_FOLLOWED,
Z_EROFS_VLE_WORK_MAX
}; };
struct z_erofs_vle_work_builder { struct z_erofs_collector {
enum z_erofs_vle_work_role role;
/*
* 'hosted = false' means that the current workgroup doesn't belong to
* the owned chained workgroups. In the other words, it is none of our
* business to submit this workgroup.
*/
bool hosted;
struct z_erofs_vle_workgroup *grp;
struct z_erofs_vle_work *work;
struct z_erofs_pagevec_ctor vector; struct z_erofs_pagevec_ctor vector;
/* pages used for reading the compressed data */ struct z_erofs_pcluster *pcl;
struct page **compressed_pages; struct z_erofs_collection *cl;
unsigned int compressed_deficit; struct page **compressedpages;
z_erofs_next_pcluster_t owned_head;
enum z_erofs_collectmode mode;
}; };
#define VLE_WORK_BUILDER_INIT() \ struct z_erofs_decompress_frontend {
{ .work = NULL, .role = Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED } struct inode *const inode;
struct z_erofs_collector clt;
struct erofs_map_blocks map;
/* used for applying cache strategy on the fly */
bool backmost;
erofs_off_t headoffset;
};
#define COLLECTOR_INIT() { \
.owned_head = Z_EROFS_PCLUSTER_TAIL, \
.mode = COLLECT_PRIMARY_FOLLOWED }
#define DECOMPRESS_FRONTEND_INIT(__i) { \
.inode = __i, .clt = COLLECTOR_INIT(), \
.backmost = true, }
static struct page *z_pagemap_global[Z_EROFS_VMAP_GLOBAL_PAGES];
static DEFINE_MUTEX(z_pagemap_global_lock);
#ifdef EROFS_FS_HAS_MANAGED_CACHE #ifdef EROFS_FS_HAS_MANAGED_CACHE
static void preload_compressed_pages(struct z_erofs_vle_work_builder *bl, static void preload_compressed_pages(struct z_erofs_collector *clt,
struct address_space *mc, struct address_space *mc,
pgoff_t index,
unsigned int clusterpages,
enum z_erofs_cache_alloctype type, enum z_erofs_cache_alloctype type,
struct list_head *pagepool, struct list_head *pagepool)
gfp_t gfp)
{ {
struct page **const pages = bl->compressed_pages; const struct z_erofs_pcluster *pcl = clt->pcl;
const unsigned int remaining = bl->compressed_deficit; const unsigned int clusterpages = BIT(pcl->clusterbits);
struct page **pages = clt->compressedpages;
pgoff_t index = pcl->obj.index + (pages - pcl->compressed_pages);
bool standalone = true; bool standalone = true;
unsigned int i, j = 0;
if (bl->role < Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED) if (clt->mode < COLLECT_PRIMARY_FOLLOWED)
return; return;
gfp = mapping_gfp_constraint(mc, gfp) & ~__GFP_RECLAIM; for (; pages < pcl->compressed_pages + clusterpages; ++pages) {
index += clusterpages - remaining;
for (i = 0; i < remaining; ++i) {
struct page *page; struct page *page;
compressed_page_t t; compressed_page_t t;
/* the compressed page was loaded before */ /* the compressed page was loaded before */
if (READ_ONCE(pages[i])) if (READ_ONCE(*pages))
continue; continue;
page = find_get_page(mc, index + i); page = find_get_page(mc, index);
if (page) { if (page) {
t = tag_compressed_page_justfound(page); t = tag_compressed_page_justfound(page);
...@@ -191,32 +193,30 @@ static void preload_compressed_pages(struct z_erofs_vle_work_builder *bl, ...@@ -191,32 +193,30 @@ static void preload_compressed_pages(struct z_erofs_vle_work_builder *bl,
t = tagptr_init(compressed_page_t, PAGE_UNALLOCATED); t = tagptr_init(compressed_page_t, PAGE_UNALLOCATED);
} else { /* DONTALLOC */ } else { /* DONTALLOC */
if (standalone) if (standalone)
j = i; clt->compressedpages = pages;
standalone = false; standalone = false;
continue; continue;
} }
if (!cmpxchg_relaxed(&pages[i], NULL, tagptr_cast_ptr(t))) if (!cmpxchg_relaxed(pages, NULL, tagptr_cast_ptr(t)))
continue; continue;
if (page) if (page)
put_page(page); put_page(page);
} }
bl->compressed_pages += j;
bl->compressed_deficit = remaining - j;
if (standalone) if (standalone) /* downgrade to PRIMARY_FOLLOWED_NOINPLACE */
bl->role = Z_EROFS_VLE_WORK_PRIMARY; clt->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE;
} }
/* called by erofs_shrinker to get rid of all compressed_pages */ /* called by erofs_shrinker to get rid of all compressed_pages */
int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
struct erofs_workgroup *egrp) struct erofs_workgroup *grp)
{ {
struct z_erofs_vle_workgroup *const grp = struct z_erofs_pcluster *const pcl =
container_of(egrp, struct z_erofs_vle_workgroup, obj); container_of(grp, struct z_erofs_pcluster, obj);
struct address_space *const mapping = MNGD_MAPPING(sbi); struct address_space *const mapping = MNGD_MAPPING(sbi);
const int clusterpages = erofs_clusterpages(sbi); const unsigned int clusterpages = BIT(pcl->clusterbits);
int i; int i;
/* /*
...@@ -224,18 +224,20 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, ...@@ -224,18 +224,20 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
* therefore no need to worry about available decompression users. * therefore no need to worry about available decompression users.
*/ */
for (i = 0; i < clusterpages; ++i) { for (i = 0; i < clusterpages; ++i) {
struct page *page = grp->compressed_pages[i]; struct page *page = pcl->compressed_pages[i];
if (!page || page->mapping != mapping) if (!page)
continue; continue;
/* block other users from reclaiming or migrating the page */ /* block other users from reclaiming or migrating the page */
if (!trylock_page(page)) if (!trylock_page(page))
return -EBUSY; return -EBUSY;
/* barrier is implied in the following 'unlock_page' */ if (unlikely(page->mapping != mapping))
WRITE_ONCE(grp->compressed_pages[i], NULL); continue;
/* barrier is implied in the following 'unlock_page' */
WRITE_ONCE(pcl->compressed_pages[i], NULL);
set_page_private(page, 0); set_page_private(page, 0);
ClearPagePrivate(page); ClearPagePrivate(page);
...@@ -248,22 +250,21 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, ...@@ -248,22 +250,21 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
int erofs_try_to_free_cached_page(struct address_space *mapping, int erofs_try_to_free_cached_page(struct address_space *mapping,
struct page *page) struct page *page)
{ {
struct erofs_sb_info *const sbi = EROFS_SB(mapping->host->i_sb); struct z_erofs_pcluster *const pcl = (void *)page_private(page);
const unsigned int clusterpages = erofs_clusterpages(sbi); const unsigned int clusterpages = BIT(pcl->clusterbits);
struct z_erofs_vle_workgroup *const grp = (void *)page_private(page);
int ret = 0; /* 0 - busy */ int ret = 0; /* 0 - busy */
if (erofs_workgroup_try_to_freeze(&grp->obj, 1)) { if (erofs_workgroup_try_to_freeze(&pcl->obj, 1)) {
unsigned int i; unsigned int i;
for (i = 0; i < clusterpages; ++i) { for (i = 0; i < clusterpages; ++i) {
if (grp->compressed_pages[i] == page) { if (pcl->compressed_pages[i] == page) {
WRITE_ONCE(grp->compressed_pages[i], NULL); WRITE_ONCE(pcl->compressed_pages[i], NULL);
ret = 1; ret = 1;
break; break;
} }
} }
erofs_workgroup_unfreeze(&grp->obj, 1); erofs_workgroup_unfreeze(&pcl->obj, 1);
if (ret) { if (ret) {
ClearPagePrivate(page); ClearPagePrivate(page);
...@@ -273,361 +274,267 @@ int erofs_try_to_free_cached_page(struct address_space *mapping, ...@@ -273,361 +274,267 @@ int erofs_try_to_free_cached_page(struct address_space *mapping,
return ret; return ret;
} }
#else #else
static void preload_compressed_pages(struct z_erofs_vle_work_builder *bl, static void preload_compressed_pages(struct z_erofs_collector *clt,
struct address_space *mc, struct address_space *mc,
pgoff_t index,
unsigned int clusterpages,
enum z_erofs_cache_alloctype type, enum z_erofs_cache_alloctype type,
struct list_head *pagepool, struct list_head *pagepool)
gfp_t gfp)
{ {
/* nowhere to load compressed pages from */ /* nowhere to load compressed pages from */
} }
#endif #endif
/* page_type must be Z_EROFS_PAGE_TYPE_EXCLUSIVE */ /* page_type must be Z_EROFS_PAGE_TYPE_EXCLUSIVE */
static inline bool try_to_reuse_as_compressed_page( static inline bool try_inplace_io(struct z_erofs_collector *clt,
struct z_erofs_vle_work_builder *b, struct page *page)
struct page *page)
{ {
while (b->compressed_deficit) { struct z_erofs_pcluster *const pcl = clt->pcl;
--b->compressed_deficit; const unsigned int clusterpages = BIT(pcl->clusterbits);
if (!cmpxchg(b->compressed_pages++, NULL, page))
while (clt->compressedpages < pcl->compressed_pages + clusterpages) {
if (!cmpxchg(clt->compressedpages++, NULL, page))
return true; return true;
} }
return false; return false;
} }
/* callers must be with work->lock held */ /* callers must be with collection lock held */
static int z_erofs_vle_work_add_page( static int z_erofs_attach_page(struct z_erofs_collector *clt,
struct z_erofs_vle_work_builder *builder, struct page *page,
struct page *page, enum z_erofs_page_type type)
enum z_erofs_page_type type)
{ {
int ret; int ret;
bool occupied; bool occupied;
/* give priority for the compressed data storage */ /* give priority for inplaceio */
if (builder->role >= Z_EROFS_VLE_WORK_PRIMARY && if (clt->mode >= COLLECT_PRIMARY &&
type == Z_EROFS_PAGE_TYPE_EXCLUSIVE && type == Z_EROFS_PAGE_TYPE_EXCLUSIVE &&
try_to_reuse_as_compressed_page(builder, page)) try_inplace_io(clt, page))
return 0; return 0;
ret = z_erofs_pagevec_enqueue(&builder->vector, ret = z_erofs_pagevec_enqueue(&clt->vector,
page, type, &occupied); page, type, &occupied);
builder->work->vcnt += (unsigned int)ret; clt->cl->vcnt += (unsigned int)ret;
return ret ? 0 : -EAGAIN; return ret ? 0 : -EAGAIN;
} }
static enum z_erofs_vle_work_role static enum z_erofs_collectmode
try_to_claim_workgroup(struct z_erofs_vle_workgroup *grp, try_to_claim_pcluster(struct z_erofs_pcluster *pcl,
z_erofs_vle_owned_workgrp_t *owned_head, z_erofs_next_pcluster_t *owned_head)
bool *hosted)
{ {
DBG_BUGON(*hosted); /* let's claim these following types of pclusters */
/* let's claim these following types of workgroup */
retry: retry:
if (grp->next == Z_EROFS_VLE_WORKGRP_NIL) { if (pcl->next == Z_EROFS_PCLUSTER_NIL) {
/* type 1, nil workgroup */ /* type 1, nil pcluster */
if (cmpxchg(&grp->next, Z_EROFS_VLE_WORKGRP_NIL, if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL,
*owned_head) != Z_EROFS_VLE_WORKGRP_NIL) *owned_head) != Z_EROFS_PCLUSTER_NIL)
goto retry; goto retry;
*owned_head = &grp->next; *owned_head = &pcl->next;
*hosted = true;
/* lucky, I am the followee :) */ /* lucky, I am the followee :) */
return Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED; return COLLECT_PRIMARY_FOLLOWED;
} else if (pcl->next == Z_EROFS_PCLUSTER_TAIL) {
} else if (grp->next == Z_EROFS_VLE_WORKGRP_TAIL) {
/* /*
* type 2, link to the end of a existing open chain, * type 2, link to the end of a existing open chain,
* be careful that its submission itself is governed * be careful that its submission itself is governed
* by the original owned chain. * by the original owned chain.
*/ */
if (cmpxchg(&grp->next, Z_EROFS_VLE_WORKGRP_TAIL, if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,
*owned_head) != Z_EROFS_VLE_WORKGRP_TAIL) *owned_head) != Z_EROFS_PCLUSTER_TAIL)
goto retry; goto retry;
*owned_head = Z_EROFS_VLE_WORKGRP_TAIL; *owned_head = Z_EROFS_PCLUSTER_TAIL;
return Z_EROFS_VLE_WORK_PRIMARY_HOOKED; return COLLECT_PRIMARY_HOOKED;
} }
return COLLECT_PRIMARY; /* :( better luck next time */
return Z_EROFS_VLE_WORK_PRIMARY; /* :( better luck next time */
} }
struct z_erofs_vle_work_finder { static struct z_erofs_collection *cllookup(struct z_erofs_collector *clt,
struct super_block *sb; struct inode *inode,
pgoff_t idx; struct erofs_map_blocks *map)
unsigned int pageofs;
struct z_erofs_vle_workgroup **grp_ret;
enum z_erofs_vle_work_role *role;
z_erofs_vle_owned_workgrp_t *owned_head;
bool *hosted;
};
static struct z_erofs_vle_work *
z_erofs_vle_work_lookup(const struct z_erofs_vle_work_finder *f)
{ {
bool tag, primary; struct erofs_workgroup *grp;
struct erofs_workgroup *egrp; struct z_erofs_pcluster *pcl;
struct z_erofs_vle_workgroup *grp; struct z_erofs_collection *cl;
struct z_erofs_vle_work *work; unsigned int length;
bool tag;
egrp = erofs_find_workgroup(f->sb, f->idx, &tag);
if (!egrp) { grp = erofs_find_workgroup(inode->i_sb, map->m_pa >> PAGE_SHIFT, &tag);
*f->grp_ret = NULL; if (!grp)
return NULL; return NULL;
}
grp = container_of(egrp, struct z_erofs_vle_workgroup, obj); pcl = container_of(grp, struct z_erofs_pcluster, obj);
*f->grp_ret = grp;
cl = z_erofs_primarycollection(pcl);
if (unlikely(cl->pageofs != (map->m_la & ~PAGE_MASK))) {
DBG_BUGON(1);
return ERR_PTR(-EIO);
}
work = z_erofs_vle_grab_work(grp, f->pageofs); length = READ_ONCE(pcl->length);
/* if multiref is disabled, `primary' is always true */ if (length & Z_EROFS_PCLUSTER_FULL_LENGTH) {
primary = true; if ((map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) > length) {
DBG_BUGON(1);
return ERR_PTR(-EIO);
}
} else {
unsigned int llen = map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT;
DBG_BUGON(work->pageofs != f->pageofs); if (map->m_flags & EROFS_MAP_FULL_MAPPED)
llen |= Z_EROFS_PCLUSTER_FULL_LENGTH;
/* while (llen > length &&
* lock must be taken first to avoid grp->next == NIL between length != cmpxchg_relaxed(&pcl->length, length, llen)) {
* claiming workgroup and adding pages: cpu_relax();
* grp->next != NIL length = READ_ONCE(pcl->length);
* grp->next = NIL }
* mutex_unlock_all }
* mutex_lock(&work->lock) mutex_lock(&cl->lock);
* add all pages to pagevec clt->mode = try_to_claim_pcluster(pcl, &clt->owned_head);
* clt->pcl = pcl;
* [correct locking case 1]: clt->cl = cl;
* mutex_lock(grp->work[a]) return cl;
* ...
* mutex_lock(grp->work[b]) mutex_lock(grp->work[c])
* ... *role = SECONDARY
* add all pages to pagevec
* ...
* mutex_unlock(grp->work[c])
* mutex_lock(grp->work[c])
* ...
* grp->next = NIL
* mutex_unlock_all
*
* [correct locking case 2]:
* mutex_lock(grp->work[b])
* ...
* mutex_lock(grp->work[a])
* ...
* mutex_lock(grp->work[c])
* ...
* grp->next = NIL
* mutex_unlock_all
* mutex_lock(grp->work[a])
* *role = PRIMARY_OWNER
* add all pages to pagevec
* ...
*/
mutex_lock(&work->lock);
*f->hosted = false;
if (!primary)
*f->role = Z_EROFS_VLE_WORK_SECONDARY;
else /* claim the workgroup if possible */
*f->role = try_to_claim_workgroup(grp, f->owned_head,
f->hosted);
return work;
} }
static struct z_erofs_vle_work * static struct z_erofs_collection *clregister(struct z_erofs_collector *clt,
z_erofs_vle_work_register(const struct z_erofs_vle_work_finder *f, struct inode *inode,
struct erofs_map_blocks *map) struct erofs_map_blocks *map)
{ {
bool gnew = false; struct z_erofs_pcluster *pcl;
struct z_erofs_vle_workgroup *grp = *f->grp_ret; struct z_erofs_collection *cl;
struct z_erofs_vle_work *work; int err;
/* if multiref is disabled, grp should never be nullptr */
if (unlikely(grp)) {
DBG_BUGON(1);
return ERR_PTR(-EINVAL);
}
/* no available workgroup, let's allocate one */ /* no available workgroup, let's allocate one */
grp = kmem_cache_alloc(z_erofs_workgroup_cachep, GFP_NOFS); pcl = kmem_cache_alloc(pcluster_cachep, GFP_NOFS);
if (unlikely(!grp)) if (unlikely(!pcl))
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
init_always(grp); init_always(pcl);
grp->obj.index = f->idx; pcl->obj.index = map->m_pa >> PAGE_SHIFT;
grp->llen = map->m_llen;
z_erofs_vle_set_workgrp_fmt(grp, (map->m_flags & EROFS_MAP_ZIPPED) ? pcl->length = (map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) |
Z_EROFS_VLE_WORKGRP_FMT_LZ4 : (map->m_flags & EROFS_MAP_FULL_MAPPED ?
Z_EROFS_VLE_WORKGRP_FMT_PLAIN); Z_EROFS_PCLUSTER_FULL_LENGTH : 0);
if (map->m_flags & EROFS_MAP_FULL_MAPPED) if (map->m_flags & EROFS_MAP_ZIPPED)
grp->flags |= Z_EROFS_VLE_WORKGRP_FULL_LENGTH; pcl->algorithmformat = Z_EROFS_COMPRESSION_LZ4;
else
pcl->algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
pcl->clusterbits = EROFS_V(inode)->z_physical_clusterbits[0];
pcl->clusterbits -= PAGE_SHIFT;
/* new workgrps have been claimed as type 1 */ /* new pclusters should be claimed as type 1, primary and followed */
WRITE_ONCE(grp->next, *f->owned_head); pcl->next = clt->owned_head;
/* primary and followed work for all new workgrps */ clt->mode = COLLECT_PRIMARY_FOLLOWED;
*f->role = Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED;
/* it should be submitted by ourselves */
*f->hosted = true;
gnew = true; cl = z_erofs_primarycollection(pcl);
work = z_erofs_vle_grab_primary_work(grp); cl->pageofs = map->m_la & ~PAGE_MASK;
work->pageofs = f->pageofs;
/* /*
* lock all primary followed works before visible to others * lock all primary followed works before visible to others
* and mutex_trylock *never* fails for a new workgroup. * and mutex_trylock *never* fails for a new pcluster.
*/ */
mutex_trylock(&work->lock); mutex_trylock(&cl->lock);
if (gnew) { err = erofs_register_workgroup(inode->i_sb, &pcl->obj, 0);
int err = erofs_register_workgroup(f->sb, &grp->obj, 0); if (err) {
mutex_unlock(&cl->lock);
if (err) { kmem_cache_free(pcluster_cachep, pcl);
mutex_unlock(&work->lock); return ERR_PTR(-EAGAIN);
kmem_cache_free(z_erofs_workgroup_cachep, grp);
return ERR_PTR(-EAGAIN);
}
} }
clt->owned_head = &pcl->next;
*f->owned_head = &grp->next; clt->pcl = pcl;
*f->grp_ret = grp; clt->cl = cl;
return work; return cl;
} }
#define builder_is_hooked(builder) \ static int z_erofs_collector_begin(struct z_erofs_collector *clt,
((builder)->role >= Z_EROFS_VLE_WORK_PRIMARY_HOOKED) struct inode *inode,
struct erofs_map_blocks *map)
{
struct z_erofs_collection *cl;
#define builder_is_followed(builder) \ DBG_BUGON(clt->cl);
((builder)->role >= Z_EROFS_VLE_WORK_PRIMARY_FOLLOWED)
static int z_erofs_vle_work_iter_begin(struct z_erofs_vle_work_builder *builder, /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous collection */
struct super_block *sb, DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_NIL);
struct erofs_map_blocks *map, DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
z_erofs_vle_owned_workgrp_t *owned_head)
{
const unsigned int clusterpages = erofs_clusterpages(EROFS_SB(sb));
struct z_erofs_vle_workgroup *grp;
const struct z_erofs_vle_work_finder finder = {
.sb = sb,
.idx = erofs_blknr(map->m_pa),
.pageofs = map->m_la & ~PAGE_MASK,
.grp_ret = &grp,
.role = &builder->role,
.owned_head = owned_head,
.hosted = &builder->hosted
};
struct z_erofs_vle_work *work;
DBG_BUGON(builder->work);
/* must be Z_EROFS_WORK_TAIL or the next chained work */
DBG_BUGON(*owned_head == Z_EROFS_VLE_WORKGRP_NIL);
DBG_BUGON(*owned_head == Z_EROFS_VLE_WORKGRP_TAIL_CLOSED);
DBG_BUGON(erofs_blkoff(map->m_pa));
repeat: if (!PAGE_ALIGNED(map->m_pa)) {
work = z_erofs_vle_work_lookup(&finder); DBG_BUGON(1);
if (work) { return -EINVAL;
unsigned int orig_llen;
/* increase workgroup `llen' if needed */
while ((orig_llen = READ_ONCE(grp->llen)) < map->m_llen &&
orig_llen != cmpxchg_relaxed(&grp->llen,
orig_llen, map->m_llen))
cpu_relax();
goto got_it;
} }
work = z_erofs_vle_work_register(&finder, map); repeat:
if (unlikely(work == ERR_PTR(-EAGAIN))) cl = cllookup(clt, inode, map);
goto repeat; if (!cl) {
cl = clregister(clt, inode, map);
if (IS_ERR(work))
return PTR_ERR(work);
got_it:
z_erofs_pagevec_ctor_init(&builder->vector, Z_EROFS_NR_INLINE_PAGEVECS,
work->pagevec, work->vcnt);
if (builder->role >= Z_EROFS_VLE_WORK_PRIMARY) { if (unlikely(cl == ERR_PTR(-EAGAIN)))
/* enable possibly in-place decompression */ goto repeat;
builder->compressed_pages = grp->compressed_pages;
builder->compressed_deficit = clusterpages;
} else {
builder->compressed_pages = NULL;
builder->compressed_deficit = 0;
} }
builder->grp = grp; if (IS_ERR(cl))
builder->work = work; return PTR_ERR(cl);
z_erofs_pagevec_ctor_init(&clt->vector, Z_EROFS_NR_INLINE_PAGEVECS,
cl->pagevec, cl->vcnt);
clt->compressedpages = clt->pcl->compressed_pages;
if (clt->mode <= COLLECT_PRIMARY) /* cannot do in-place I/O */
clt->compressedpages += Z_EROFS_CLUSTER_MAX_PAGES;
return 0; return 0;
} }
/* /*
* keep in mind that no referenced workgroups will be freed * keep in mind that no referenced pclusters will be freed
* only after a RCU grace period, so rcu_read_lock() could * only after a RCU grace period.
* prevent a workgroup from being freed.
*/ */
static void z_erofs_rcu_callback(struct rcu_head *head) static void z_erofs_rcu_callback(struct rcu_head *head)
{ {
struct z_erofs_vle_work *work = container_of(head, struct z_erofs_collection *const cl =
struct z_erofs_vle_work, rcu); container_of(head, struct z_erofs_collection, rcu);
struct z_erofs_vle_workgroup *grp =
z_erofs_vle_work_workgroup(work, true);
kmem_cache_free(z_erofs_workgroup_cachep, grp); kmem_cache_free(pcluster_cachep,
container_of(cl, struct z_erofs_pcluster,
primary_collection));
} }
void erofs_workgroup_free_rcu(struct erofs_workgroup *grp) void erofs_workgroup_free_rcu(struct erofs_workgroup *grp)
{ {
struct z_erofs_vle_workgroup *const vgrp = container_of(grp, struct z_erofs_pcluster *const pcl =
struct z_erofs_vle_workgroup, obj); container_of(grp, struct z_erofs_pcluster, obj);
struct z_erofs_vle_work *const work = &vgrp->work; struct z_erofs_collection *const cl = z_erofs_primarycollection(pcl);
call_rcu(&work->rcu, z_erofs_rcu_callback); call_rcu(&cl->rcu, z_erofs_rcu_callback);
} }
static void static void z_erofs_collection_put(struct z_erofs_collection *cl)
__z_erofs_vle_work_release(struct z_erofs_vle_workgroup *grp,
struct z_erofs_vle_work *work __maybe_unused)
{ {
erofs_workgroup_put(&grp->obj); struct z_erofs_pcluster *const pcl =
} container_of(cl, struct z_erofs_pcluster, primary_collection);
static void z_erofs_vle_work_release(struct z_erofs_vle_work *work) erofs_workgroup_put(&pcl->obj);
{
struct z_erofs_vle_workgroup *grp =
z_erofs_vle_work_workgroup(work, true);
__z_erofs_vle_work_release(grp, work);
} }
static inline bool static bool z_erofs_collector_end(struct z_erofs_collector *clt)
z_erofs_vle_work_iter_end(struct z_erofs_vle_work_builder *builder)
{ {
struct z_erofs_vle_work *work = builder->work; struct z_erofs_collection *cl = clt->cl;
if (!work) if (!cl)
return false; return false;
z_erofs_pagevec_ctor_exit(&builder->vector, false); z_erofs_pagevec_ctor_exit(&clt->vector, false);
mutex_unlock(&work->lock); mutex_unlock(&cl->lock);
/* /*
* if all pending pages are added, don't hold work reference * if all pending pages are added, don't hold its reference
* any longer if the current work isn't hosted by ourselves. * any longer if the pcluster isn't hosted by ourselves.
*/ */
if (!builder->hosted) if (clt->mode < COLLECT_PRIMARY_FOLLOWED_NOINPLACE)
__z_erofs_vle_work_release(builder->grp, work); z_erofs_collection_put(cl);
builder->work = NULL; clt->cl = NULL;
builder->grp = NULL;
return true; return true;
} }
...@@ -640,33 +547,9 @@ static inline struct page *__stagingpage_alloc(struct list_head *pagepool, ...@@ -640,33 +547,9 @@ static inline struct page *__stagingpage_alloc(struct list_head *pagepool,
return page; return page;
} }
struct z_erofs_vle_frontend {
struct inode *const inode;
struct z_erofs_vle_work_builder builder;
struct erofs_map_blocks map;
z_erofs_vle_owned_workgrp_t owned_head;
/* used for applying cache strategy on the fly */
bool backmost;
erofs_off_t headoffset;
};
#define VLE_FRONTEND_INIT(__i) { \
.inode = __i, \
.map = { \
.m_llen = 0, \
.m_plen = 0, \
.mpage = NULL \
}, \
.builder = VLE_WORK_BUILDER_INIT(), \
.owned_head = Z_EROFS_VLE_WORKGRP_TAIL, \
.backmost = true, }
#ifdef EROFS_FS_HAS_MANAGED_CACHE #ifdef EROFS_FS_HAS_MANAGED_CACHE
static inline bool static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe,
should_alloc_managed_pages(struct z_erofs_vle_frontend *fe, erofs_off_t la) erofs_off_t la)
{ {
if (fe->backmost) if (fe->backmost)
return true; return true;
...@@ -677,25 +560,23 @@ should_alloc_managed_pages(struct z_erofs_vle_frontend *fe, erofs_off_t la) ...@@ -677,25 +560,23 @@ should_alloc_managed_pages(struct z_erofs_vle_frontend *fe, erofs_off_t la)
return false; return false;
} }
#else #else
static inline bool static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe,
should_alloc_managed_pages(struct z_erofs_vle_frontend *fe, erofs_off_t la) erofs_off_t la)
{ {
return false; return false;
} }
#endif #endif
static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe, static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe,
struct page *page, struct page *page,
struct list_head *page_pool) struct list_head *pagepool)
{ {
struct super_block *const sb = fe->inode->i_sb; struct inode *const inode = fe->inode;
struct erofs_sb_info *const sbi __maybe_unused = EROFS_SB(sb); struct erofs_sb_info *const sbi __maybe_unused = EROFS_I_SB(inode);
struct erofs_map_blocks *const map = &fe->map; struct erofs_map_blocks *const map = &fe->map;
struct z_erofs_vle_work_builder *const builder = &fe->builder; struct z_erofs_collector *const clt = &fe->clt;
const loff_t offset = page_offset(page); const loff_t offset = page_offset(page);
bool tight = (clt->mode >= COLLECT_PRIMARY_HOOKED);
bool tight = builder_is_hooked(builder);
struct z_erofs_vle_work *work = builder->work;
enum z_erofs_cache_alloctype cache_strategy; enum z_erofs_cache_alloctype cache_strategy;
enum z_erofs_page_type page_type; enum z_erofs_page_type page_type;
...@@ -713,8 +594,8 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe, ...@@ -713,8 +594,8 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe,
/* lucky, within the range of the current map_blocks */ /* lucky, within the range of the current map_blocks */
if (offset + cur >= map->m_la && if (offset + cur >= map->m_la &&
offset + cur < map->m_la + map->m_llen) { offset + cur < map->m_la + map->m_llen) {
/* didn't get a valid unzip work previously (very rare) */ /* didn't get a valid collection previously (very rare) */
if (!builder->work) if (!clt->cl)
goto restart_now; goto restart_now;
goto hitted; goto hitted;
} }
...@@ -722,12 +603,12 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe, ...@@ -722,12 +603,12 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe,
/* go ahead the next map_blocks */ /* go ahead the next map_blocks */
debugln("%s: [out-of-range] pos %llu", __func__, offset + cur); debugln("%s: [out-of-range] pos %llu", __func__, offset + cur);
if (z_erofs_vle_work_iter_end(builder)) if (z_erofs_collector_end(clt))
fe->backmost = false; fe->backmost = false;
map->m_la = offset + cur; map->m_la = offset + cur;
map->m_llen = 0; map->m_llen = 0;
err = z_erofs_map_blocks_iter(fe->inode, map, 0); err = z_erofs_map_blocks_iter(inode, map, 0);
if (unlikely(err)) if (unlikely(err))
goto err_out; goto err_out;
...@@ -735,10 +616,7 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe, ...@@ -735,10 +616,7 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe,
if (unlikely(!(map->m_flags & EROFS_MAP_MAPPED))) if (unlikely(!(map->m_flags & EROFS_MAP_MAPPED)))
goto hitted; goto hitted;
DBG_BUGON(map->m_plen != 1 << sbi->clusterbits); err = z_erofs_collector_begin(clt, inode, map);
DBG_BUGON(erofs_blkoff(map->m_pa));
err = z_erofs_vle_work_iter_begin(builder, sb, map, &fe->owned_head);
if (unlikely(err)) if (unlikely(err))
goto err_out; goto err_out;
...@@ -748,13 +626,10 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe, ...@@ -748,13 +626,10 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe,
else else
cache_strategy = DONTALLOC; cache_strategy = DONTALLOC;
preload_compressed_pages(builder, MNGD_MAPPING(sbi), preload_compressed_pages(clt, MNGD_MAPPING(sbi),
map->m_pa / PAGE_SIZE, cache_strategy, pagepool);
map->m_plen / PAGE_SIZE,
cache_strategy, page_pool, GFP_KERNEL);
tight &= builder_is_hooked(builder); tight &= (clt->mode >= COLLECT_PRIMARY_HOOKED);
work = builder->work;
hitted: hitted:
cur = end - min_t(unsigned int, offset + end - map->m_la, end); cur = end - min_t(unsigned int, offset + end - map->m_la, end);
if (unlikely(!(map->m_flags & EROFS_MAP_MAPPED))) { if (unlikely(!(map->m_flags & EROFS_MAP_MAPPED))) {
...@@ -769,17 +644,17 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe, ...@@ -769,17 +644,17 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe,
Z_EROFS_VLE_PAGE_TYPE_TAIL_SHARED)); Z_EROFS_VLE_PAGE_TYPE_TAIL_SHARED));
if (cur) if (cur)
tight &= builder_is_followed(builder); tight &= (clt->mode >= COLLECT_PRIMARY_FOLLOWED);
retry: retry:
err = z_erofs_vle_work_add_page(builder, page, page_type); err = z_erofs_attach_page(clt, page, page_type);
/* should allocate an additional staging page for pagevec */ /* should allocate an additional staging page for pagevec */
if (err == -EAGAIN) { if (err == -EAGAIN) {
struct page *const newpage = struct page *const newpage =
__stagingpage_alloc(page_pool, GFP_NOFS); __stagingpage_alloc(pagepool, GFP_NOFS);
err = z_erofs_vle_work_add_page(builder, newpage, err = z_erofs_attach_page(clt, newpage,
Z_EROFS_PAGE_TYPE_EXCLUSIVE); Z_EROFS_PAGE_TYPE_EXCLUSIVE);
if (likely(!err)) if (likely(!err))
goto retry; goto retry;
} }
...@@ -787,15 +662,14 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe, ...@@ -787,15 +662,14 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe,
if (unlikely(err)) if (unlikely(err))
goto err_out; goto err_out;
index = page->index - map->m_la / PAGE_SIZE; index = page->index - (map->m_la >> PAGE_SHIFT);
/* FIXME! avoid the last relundant fixup & endio */
z_erofs_onlinepage_fixup(page, index, true); z_erofs_onlinepage_fixup(page, index, true);
/* bump up the number of spiltted parts of a page */ /* bump up the number of spiltted parts of a page */
++spiltted; ++spiltted;
/* also update nr_pages */ /* also update nr_pages */
work->nr_pages = max_t(pgoff_t, work->nr_pages, index + 1); clt->cl->nr_pages = max_t(pgoff_t, clt->cl->nr_pages, index + 1);
next_part: next_part:
/* can be used for verification */ /* can be used for verification */
map->m_llen = offset + cur - map->m_la; map->m_llen = offset + cur - map->m_la;
...@@ -805,7 +679,6 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe, ...@@ -805,7 +679,6 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe,
goto repeat; goto repeat;
out: out:
/* FIXME! avoid the last relundant fixup & endio */
z_erofs_onlinepage_endio(page); z_erofs_onlinepage_endio(page);
debugln("%s, finish page: %pK spiltted: %u map->m_llen %llu", debugln("%s, finish page: %pK spiltted: %u map->m_llen %llu",
...@@ -821,7 +694,7 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe, ...@@ -821,7 +694,7 @@ static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe,
static void z_erofs_vle_unzip_kickoff(void *ptr, int bios) static void z_erofs_vle_unzip_kickoff(void *ptr, int bios)
{ {
tagptr1_t t = tagptr_init(tagptr1_t, ptr); tagptr1_t t = tagptr_init(tagptr1_t, ptr);
struct z_erofs_vle_unzip_io *io = tagptr_unfold_ptr(t); struct z_erofs_unzip_io *io = tagptr_unfold_ptr(t);
bool background = tagptr_unfold_tags(t); bool background = tagptr_unfold_tags(t);
if (!background) { if (!background) {
...@@ -878,45 +751,38 @@ static inline void z_erofs_vle_read_endio(struct bio *bio) ...@@ -878,45 +751,38 @@ static inline void z_erofs_vle_read_endio(struct bio *bio)
bio_put(bio); bio_put(bio);
} }
static struct page *z_pagemap_global[Z_EROFS_VLE_VMAP_GLOBAL_PAGES]; static int z_erofs_decompress_pcluster(struct super_block *sb,
static DEFINE_MUTEX(z_pagemap_global_lock); struct z_erofs_pcluster *pcl,
struct list_head *pagepool)
static int z_erofs_vle_unzip(struct super_block *sb,
struct z_erofs_vle_workgroup *grp,
struct list_head *page_pool)
{ {
struct erofs_sb_info *const sbi = EROFS_SB(sb); struct erofs_sb_info *const sbi = EROFS_SB(sb);
const unsigned int clusterpages = erofs_clusterpages(sbi); const unsigned int clusterpages = BIT(pcl->clusterbits);
struct z_erofs_pagevec_ctor ctor; struct z_erofs_pagevec_ctor ctor;
unsigned int nr_pages; unsigned int i, outputsize, llen, nr_pages;
unsigned int sparsemem_pages = 0; struct page *pages_onstack[Z_EROFS_VMAP_ONSTACK_PAGES];
struct page *pages_onstack[Z_EROFS_VLE_VMAP_ONSTACK_PAGES];
struct page **pages, **compressed_pages, *page; struct page **pages, **compressed_pages, *page;
unsigned int algorithm;
unsigned int i, outputsize;
enum z_erofs_page_type page_type; enum z_erofs_page_type page_type;
bool overlapped, partial; bool overlapped, partial;
struct z_erofs_vle_work *work; struct z_erofs_collection *cl;
int err; int err;
might_sleep(); might_sleep();
work = z_erofs_vle_grab_primary_work(grp); cl = z_erofs_primarycollection(pcl);
DBG_BUGON(!READ_ONCE(work->nr_pages)); DBG_BUGON(!READ_ONCE(cl->nr_pages));
mutex_lock(&work->lock); mutex_lock(&cl->lock);
nr_pages = work->nr_pages; nr_pages = cl->nr_pages;
if (likely(nr_pages <= Z_EROFS_VLE_VMAP_ONSTACK_PAGES)) if (likely(nr_pages <= Z_EROFS_VMAP_ONSTACK_PAGES)) {
pages = pages_onstack; pages = pages_onstack;
else if (nr_pages <= Z_EROFS_VLE_VMAP_GLOBAL_PAGES && } else if (nr_pages <= Z_EROFS_VMAP_GLOBAL_PAGES &&
mutex_trylock(&z_pagemap_global_lock)) mutex_trylock(&z_pagemap_global_lock)) {
pages = z_pagemap_global; pages = z_pagemap_global;
else { } else {
gfp_t gfp_flags = GFP_KERNEL; gfp_t gfp_flags = GFP_KERNEL;
if (nr_pages > Z_EROFS_VLE_VMAP_GLOBAL_PAGES) if (nr_pages > Z_EROFS_VMAP_GLOBAL_PAGES)
gfp_flags |= __GFP_NOFAIL; gfp_flags |= __GFP_NOFAIL;
pages = kvmalloc_array(nr_pages, sizeof(struct page *), pages = kvmalloc_array(nr_pages, sizeof(struct page *),
...@@ -933,9 +799,9 @@ static int z_erofs_vle_unzip(struct super_block *sb, ...@@ -933,9 +799,9 @@ static int z_erofs_vle_unzip(struct super_block *sb,
pages[i] = NULL; pages[i] = NULL;
z_erofs_pagevec_ctor_init(&ctor, Z_EROFS_NR_INLINE_PAGEVECS, z_erofs_pagevec_ctor_init(&ctor, Z_EROFS_NR_INLINE_PAGEVECS,
work->pagevec, 0); cl->pagevec, 0);
for (i = 0; i < work->vcnt; ++i) { for (i = 0; i < cl->vcnt; ++i) {
unsigned int pagenr; unsigned int pagenr;
page = z_erofs_pagevec_dequeue(&ctor, &page_type); page = z_erofs_pagevec_dequeue(&ctor, &page_type);
...@@ -944,7 +810,7 @@ static int z_erofs_vle_unzip(struct super_block *sb, ...@@ -944,7 +810,7 @@ static int z_erofs_vle_unzip(struct super_block *sb,
DBG_BUGON(!page); DBG_BUGON(!page);
DBG_BUGON(!page->mapping); DBG_BUGON(!page->mapping);
if (z_erofs_put_stagingpage(page_pool, page)) if (z_erofs_put_stagingpage(pagepool, page))
continue; continue;
if (page_type == Z_EROFS_VLE_PAGE_TYPE_HEAD) if (page_type == Z_EROFS_VLE_PAGE_TYPE_HEAD)
...@@ -957,12 +823,10 @@ static int z_erofs_vle_unzip(struct super_block *sb, ...@@ -957,12 +823,10 @@ static int z_erofs_vle_unzip(struct super_block *sb,
pages[pagenr] = page; pages[pagenr] = page;
} }
sparsemem_pages = i;
z_erofs_pagevec_ctor_exit(&ctor, true); z_erofs_pagevec_ctor_exit(&ctor, true);
overlapped = false; overlapped = false;
compressed_pages = grp->compressed_pages; compressed_pages = pcl->compressed_pages;
err = 0; err = 0;
for (i = 0; i < clusterpages; ++i) { for (i = 0; i < clusterpages; ++i) {
...@@ -989,7 +853,6 @@ static int z_erofs_vle_unzip(struct super_block *sb, ...@@ -989,7 +853,6 @@ static int z_erofs_vle_unzip(struct super_block *sb,
DBG_BUGON(pagenr >= nr_pages); DBG_BUGON(pagenr >= nr_pages);
DBG_BUGON(pages[pagenr]); DBG_BUGON(pages[pagenr]);
++sparsemem_pages;
pages[pagenr] = page; pages[pagenr] = page;
overlapped = true; overlapped = true;
...@@ -1005,30 +868,26 @@ static int z_erofs_vle_unzip(struct super_block *sb, ...@@ -1005,30 +868,26 @@ static int z_erofs_vle_unzip(struct super_block *sb,
if (unlikely(err)) if (unlikely(err))
goto out; goto out;
if (nr_pages << PAGE_SHIFT >= work->pageofs + grp->llen) { llen = pcl->length >> Z_EROFS_PCLUSTER_LENGTH_BIT;
outputsize = grp->llen; if (nr_pages << PAGE_SHIFT >= cl->pageofs + llen) {
partial = !(grp->flags & Z_EROFS_VLE_WORKGRP_FULL_LENGTH); outputsize = llen;
partial = !(pcl->length & Z_EROFS_PCLUSTER_FULL_LENGTH);
} else { } else {
outputsize = (nr_pages << PAGE_SHIFT) - work->pageofs; outputsize = (nr_pages << PAGE_SHIFT) - cl->pageofs;
partial = true; partial = true;
} }
if (z_erofs_vle_workgrp_fmt(grp) == Z_EROFS_VLE_WORKGRP_FMT_PLAIN)
algorithm = Z_EROFS_COMPRESSION_SHIFTED;
else
algorithm = Z_EROFS_COMPRESSION_LZ4;
err = z_erofs_decompress(&(struct z_erofs_decompress_req) { err = z_erofs_decompress(&(struct z_erofs_decompress_req) {
.sb = sb, .sb = sb,
.in = compressed_pages, .in = compressed_pages,
.out = pages, .out = pages,
.pageofs_out = work->pageofs, .pageofs_out = cl->pageofs,
.inputsize = PAGE_SIZE, .inputsize = PAGE_SIZE,
.outputsize = outputsize, .outputsize = outputsize,
.alg = algorithm, .alg = pcl->algorithmformat,
.inplace_io = overlapped, .inplace_io = overlapped,
.partial_decoding = partial .partial_decoding = partial
}, page_pool); }, pagepool);
out: out:
/* must handle all compressed pages before endding pages */ /* must handle all compressed pages before endding pages */
...@@ -1039,7 +898,7 @@ static int z_erofs_vle_unzip(struct super_block *sb, ...@@ -1039,7 +898,7 @@ static int z_erofs_vle_unzip(struct super_block *sb,
continue; continue;
/* recycle all individual staging pages */ /* recycle all individual staging pages */
(void)z_erofs_put_stagingpage(page_pool, page); (void)z_erofs_put_stagingpage(pagepool, page);
WRITE_ONCE(compressed_pages[i], NULL); WRITE_ONCE(compressed_pages[i], NULL);
} }
...@@ -1052,7 +911,7 @@ static int z_erofs_vle_unzip(struct super_block *sb, ...@@ -1052,7 +911,7 @@ static int z_erofs_vle_unzip(struct super_block *sb,
DBG_BUGON(!page->mapping); DBG_BUGON(!page->mapping);
/* recycle all individual staging pages */ /* recycle all individual staging pages */
if (z_erofs_put_stagingpage(page_pool, page)) if (z_erofs_put_stagingpage(pagepool, page))
continue; continue;
if (unlikely(err < 0)) if (unlikely(err < 0))
...@@ -1066,65 +925,63 @@ static int z_erofs_vle_unzip(struct super_block *sb, ...@@ -1066,65 +925,63 @@ static int z_erofs_vle_unzip(struct super_block *sb,
else if (unlikely(pages != pages_onstack)) else if (unlikely(pages != pages_onstack))
kvfree(pages); kvfree(pages);
work->nr_pages = 0; cl->nr_pages = 0;
work->vcnt = 0; cl->vcnt = 0;
/* all work locks MUST be taken before the following line */ /* all cl locks MUST be taken before the following line */
WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL);
WRITE_ONCE(grp->next, Z_EROFS_VLE_WORKGRP_NIL); /* all cl locks SHOULD be released right now */
mutex_unlock(&cl->lock);
/* all work locks SHOULD be released right now */ z_erofs_collection_put(cl);
mutex_unlock(&work->lock);
z_erofs_vle_work_release(work);
return err; return err;
} }
static void z_erofs_vle_unzip_all(struct super_block *sb, static void z_erofs_vle_unzip_all(struct super_block *sb,
struct z_erofs_vle_unzip_io *io, struct z_erofs_unzip_io *io,
struct list_head *page_pool) struct list_head *pagepool)
{ {
z_erofs_vle_owned_workgrp_t owned = io->head; z_erofs_next_pcluster_t owned = io->head;
while (owned != Z_EROFS_VLE_WORKGRP_TAIL_CLOSED) { while (owned != Z_EROFS_PCLUSTER_TAIL_CLOSED) {
struct z_erofs_vle_workgroup *grp; struct z_erofs_pcluster *pcl;
/* no possible that 'owned' equals Z_EROFS_WORK_TPTR_TAIL */ /* no possible that 'owned' equals Z_EROFS_WORK_TPTR_TAIL */
DBG_BUGON(owned == Z_EROFS_VLE_WORKGRP_TAIL); DBG_BUGON(owned == Z_EROFS_PCLUSTER_TAIL);
/* no possible that 'owned' equals NULL */ /* no possible that 'owned' equals NULL */
DBG_BUGON(owned == Z_EROFS_VLE_WORKGRP_NIL); DBG_BUGON(owned == Z_EROFS_PCLUSTER_NIL);
grp = container_of(owned, struct z_erofs_vle_workgroup, next); pcl = container_of(owned, struct z_erofs_pcluster, next);
owned = READ_ONCE(grp->next); owned = READ_ONCE(pcl->next);
z_erofs_vle_unzip(sb, grp, page_pool); z_erofs_decompress_pcluster(sb, pcl, pagepool);
} }
} }
static void z_erofs_vle_unzip_wq(struct work_struct *work) static void z_erofs_vle_unzip_wq(struct work_struct *work)
{ {
struct z_erofs_vle_unzip_io_sb *iosb = container_of(work, struct z_erofs_unzip_io_sb *iosb =
struct z_erofs_vle_unzip_io_sb, io.u.work); container_of(work, struct z_erofs_unzip_io_sb, io.u.work);
LIST_HEAD(page_pool); LIST_HEAD(pagepool);
DBG_BUGON(iosb->io.head == Z_EROFS_VLE_WORKGRP_TAIL_CLOSED); DBG_BUGON(iosb->io.head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
z_erofs_vle_unzip_all(iosb->sb, &iosb->io, &page_pool); z_erofs_vle_unzip_all(iosb->sb, &iosb->io, &pagepool);
put_pages_list(&page_pool); put_pages_list(&pagepool);
kvfree(iosb); kvfree(iosb);
} }
static struct page * static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl,
pickup_page_for_submission(struct z_erofs_vle_workgroup *grp, unsigned int nr,
unsigned int nr, struct list_head *pagepool,
struct list_head *pagepool, struct address_space *mc,
struct address_space *mc, gfp_t gfp)
gfp_t gfp)
{ {
/* determined at compile time to avoid too many #ifdefs */ /* determined at compile time to avoid too many #ifdefs */
const bool nocache = __builtin_constant_p(mc) ? !mc : false; const bool nocache = __builtin_constant_p(mc) ? !mc : false;
const pgoff_t index = grp->obj.index; const pgoff_t index = pcl->obj.index;
bool tocache = false; bool tocache = false;
struct address_space *mapping; struct address_space *mapping;
...@@ -1134,7 +991,7 @@ pickup_page_for_submission(struct z_erofs_vle_workgroup *grp, ...@@ -1134,7 +991,7 @@ pickup_page_for_submission(struct z_erofs_vle_workgroup *grp,
int justfound; int justfound;
repeat: repeat:
page = READ_ONCE(grp->compressed_pages[nr]); page = READ_ONCE(pcl->compressed_pages[nr]);
oldpage = page; oldpage = page;
if (!page) if (!page)
...@@ -1186,7 +1043,7 @@ pickup_page_for_submission(struct z_erofs_vle_workgroup *grp, ...@@ -1186,7 +1043,7 @@ pickup_page_for_submission(struct z_erofs_vle_workgroup *grp,
/* the page is still in manage cache */ /* the page is still in manage cache */
if (page->mapping == mc) { if (page->mapping == mc) {
WRITE_ONCE(grp->compressed_pages[nr], page); WRITE_ONCE(pcl->compressed_pages[nr], page);
ClearPageError(page); ClearPageError(page);
if (!PagePrivate(page)) { if (!PagePrivate(page)) {
...@@ -1198,7 +1055,7 @@ pickup_page_for_submission(struct z_erofs_vle_workgroup *grp, ...@@ -1198,7 +1055,7 @@ pickup_page_for_submission(struct z_erofs_vle_workgroup *grp,
DBG_BUGON(!justfound); DBG_BUGON(!justfound);
justfound = 0; justfound = 0;
set_page_private(page, (unsigned long)grp); set_page_private(page, (unsigned long)pcl);
SetPagePrivate(page); SetPagePrivate(page);
} }
...@@ -1222,7 +1079,7 @@ pickup_page_for_submission(struct z_erofs_vle_workgroup *grp, ...@@ -1222,7 +1079,7 @@ pickup_page_for_submission(struct z_erofs_vle_workgroup *grp,
put_page(page); put_page(page);
out_allocpage: out_allocpage:
page = __stagingpage_alloc(pagepool, gfp); page = __stagingpage_alloc(pagepool, gfp);
if (oldpage != cmpxchg(&grp->compressed_pages[nr], oldpage, page)) { if (oldpage != cmpxchg(&pcl->compressed_pages[nr], oldpage, page)) {
list_add(&page->lru, pagepool); list_add(&page->lru, pagepool);
cpu_relax(); cpu_relax();
goto repeat; goto repeat;
...@@ -1234,18 +1091,17 @@ pickup_page_for_submission(struct z_erofs_vle_workgroup *grp, ...@@ -1234,18 +1091,17 @@ pickup_page_for_submission(struct z_erofs_vle_workgroup *grp,
goto out; goto out;
} }
set_page_private(page, (unsigned long)grp); set_page_private(page, (unsigned long)pcl);
SetPagePrivate(page); SetPagePrivate(page);
out: /* the only exit (for tracing and debugging) */ out: /* the only exit (for tracing and debugging) */
return page; return page;
} }
static struct z_erofs_vle_unzip_io * static struct z_erofs_unzip_io *jobqueue_init(struct super_block *sb,
jobqueue_init(struct super_block *sb, struct z_erofs_unzip_io *io,
struct z_erofs_vle_unzip_io *io, bool foreground)
bool foreground)
{ {
struct z_erofs_vle_unzip_io_sb *iosb; struct z_erofs_unzip_io_sb *iosb;
if (foreground) { if (foreground) {
/* waitqueue available for foreground io */ /* waitqueue available for foreground io */
...@@ -1264,11 +1120,11 @@ jobqueue_init(struct super_block *sb, ...@@ -1264,11 +1120,11 @@ jobqueue_init(struct super_block *sb,
iosb->sb = sb; iosb->sb = sb;
INIT_WORK(&io->u.work, z_erofs_vle_unzip_wq); INIT_WORK(&io->u.work, z_erofs_vle_unzip_wq);
out: out:
io->head = Z_EROFS_VLE_WORKGRP_TAIL_CLOSED; io->head = Z_EROFS_PCLUSTER_TAIL_CLOSED;
return io; return io;
} }
/* define workgroup jobqueue types */ /* define decompression jobqueue types */
enum { enum {
#ifdef EROFS_FS_HAS_MANAGED_CACHE #ifdef EROFS_FS_HAS_MANAGED_CACHE
JQ_BYPASS, JQ_BYPASS,
...@@ -1278,15 +1134,15 @@ enum { ...@@ -1278,15 +1134,15 @@ enum {
}; };
static void *jobqueueset_init(struct super_block *sb, static void *jobqueueset_init(struct super_block *sb,
z_erofs_vle_owned_workgrp_t qtail[], z_erofs_next_pcluster_t qtail[],
struct z_erofs_vle_unzip_io *q[], struct z_erofs_unzip_io *q[],
struct z_erofs_vle_unzip_io *fgq, struct z_erofs_unzip_io *fgq,
bool forcefg) bool forcefg)
{ {
#ifdef EROFS_FS_HAS_MANAGED_CACHE #ifdef EROFS_FS_HAS_MANAGED_CACHE
/* /*
* if managed cache is enabled, bypass jobqueue is needed, * if managed cache is enabled, bypass jobqueue is needed,
* no need to read from device for all workgroups in this queue. * no need to read from device for all pclusters in this queue.
*/ */
q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, true); q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, true);
qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head; qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head;
...@@ -1299,26 +1155,26 @@ static void *jobqueueset_init(struct super_block *sb, ...@@ -1299,26 +1155,26 @@ static void *jobqueueset_init(struct super_block *sb,
} }
#ifdef EROFS_FS_HAS_MANAGED_CACHE #ifdef EROFS_FS_HAS_MANAGED_CACHE
static void move_to_bypass_jobqueue(struct z_erofs_vle_workgroup *grp, static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl,
z_erofs_vle_owned_workgrp_t qtail[], z_erofs_next_pcluster_t qtail[],
z_erofs_vle_owned_workgrp_t owned_head) z_erofs_next_pcluster_t owned_head)
{ {
z_erofs_vle_owned_workgrp_t *const submit_qtail = qtail[JQ_SUBMIT]; z_erofs_next_pcluster_t *const submit_qtail = qtail[JQ_SUBMIT];
z_erofs_vle_owned_workgrp_t *const bypass_qtail = qtail[JQ_BYPASS]; z_erofs_next_pcluster_t *const bypass_qtail = qtail[JQ_BYPASS];
DBG_BUGON(owned_head == Z_EROFS_VLE_WORKGRP_TAIL_CLOSED); DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
if (owned_head == Z_EROFS_VLE_WORKGRP_TAIL) if (owned_head == Z_EROFS_PCLUSTER_TAIL)
owned_head = Z_EROFS_VLE_WORKGRP_TAIL_CLOSED; owned_head = Z_EROFS_PCLUSTER_TAIL_CLOSED;
WRITE_ONCE(grp->next, Z_EROFS_VLE_WORKGRP_TAIL_CLOSED); WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED);
WRITE_ONCE(*submit_qtail, owned_head); WRITE_ONCE(*submit_qtail, owned_head);
WRITE_ONCE(*bypass_qtail, &grp->next); WRITE_ONCE(*bypass_qtail, &pcl->next);
qtail[JQ_BYPASS] = &grp->next; qtail[JQ_BYPASS] = &pcl->next;
} }
static bool postsubmit_is_all_bypassed(struct z_erofs_vle_unzip_io *q[], static bool postsubmit_is_all_bypassed(struct z_erofs_unzip_io *q[],
unsigned int nr_bios, unsigned int nr_bios,
bool force_fg) bool force_fg)
{ {
...@@ -1329,21 +1185,19 @@ static bool postsubmit_is_all_bypassed(struct z_erofs_vle_unzip_io *q[], ...@@ -1329,21 +1185,19 @@ static bool postsubmit_is_all_bypassed(struct z_erofs_vle_unzip_io *q[],
if (force_fg || nr_bios) if (force_fg || nr_bios)
return false; return false;
kvfree(container_of(q[JQ_SUBMIT], kvfree(container_of(q[JQ_SUBMIT], struct z_erofs_unzip_io_sb, io));
struct z_erofs_vle_unzip_io_sb,
io));
return true; return true;
} }
#else #else
static void move_to_bypass_jobqueue(struct z_erofs_vle_workgroup *grp, static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl,
z_erofs_vle_owned_workgrp_t qtail[], z_erofs_next_pcluster_t qtail[],
z_erofs_vle_owned_workgrp_t owned_head) z_erofs_next_pcluster_t owned_head)
{ {
/* impossible to bypass submission for managed cache disabled */ /* impossible to bypass submission for managed cache disabled */
DBG_BUGON(1); DBG_BUGON(1);
} }
static bool postsubmit_is_all_bypassed(struct z_erofs_vle_unzip_io *q[], static bool postsubmit_is_all_bypassed(struct z_erofs_unzip_io *q[],
unsigned int nr_bios, unsigned int nr_bios,
bool force_fg) bool force_fg)
{ {
...@@ -1354,17 +1208,14 @@ static bool postsubmit_is_all_bypassed(struct z_erofs_vle_unzip_io *q[], ...@@ -1354,17 +1208,14 @@ static bool postsubmit_is_all_bypassed(struct z_erofs_vle_unzip_io *q[],
#endif #endif
static bool z_erofs_vle_submit_all(struct super_block *sb, static bool z_erofs_vle_submit_all(struct super_block *sb,
z_erofs_vle_owned_workgrp_t owned_head, z_erofs_next_pcluster_t owned_head,
struct list_head *pagepool, struct list_head *pagepool,
struct z_erofs_vle_unzip_io *fgq, struct z_erofs_unzip_io *fgq,
bool force_fg) bool force_fg)
{ {
struct erofs_sb_info *const sbi = EROFS_SB(sb); struct erofs_sb_info *const sbi __maybe_unused = EROFS_SB(sb);
const unsigned int clusterpages = erofs_clusterpages(sbi); z_erofs_next_pcluster_t qtail[NR_JOBQUEUES];
const gfp_t gfp = GFP_NOFS; struct z_erofs_unzip_io *q[NR_JOBQUEUES];
z_erofs_vle_owned_workgrp_t qtail[NR_JOBQUEUES];
struct z_erofs_vle_unzip_io *q[NR_JOBQUEUES];
struct bio *bio; struct bio *bio;
void *bi_private; void *bi_private;
/* since bio will be NULL, no need to initialize last_index */ /* since bio will be NULL, no need to initialize last_index */
...@@ -1372,7 +1223,7 @@ static bool z_erofs_vle_submit_all(struct super_block *sb, ...@@ -1372,7 +1223,7 @@ static bool z_erofs_vle_submit_all(struct super_block *sb,
bool force_submit = false; bool force_submit = false;
unsigned int nr_bios; unsigned int nr_bios;
if (unlikely(owned_head == Z_EROFS_VLE_WORKGRP_TAIL)) if (unlikely(owned_head == Z_EROFS_PCLUSTER_TAIL))
return false; return false;
force_submit = false; force_submit = false;
...@@ -1384,29 +1235,32 @@ static bool z_erofs_vle_submit_all(struct super_block *sb, ...@@ -1384,29 +1235,32 @@ static bool z_erofs_vle_submit_all(struct super_block *sb,
q[JQ_SUBMIT]->head = owned_head; q[JQ_SUBMIT]->head = owned_head;
do { do {
struct z_erofs_vle_workgroup *grp; struct z_erofs_pcluster *pcl;
unsigned int clusterpages;
pgoff_t first_index; pgoff_t first_index;
struct page *page; struct page *page;
unsigned int i = 0, bypass = 0; unsigned int i = 0, bypass = 0;
int err; int err;
/* no possible 'owned_head' equals the following */ /* no possible 'owned_head' equals the following */
DBG_BUGON(owned_head == Z_EROFS_VLE_WORKGRP_TAIL_CLOSED); DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
DBG_BUGON(owned_head == Z_EROFS_VLE_WORKGRP_NIL); DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_NIL);
pcl = container_of(owned_head, struct z_erofs_pcluster, next);
grp = container_of(owned_head, clusterpages = BIT(pcl->clusterbits);
struct z_erofs_vle_workgroup, next);
/* close the main owned chain at first */ /* close the main owned chain at first */
owned_head = cmpxchg(&grp->next, Z_EROFS_VLE_WORKGRP_TAIL, owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,
Z_EROFS_VLE_WORKGRP_TAIL_CLOSED); Z_EROFS_PCLUSTER_TAIL_CLOSED);
first_index = grp->obj.index; first_index = pcl->obj.index;
force_submit |= (first_index != last_index + 1); force_submit |= (first_index != last_index + 1);
repeat: repeat:
page = pickup_page_for_submission(grp, i, pagepool, page = pickup_page_for_submission(pcl, i, pagepool,
MNGD_MAPPING(sbi), gfp); MNGD_MAPPING(sbi),
GFP_NOFS);
if (!page) { if (!page) {
force_submit = true; force_submit = true;
++bypass; ++bypass;
...@@ -1437,10 +1291,10 @@ static bool z_erofs_vle_submit_all(struct super_block *sb, ...@@ -1437,10 +1291,10 @@ static bool z_erofs_vle_submit_all(struct super_block *sb,
goto repeat; goto repeat;
if (bypass < clusterpages) if (bypass < clusterpages)
qtail[JQ_SUBMIT] = &grp->next; qtail[JQ_SUBMIT] = &pcl->next;
else else
move_to_bypass_jobqueue(grp, qtail, owned_head); move_to_bypass_jobqueue(pcl, qtail, owned_head);
} while (owned_head != Z_EROFS_VLE_WORKGRP_TAIL); } while (owned_head != Z_EROFS_PCLUSTER_TAIL);
if (bio) if (bio)
__submit_bio(bio, REQ_OP_READ, 0); __submit_bio(bio, REQ_OP_READ, 0);
...@@ -1452,17 +1306,19 @@ static bool z_erofs_vle_submit_all(struct super_block *sb, ...@@ -1452,17 +1306,19 @@ static bool z_erofs_vle_submit_all(struct super_block *sb,
return true; return true;
} }
static void z_erofs_submit_and_unzip(struct z_erofs_vle_frontend *f, static void z_erofs_submit_and_unzip(struct super_block *sb,
struct z_erofs_collector *clt,
struct list_head *pagepool, struct list_head *pagepool,
bool force_fg) bool force_fg)
{ {
struct super_block *sb = f->inode->i_sb; struct z_erofs_unzip_io io[NR_JOBQUEUES];
struct z_erofs_vle_unzip_io io[NR_JOBQUEUES];
if (!z_erofs_vle_submit_all(sb, f->owned_head, pagepool, io, force_fg)) if (!z_erofs_vle_submit_all(sb, clt->owned_head,
pagepool, io, force_fg))
return; return;
#ifdef EROFS_FS_HAS_MANAGED_CACHE #ifdef EROFS_FS_HAS_MANAGED_CACHE
/* decompress no I/O pclusters immediately */
z_erofs_vle_unzip_all(sb, &io[JQ_BYPASS], pagepool); z_erofs_vle_unzip_all(sb, &io[JQ_BYPASS], pagepool);
#endif #endif
if (!force_fg) if (!force_fg)
...@@ -1480,7 +1336,7 @@ static int z_erofs_vle_normalaccess_readpage(struct file *file, ...@@ -1480,7 +1336,7 @@ static int z_erofs_vle_normalaccess_readpage(struct file *file,
struct page *page) struct page *page)
{ {
struct inode *const inode = page->mapping->host; struct inode *const inode = page->mapping->host;
struct z_erofs_vle_frontend f = VLE_FRONTEND_INIT(inode); struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode);
int err; int err;
LIST_HEAD(pagepool); LIST_HEAD(pagepool);
...@@ -1489,14 +1345,14 @@ static int z_erofs_vle_normalaccess_readpage(struct file *file, ...@@ -1489,14 +1345,14 @@ static int z_erofs_vle_normalaccess_readpage(struct file *file,
f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT; f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT;
err = z_erofs_do_read_page(&f, page, &pagepool); err = z_erofs_do_read_page(&f, page, &pagepool);
(void)z_erofs_vle_work_iter_end(&f.builder); (void)z_erofs_collector_end(&f.clt);
if (err) { if (err) {
errln("%s, failed to read, err [%d]", __func__, err); errln("%s, failed to read, err [%d]", __func__, err);
goto out; goto out;
} }
z_erofs_submit_and_unzip(&f, &pagepool, true); z_erofs_submit_and_unzip(inode->i_sb, &f.clt, &pagepool, true);
out: out:
if (f.map.mpage) if (f.map.mpage)
put_page(f.map.mpage); put_page(f.map.mpage);
...@@ -1521,7 +1377,7 @@ static int z_erofs_vle_normalaccess_readpages(struct file *filp, ...@@ -1521,7 +1377,7 @@ static int z_erofs_vle_normalaccess_readpages(struct file *filp,
struct erofs_sb_info *const sbi = EROFS_I_SB(inode); struct erofs_sb_info *const sbi = EROFS_I_SB(inode);
bool sync = should_decompress_synchronously(sbi, nr_pages); bool sync = should_decompress_synchronously(sbi, nr_pages);
struct z_erofs_vle_frontend f = VLE_FRONTEND_INIT(inode); struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode);
gfp_t gfp = mapping_gfp_constraint(mapping, GFP_KERNEL); gfp_t gfp = mapping_gfp_constraint(mapping, GFP_KERNEL);
struct page *head = NULL; struct page *head = NULL;
LIST_HEAD(pagepool); LIST_HEAD(pagepool);
...@@ -1567,13 +1423,12 @@ static int z_erofs_vle_normalaccess_readpages(struct file *filp, ...@@ -1567,13 +1423,12 @@ static int z_erofs_vle_normalaccess_readpages(struct file *filp,
errln("%s, readahead error at page %lu of nid %llu", errln("%s, readahead error at page %lu of nid %llu",
__func__, page->index, vi->nid); __func__, page->index, vi->nid);
} }
put_page(page); put_page(page);
} }
(void)z_erofs_vle_work_iter_end(&f.builder); (void)z_erofs_collector_end(&f.clt);
z_erofs_submit_and_unzip(&f, &pagepool, sync); z_erofs_submit_and_unzip(inode->i_sb, &f.clt, &pagepool, sync);
if (f.map.mpage) if (f.map.mpage)
put_page(f.map.mpage); put_page(f.map.mpage);
......
drivers/staging/erofs/zdata.h
View file @ 97e86a85
...@@ -18,80 +18,77 @@ ...@@ -18,80 +18,77 @@
* Structure fields follow one of the following exclusion rules. * Structure fields follow one of the following exclusion rules.
* *
* I: Modifiable by initialization/destruction paths and read-only * I: Modifiable by initialization/destruction paths and read-only
* for everyone else. * for everyone else;
* *
* L: Field should be protected by pageset lock;
*
* A: Field should be accessed / updated in atomic for parallelized code.
*/ */
struct z_erofs_collection {
struct z_erofs_vle_work {
struct mutex lock; struct mutex lock;
/* I: decompression offset in page */ /* I: page offset of start position of decompression */
unsigned short pageofs; unsigned short pageofs;
/* L: maximum relative page index in pagevec[] */
unsigned short nr_pages; unsigned short nr_pages;
/* L: queued pages in pagevec[] */ /* L: total number of pages in pagevec[] */
unsigned int vcnt; unsigned int vcnt;
union { union {
/* L: pagevec */ /* L: inline a certain number of pagevecs for bootstrap */
erofs_vtptr_t pagevec[Z_EROFS_NR_INLINE_PAGEVECS]; erofs_vtptr_t pagevec[Z_EROFS_NR_INLINE_PAGEVECS];
/* I: can be used to free the pcluster by RCU. */
struct rcu_head rcu; struct rcu_head rcu;
}; };
}; };
#define Z_EROFS_VLE_WORKGRP_FMT_PLAIN 0 #define Z_EROFS_PCLUSTER_FULL_LENGTH 0x00000001
#define Z_EROFS_VLE_WORKGRP_FMT_LZ4 1 #define Z_EROFS_PCLUSTER_LENGTH_BIT 1
#define Z_EROFS_VLE_WORKGRP_FMT_MASK 1
#define Z_EROFS_VLE_WORKGRP_FULL_LENGTH 2
typedef void *z_erofs_vle_owned_workgrp_t; /*
* let's leave a type here in case of introducing
* another tagged pointer later.
*/
typedef void *z_erofs_next_pcluster_t;
struct z_erofs_vle_workgroup { struct z_erofs_pcluster {
struct erofs_workgroup obj; struct erofs_workgroup obj;
struct z_erofs_vle_work work; struct z_erofs_collection primary_collection;
/* point to next owned_workgrp_t */ /* A: point to next chained pcluster or TAILs */
z_erofs_vle_owned_workgrp_t next; z_erofs_next_pcluster_t next;
/* compressed pages (including multi-usage pages) */ /* A: compressed pages (including multi-usage pages) */
struct page *compressed_pages[Z_EROFS_CLUSTER_MAX_PAGES]; struct page *compressed_pages[Z_EROFS_CLUSTER_MAX_PAGES];
unsigned int llen, flags;
/* A: lower limit of decompressed length and if full length or not */
unsigned int length;
/* I: compression algorithm format */
unsigned char algorithmformat;
/* I: bit shift of physical cluster size */
unsigned char clusterbits;
}; };
#define z_erofs_primarycollection(pcluster) (&(pcluster)->primary_collection)
/* let's avoid the valid 32-bit kernel addresses */ /* let's avoid the valid 32-bit kernel addresses */
/* the chained workgroup has't submitted io (still open) */ /* the chained workgroup has't submitted io (still open) */
#define Z_EROFS_VLE_WORKGRP_TAIL ((void *)0x5F0ECAFE) #define Z_EROFS_PCLUSTER_TAIL ((void *)0x5F0ECAFE)
/* the chained workgroup has already submitted io */ /* the chained workgroup has already submitted io */
#define Z_EROFS_VLE_WORKGRP_TAIL_CLOSED ((void *)0x5F0EDEAD) #define Z_EROFS_PCLUSTER_TAIL_CLOSED ((void *)0x5F0EDEAD)
#define Z_EROFS_VLE_WORKGRP_NIL (NULL) #define Z_EROFS_PCLUSTER_NIL (NULL)
#define z_erofs_vle_workgrp_fmt(grp) \ #define Z_EROFS_WORKGROUP_SIZE sizeof(struct z_erofs_pcluster)
((grp)->flags & Z_EROFS_VLE_WORKGRP_FMT_MASK)
static inline void z_erofs_vle_set_workgrp_fmt( struct z_erofs_unzip_io {
struct z_erofs_vle_workgroup *grp,
unsigned int fmt)
{
grp->flags = fmt | (grp->flags & ~Z_EROFS_VLE_WORKGRP_FMT_MASK);
}
/* definitions if multiref is disabled */
#define z_erofs_vle_grab_primary_work(grp) (&(grp)->work)
#define z_erofs_vle_grab_work(grp, pageofs) (&(grp)->work)
#define z_erofs_vle_work_workgroup(wrk, primary) \
((primary) ? container_of(wrk, \
struct z_erofs_vle_workgroup, work) : \
({ BUG(); (void *)NULL; }))
#define Z_EROFS_WORKGROUP_SIZE sizeof(struct z_erofs_vle_workgroup)
struct z_erofs_vle_unzip_io {
atomic_t pending_bios; atomic_t pending_bios;
z_erofs_vle_owned_workgrp_t head; z_erofs_next_pcluster_t head;
union { union {
wait_queue_head_t wait; wait_queue_head_t wait;
...@@ -99,8 +96,8 @@ struct z_erofs_vle_unzip_io { ...@@ -99,8 +96,8 @@ struct z_erofs_vle_unzip_io {
} u; } u;
}; };
struct z_erofs_vle_unzip_io_sb { struct z_erofs_unzip_io_sb {
struct z_erofs_vle_unzip_io io; struct z_erofs_unzip_io io;
struct super_block *sb; struct super_block *sb;
}; };
...@@ -117,8 +114,8 @@ static inline bool erofs_page_is_managed(const struct erofs_sb_info *sbi, ...@@ -117,8 +114,8 @@ static inline bool erofs_page_is_managed(const struct erofs_sb_info *sbi,
struct page *page) { return false; } struct page *page) { return false; }
#endif /* !EROFS_FS_HAS_MANAGED_CACHE */ #endif /* !EROFS_FS_HAS_MANAGED_CACHE */
#define Z_EROFS_ONLINEPAGE_COUNT_BITS 2 #define Z_EROFS_ONLINEPAGE_COUNT_BITS 2
#define Z_EROFS_ONLINEPAGE_COUNT_MASK ((1 << Z_EROFS_ONLINEPAGE_COUNT_BITS) - 1) #define Z_EROFS_ONLINEPAGE_COUNT_MASK ((1 << Z_EROFS_ONLINEPAGE_COUNT_BITS) - 1)
#define Z_EROFS_ONLINEPAGE_INDEX_SHIFT (Z_EROFS_ONLINEPAGE_COUNT_BITS) #define Z_EROFS_ONLINEPAGE_INDEX_SHIFT (Z_EROFS_ONLINEPAGE_COUNT_BITS)
/* /*
...@@ -193,13 +190,12 @@ static inline void z_erofs_onlinepage_endio(struct page *page) ...@@ -193,13 +190,12 @@ static inline void z_erofs_onlinepage_endio(struct page *page)
SetPageUptodate(page); SetPageUptodate(page);
unlock_page(page); unlock_page(page);
} }
debugln("%s, page %p value %x", __func__, page, atomic_read(u.o)); debugln("%s, page %p value %x", __func__, page, atomic_read(u.o));
} }
#define Z_EROFS_VLE_VMAP_ONSTACK_PAGES \ #define Z_EROFS_VMAP_ONSTACK_PAGES \
min_t(unsigned int, THREAD_SIZE / 8 / sizeof(struct page *), 96U) min_t(unsigned int, THREAD_SIZE / 8 / sizeof(struct page *), 96U)
#define Z_EROFS_VLE_VMAP_GLOBAL_PAGES 2048 #define Z_EROFS_VMAP_GLOBAL_PAGES 2048
#endif #endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7