[PATCH v2] erofs: add a global page pool for lz4 decompression
Chunhai Guo
guochunhai at vivo.com
Tue Jan 9 18:41:43 AEDT 2024
Using a global page pool for LZ4 decompression significantly reduces the
time spent on page allocation in low memory scenarios.
The table below shows the reduction in time spent on page allocation for
LZ4 decompression when using a global page pool. The results were
obtained from multi-app launch benchmarks on ARM64 Android devices
running the 5.15 kernel with an 8-core CPU and 8GB of memory. In the
benchmark, we launched 16 frequently-used apps, and the camera app was
the last one in each round. The data in the table is the average time of
camera app for each round.
After using the page pool, there was an average improvement of 150ms in
the launch time of the camera app, which was obtained from systrace log.
+--------------+---------------+--------------+---------+
| | w/o page pool | w/ page pool | diff |
+--------------+---------------+--------------+---------+
| Average (ms) | 3434 | 21 | -99.38% |
+--------------+---------------+--------------+---------+
Based on the benchmark logs, 64 pages are sufficient for 95% of
scenarios. This value can be adjusted from the module parameter. The
default value is 0.
This patch currently only supports the LZ4 decompressor, other
decompressors will be supported in the next step.
Signed-off-by: Chunhai Guo <guochunhai at vivo.com>
---
V1 -> V2: reuse shortlived interfaces for the global pool and support
configuration based on module parameters
---
fs/erofs/decompressor.c | 5 ++-
fs/erofs/internal.h | 15 +++++++-
fs/erofs/super.c | 1 +
fs/erofs/utils.c | 82 +++++++++++++++++++++++++++++++++++++++--
4 files changed, 97 insertions(+), 6 deletions(-)
diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
index d08a6ee23ac5..49aa40e5c260 100644
--- a/fs/erofs/decompressor.c
+++ b/fs/erofs/decompressor.c
@@ -54,6 +54,7 @@ static int z_erofs_load_lz4_config(struct super_block *sb,
sbi->lz4.max_distance_pages = distance ?
DIV_ROUND_UP(distance, PAGE_SIZE) + 1 :
LZ4_MAX_DISTANCE_PAGES;
+ erofs_global_page_pool_init();
return erofs_pcpubuf_growsize(sbi->lz4.max_pclusterblks);
}
@@ -111,8 +112,8 @@ static int z_erofs_lz4_prepare_dstpages(struct z_erofs_lz4_decompress_ctx *ctx,
victim = availables[--top];
get_page(victim);
} else {
- victim = erofs_allocpage(pagepool,
- GFP_KERNEL | __GFP_NOFAIL);
+ victim = erofs_allocpage_global(pagepool,
+ GFP_KERNEL | __GFP_NOFAIL);
set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
}
rq->out[i] = victim;
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index b0409badb017..d078ccae4f11 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -445,8 +445,21 @@ int erofs_register_sysfs(struct super_block *sb);
void erofs_unregister_sysfs(struct super_block *sb);
int __init erofs_init_sysfs(void);
void erofs_exit_sysfs(void);
+int erofs_global_page_pool_init(void);
+void erofs_global_page_pool_exit(void);
+struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp,
+ bool global_allowed);
+static inline struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)
+{
+ return __erofs_allocpage(pagepool, gfp, false);
+}
+
+static inline struct page *erofs_allocpage_global(struct page **pagepool,
+ gfp_t gfp)
+{
+ return __erofs_allocpage(pagepool, gfp, true);
+}
-struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp);
static inline void erofs_pagepool_add(struct page **pagepool, struct page *page)
{
set_page_private(page, (unsigned long)*pagepool);
diff --git a/fs/erofs/super.c b/fs/erofs/super.c
index 5f60f163bd56..56ddacb1bd6e 100644
--- a/fs/erofs/super.c
+++ b/fs/erofs/super.c
@@ -946,6 +946,7 @@ static void __exit erofs_module_exit(void)
erofs_exit_shrinker();
kmem_cache_destroy(erofs_inode_cachep);
erofs_pcpubuf_exit();
+ erofs_global_page_pool_exit();
}
static int erofs_statfs(struct dentry *dentry, struct kstatfs *buf)
diff --git a/fs/erofs/utils.c b/fs/erofs/utils.c
index 5dea308764b4..1b0cec47c00c 100644
--- a/fs/erofs/utils.c
+++ b/fs/erofs/utils.c
@@ -5,7 +5,19 @@
*/
#include "internal.h"
-struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)
+struct z_erofs_global_pool {
+ struct page *pagepool;
+ spinlock_t lock;
+ unsigned int total, unused;
+ bool inited;
+};
+
+static struct z_erofs_global_pool z_erofs_global_pool;
+
+module_param_named(global_pool_size, z_erofs_global_pool.total, uint, 0444);
+
+struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp,
+ bool from_global)
{
struct page *page = *pagepool;
@@ -13,7 +25,19 @@ struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)
DBG_BUGON(page_ref_count(page) != 1);
*pagepool = (struct page *)page_private(page);
} else {
- page = alloc_page(gfp);
+ if (from_global) {
+ spin_lock(&z_erofs_global_pool.lock);
+ page = z_erofs_global_pool.pagepool;
+ if (page) {
+ z_erofs_global_pool.pagepool =
+ (struct page *)page_private(page);
+ DBG_BUGON(page_ref_count(page) != 1);
+ z_erofs_global_pool.unused--;
+ }
+ spin_unlock(&z_erofs_global_pool.lock);
+ }
+ if (!page)
+ page = alloc_page(gfp);
}
return page;
}
@@ -24,7 +48,20 @@ void erofs_release_pages(struct page **pagepool)
struct page *page = *pagepool;
*pagepool = (struct page *)page_private(page);
- put_page(page);
+ if (z_erofs_global_pool.total) {
+ spin_lock(&z_erofs_global_pool.lock);
+ if (z_erofs_global_pool.unused
+ < z_erofs_global_pool.total) {
+ erofs_pagepool_add(&z_erofs_global_pool.pagepool,
+ page);
+ z_erofs_global_pool.unused++;
+ } else {
+ put_page(page);
+ }
+ spin_unlock(&z_erofs_global_pool.lock);
+ } else {
+ put_page(page);
+ }
}
}
@@ -284,4 +321,43 @@ void erofs_exit_shrinker(void)
{
shrinker_free(erofs_shrinker_info);
}
+
+int erofs_global_page_pool_init(void)
+{
+ int i;
+ struct page *page;
+
+ if (z_erofs_global_pool.inited)
+ return 0;
+
+ spin_lock_init(&z_erofs_global_pool.lock);
+ z_erofs_global_pool.unused = 0;
+ z_erofs_global_pool.pagepool = NULL;
+ for (i = 0; i < z_erofs_global_pool.total; i++) {
+ page = alloc_page(GFP_KERNEL);
+ if (!page) {
+ erofs_err(NULL, "failed to alloc page for erofs page pool\n");
+ return 0;
+ }
+ erofs_pagepool_add(&z_erofs_global_pool.pagepool, page);
+ z_erofs_global_pool.unused++;
+ }
+ z_erofs_global_pool.inited = true;
+ return 0;
+}
+
+void erofs_global_page_pool_exit(void)
+{
+ struct page *pagepool = z_erofs_global_pool.pagepool;
+
+ while (pagepool) {
+ struct page *page = pagepool;
+
+ pagepool = (struct page *)page_private(page);
+ put_page(page);
+ z_erofs_global_pool.unused--;
+ }
+ z_erofs_global_pool.total = 0;
+}
+
#endif /* !CONFIG_EROFS_FS_ZIP */
--
2.25.1
More information about the Linux-erofs
mailing list