[PATCH v3 4/5] erofs: support inline data decompression
Yue Hu
zbestahu at gmail.com
Mon Dec 27 19:18:19 AEDT 2021
Hi Xiang,
On Sat, 25 Dec 2021 15:06:25 +0800
Gao Xiang <hsiangkao at linux.alibaba.com> wrote:
> From: Yue Hu <huyue2 at yulong.com>
>
> Currently, we have already support tail-packing inline for
> uncompressed file, let's also implement this for compressed
> files to save I/Os and storage space.
>
> Different from normal pclusters, compressed data is available
> in advance because of other metadata I/Os. Therefore, they
> directly move into the bypass queue without extra I/O submission.
>
> It's the last compression feature before folio/subpage support.
>
> Signed-off-by: Yue Hu <huyue2 at yulong.com>
> Signed-off-by: Gao Xiang <hsiangkao at linux.alibaba.com>
> ---
> fs/erofs/zdata.c | 128 ++++++++++++++++++++++++++++++++---------------
> fs/erofs/zdata.h | 24 ++++++++-
> 2 files changed, 109 insertions(+), 43 deletions(-)
>
> diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
> index bc765d8a6dc2..e6ef02634e08 100644
> --- a/fs/erofs/zdata.c
> +++ b/fs/erofs/zdata.c
> @@ -82,12 +82,13 @@ static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int nrpages)
>
> static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl)
> {
> + unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
> int i;
>
> for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) {
> struct z_erofs_pcluster_slab *pcs = pcluster_pool + i;
>
> - if (pcl->pclusterpages > pcs->maxpages)
> + if (pclusterpages > pcs->maxpages)
> continue;
>
> kmem_cache_free(pcs->slab, pcl);
> @@ -298,6 +299,7 @@ int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
> container_of(grp, struct z_erofs_pcluster, obj);
> int i;
>
> + DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
> /*
> * refcount of workgroup is now freezed as 1,
> * therefore no need to worry about available decompression users.
> @@ -331,6 +333,7 @@ int erofs_try_to_free_cached_page(struct page *page)
> if (erofs_workgroup_try_to_freeze(&pcl->obj, 1)) {
> unsigned int i;
>
> + DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
> for (i = 0; i < pcl->pclusterpages; ++i) {
> if (pcl->compressed_pages[i] == page) {
> WRITE_ONCE(pcl->compressed_pages[i], NULL);
> @@ -458,6 +461,7 @@ static int z_erofs_register_collection(struct z_erofs_collector *clt,
> struct inode *inode,
> struct erofs_map_blocks *map)
> {
> + bool ztailpacking = map->m_flags & EROFS_MAP_META;
> struct z_erofs_pcluster *pcl;
> struct z_erofs_collection *cl;
> struct erofs_workgroup *grp;
> @@ -469,12 +473,12 @@ static int z_erofs_register_collection(struct z_erofs_collector *clt,
> }
>
> /* no available pcluster, let's allocate one */
> - pcl = z_erofs_alloc_pcluster(map->m_plen >> PAGE_SHIFT);
> + pcl = z_erofs_alloc_pcluster(ztailpacking ? 1 :
> + map->m_plen >> PAGE_SHIFT);
> if (IS_ERR(pcl))
> return PTR_ERR(pcl);
>
> atomic_set(&pcl->obj.refcount, 1);
> - pcl->obj.index = map->m_pa >> PAGE_SHIFT;
> pcl->algorithmformat = map->m_algorithmformat;
> pcl->length = (map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) |
> (map->m_flags & EROFS_MAP_FULL_MAPPED ?
> @@ -494,16 +498,25 @@ static int z_erofs_register_collection(struct z_erofs_collector *clt,
> mutex_init(&cl->lock);
> DBG_BUGON(!mutex_trylock(&cl->lock));
>
> - grp = erofs_insert_workgroup(inode->i_sb, &pcl->obj);
> - if (IS_ERR(grp)) {
> - err = PTR_ERR(grp);
> - goto err_out;
> - }
> + if (ztailpacking) {
> + pcl->obj.index = 0; /* which indicates ztailpacking */
> + pcl->pageofs_in = erofs_blkoff(map->m_pa);
> + pcl->tailpacking_size = map->m_plen;
> + } else {
> + pcl->obj.index = map->m_pa >> PAGE_SHIFT;
>
> - if (grp != &pcl->obj) {
> - clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);
> - err = -EEXIST;
> - goto err_out;
> + grp = erofs_insert_workgroup(inode->i_sb, &pcl->obj);
> + if (IS_ERR(grp)) {
> + err = PTR_ERR(grp);
> + goto err_out;
> + }
> +
> + if (grp != &pcl->obj) {
> + clt->pcl = container_of(grp,
> + struct z_erofs_pcluster, obj);
> + err = -EEXIST;
> + goto err_out;
> + }
> }
> /* used to check tail merging loop due to corrupted images */
> if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)
> @@ -532,17 +545,20 @@ static int z_erofs_collector_begin(struct z_erofs_collector *clt,
> DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_NIL);
> DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
>
> - if (!PAGE_ALIGNED(map->m_pa)) {
> - DBG_BUGON(1);
> - return -EINVAL;
> + if (map->m_flags & EROFS_MAP_META) {
> + if ((map->m_pa & ~PAGE_MASK) + map->m_plen > PAGE_SIZE) {
> + DBG_BUGON(1);
> + return -EFSCORRUPTED;
> + }
> + goto tailpacking;
> }
>
> grp = erofs_find_workgroup(inode->i_sb, map->m_pa >> PAGE_SHIFT);
> if (grp) {
> clt->pcl = container_of(grp, struct z_erofs_pcluster, obj);
> } else {
> +tailpacking:
> ret = z_erofs_register_collection(clt, inode, map);
> -
> if (!ret)
> goto out;
> if (ret != -EEXIST)
> @@ -558,9 +574,9 @@ static int z_erofs_collector_begin(struct z_erofs_collector *clt,
> out:
> z_erofs_pagevec_ctor_init(&clt->vector, Z_EROFS_NR_INLINE_PAGEVECS,
> clt->cl->pagevec, clt->cl->vcnt);
> -
> /* since file-backed online pages are traversed in reverse order */
> - clt->icpage_ptr = clt->pcl->compressed_pages + clt->pcl->pclusterpages;
> + clt->icpage_ptr = clt->pcl->compressed_pages +
> + z_erofs_pclusterpages(clt->pcl);
> return 0;
> }
>
> @@ -687,8 +703,26 @@ static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe,
> else
> cache_strategy = DONTALLOC;
>
> - preload_compressed_pages(clt, MNGD_MAPPING(sbi),
> - cache_strategy, pagepool);
> + if (z_erofs_is_inline_pcluster(clt->pcl)) {
current cache_strategy is only for preload_compressed_pages(), so the cache_strategy should be
needless for inline branch.
> + struct page *mpage;
> +
> + mpage = erofs_get_meta_page(inode->i_sb,
> + erofs_blknr(map->m_pa));
could we just use the map->mpage directly if it's what we want(which is the most cases when test),
if not we erofs_get_meta_page()?
> + if (IS_ERR(mpage)) {
> + err = PTR_ERR(mpage);
> + erofs_err(inode->i_sb,
> + "failed to get inline page, err %d", err);
> + goto err_out;
> + }
> + /* TODO: new subpage feature will get rid of it */
> + unlock_page(mpage);
> +
> + WRITE_ONCE(clt->pcl->compressed_pages[0], mpage);
> + clt->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE;
> + } else {
> + preload_compressed_pages(clt, MNGD_MAPPING(sbi),
> + cache_strategy, pagepool);
> + }
>
> hitted:
> /*
> @@ -844,6 +878,7 @@ static int z_erofs_decompress_pcluster(struct super_block *sb,
> struct page **pagepool)
> {
> struct erofs_sb_info *const sbi = EROFS_SB(sb);
> + unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
> struct z_erofs_pagevec_ctor ctor;
> unsigned int i, inputsize, outputsize, llen, nr_pages;
> struct page *pages_onstack[Z_EROFS_VMAP_ONSTACK_PAGES];
> @@ -925,16 +960,15 @@ static int z_erofs_decompress_pcluster(struct super_block *sb,
> overlapped = false;
> compressed_pages = pcl->compressed_pages;
>
> - for (i = 0; i < pcl->pclusterpages; ++i) {
> + for (i = 0; i < pclusterpages; ++i) {
> unsigned int pagenr;
>
> page = compressed_pages[i];
> -
> /* all compressed pages ought to be valid */
> DBG_BUGON(!page);
> - DBG_BUGON(z_erofs_page_is_invalidated(page));
>
> - if (!z_erofs_is_shortlived_page(page)) {
> + if (!z_erofs_is_inline_pcluster(pcl) &&
some inline checks may exist for noinline case if it's bigpcluster. And i understand the
behavior of ztailpacking page is differ from normal page. So better to branch them? moving
the inline check outside the for loop?
> + !z_erofs_is_shortlived_page(page)) {
> if (erofs_page_is_managed(sbi, page)) {
> if (!PageUptodate(page))
> err = -EIO;
> @@ -960,10 +994,8 @@ static int z_erofs_decompress_pcluster(struct super_block *sb,
> }
>
> /* PG_error needs checking for all non-managed pages */
> - if (PageError(page)) {
> - DBG_BUGON(PageUptodate(page));
> + if (!PageUptodate(page))
> err = -EIO;
> - }
> }
>
> if (err)
> @@ -978,11 +1010,16 @@ static int z_erofs_decompress_pcluster(struct super_block *sb,
> partial = true;
> }
>
> - inputsize = pcl->pclusterpages * PAGE_SIZE;
> + if (z_erofs_is_inline_pcluster(pcl))
> + inputsize = pcl->tailpacking_size;
> + else
> + inputsize = pclusterpages * PAGE_SIZE;
> +
> err = z_erofs_decompress(&(struct z_erofs_decompress_req) {
> .sb = sb,
> .in = compressed_pages,
> .out = pages,
> + .pageofs_in = pcl->pageofs_in,
> .pageofs_out = cl->pageofs,
> .inputsize = inputsize,
> .outputsize = outputsize,
> @@ -992,17 +1029,22 @@ static int z_erofs_decompress_pcluster(struct super_block *sb,
> }, pagepool);
>
> out:
> - /* must handle all compressed pages before ending pages */
> - for (i = 0; i < pcl->pclusterpages; ++i) {
> - page = compressed_pages[i];
> -
> - if (erofs_page_is_managed(sbi, page))
> - continue;
> + /* must handle all compressed pages before actual file pages */
> + if (z_erofs_is_inline_pcluster(pcl)) {
> + page = compressed_pages[0];
> + WRITE_ONCE(compressed_pages[0], NULL);
> + put_page(page);
> + } else {
> + for (i = 0; i < pclusterpages; ++i) {
> + page = compressed_pages[i];
>
> - /* recycle all individual short-lived pages */
> - (void)z_erofs_put_shortlivedpage(pagepool, page);
> + if (erofs_page_is_managed(sbi, page))
> + continue;
>
> - WRITE_ONCE(compressed_pages[i], NULL);
> + /* recycle all individual short-lived pages */
> + (void)z_erofs_put_shortlivedpage(pagepool, page);
> + WRITE_ONCE(compressed_pages[i], NULL);
> + }
> }
>
> for (i = 0; i < nr_pages; ++i) {
> @@ -1288,6 +1330,14 @@ static void z_erofs_submit_queue(struct super_block *sb,
>
> pcl = container_of(owned_head, struct z_erofs_pcluster, next);
>
> + /* close the main owned chain at first */
> + owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,
> + Z_EROFS_PCLUSTER_TAIL_CLOSED);
> + if (z_erofs_is_inline_pcluster(pcl)) {
> + move_to_bypass_jobqueue(pcl, qtail, owned_head);
> + continue;
> + }
> +
> /* no device id here, thus it will always succeed */
> mdev = (struct erofs_map_dev) {
> .m_pa = blknr_to_addr(pcl->obj.index),
> @@ -1297,10 +1347,6 @@ static void z_erofs_submit_queue(struct super_block *sb,
> cur = erofs_blknr(mdev.m_pa);
> end = cur + pcl->pclusterpages;
>
> - /* close the main owned chain at first */
> - owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,
> - Z_EROFS_PCLUSTER_TAIL_CLOSED);
> -
> do {
> struct page *page;
>
> diff --git a/fs/erofs/zdata.h b/fs/erofs/zdata.h
> index 4a69515dea75..e043216b545f 100644
> --- a/fs/erofs/zdata.h
> +++ b/fs/erofs/zdata.h
> @@ -62,8 +62,16 @@ struct z_erofs_pcluster {
> /* A: lower limit of decompressed length and if full length or not */
> unsigned int length;
>
> - /* I: physical cluster size in pages */
> - unsigned short pclusterpages;
> + /* I: page offset of inline compressed data */
> + unsigned short pageofs_in;
> +
> + union {
> + /* I: physical cluster size in pages */
> + unsigned short pclusterpages;
> +
> + /* I: tailpacking inline compressed size */
> + unsigned short tailpacking_size;
> + };
>
> /* I: compression algorithm format */
> unsigned char algorithmformat;
> @@ -94,6 +102,18 @@ struct z_erofs_decompressqueue {
> } u;
> };
>
> +static inline bool z_erofs_is_inline_pcluster(struct z_erofs_pcluster *pcl)
> +{
> + return !pcl->obj.index;
> +}
> +
> +static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl)
> +{
> + if (z_erofs_is_inline_pcluster(pcl))
> + return 1;
> + return pcl->pclusterpages;
> +}
> +
> #define Z_EROFS_ONLINEPAGE_COUNT_BITS 2
> #define Z_EROFS_ONLINEPAGE_COUNT_MASK ((1 << Z_EROFS_ONLINEPAGE_COUNT_BITS) - 1)
> #define Z_EROFS_ONLINEPAGE_INDEX_SHIFT (Z_EROFS_ONLINEPAGE_COUNT_BITS)
More information about the Linux-erofs
mailing list