[PATCH v4 08/10] erofs-utils: lib: add erofs_rebuild_load_tree() helper
Gao Xiang
hsiangkao at linux.alibaba.com
Wed Aug 23 18:26:52 AEST 2023
On 2023/8/23 15:15, Jingbo Xu wrote:
> Add erofs_rebuild_load_tree() helper loading inode tree from given erofs
> image, and making it merged into a given inode tree in an overlayfs like
> model.
>
> Since we need to read the content of the symlink file from disk when
> loading tree, add dependency on zlib_LIBS for mkfs.erofs.
>
> Also rename tarerofs_dump_tree() to erofs_rebuild_dump_tree(), so that
> it could also be called from rebuild mode.
>
> Signed-off-by: Jingbo Xu <jefflexu at linux.alibaba.com>
> ---
> include/erofs/inode.h | 2 +-
> include/erofs/internal.h | 1 +
> include/erofs/rebuild.h | 2 +
> lib/inode.c | 4 +-
> lib/rebuild.c | 271 +++++++++++++++++++++++++++++++++++++++
> mkfs/main.c | 2 +-
> 6 files changed, 278 insertions(+), 4 deletions(-)
>
> diff --git a/include/erofs/inode.h b/include/erofs/inode.h
> index 1c602a8..fe9dda2 100644
> --- a/include/erofs/inode.h
> +++ b/include/erofs/inode.h
> @@ -32,7 +32,7 @@ unsigned int erofs_iput(struct erofs_inode *inode);
> erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
> struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
> const char *name);
> -int tarerofs_dump_tree(struct erofs_inode *dir);
> +int erofs_rebuild_dump_tree(struct erofs_inode *dir);
> int erofs_init_empty_dir(struct erofs_inode *dir);
> struct erofs_inode *erofs_new_inode(void);
> struct erofs_inode *erofs_mkfs_build_tree_from_path(const char *path);
> diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> index 455a73a..457c3dd 100644
> --- a/include/erofs/internal.h
> +++ b/include/erofs/internal.h
> @@ -108,6 +108,7 @@ struct erofs_sb_info {
>
> int devfd;
> u64 devsz;
> + dev_t dev;
> unsigned int nblobs;
> unsigned int blobfd[256];
> };
> diff --git a/include/erofs/rebuild.h b/include/erofs/rebuild.h
> index 92873c9..3ac074c 100644
> --- a/include/erofs/rebuild.h
> +++ b/include/erofs/rebuild.h
> @@ -12,6 +12,8 @@ extern "C"
> struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
> char *path, bool aufs, bool *whout, bool *opq);
>
> +int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi);
> +
> #ifdef __cplusplus
> }
> #endif
> diff --git a/lib/inode.c b/lib/inode.c
> index b967aab..1cd69ec 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -1323,7 +1323,7 @@ struct erofs_inode *erofs_mkfs_build_special_from_fd(int fd, const char *name)
> return inode;
> }
>
> -int tarerofs_dump_tree(struct erofs_inode *dir)
> +int erofs_rebuild_dump_tree(struct erofs_inode *dir)
> {
> struct erofs_dentry *d;
> unsigned int nr_subdirs;
> @@ -1391,7 +1391,7 @@ int tarerofs_dump_tree(struct erofs_inode *dir)
> continue;
>
> inode = erofs_igrab(d->inode);
> - ret = tarerofs_dump_tree(inode);
> + ret = erofs_rebuild_dump_tree(inode);
> dir->i_nlink += (erofs_mode_to_ftype(inode->i_mode) == EROFS_FT_DIR);
> erofs_iput(inode);
> if (ret)
> diff --git a/lib/rebuild.c b/lib/rebuild.c
> index 7aaa071..e2525fe 100644
> --- a/lib/rebuild.c
> +++ b/lib/rebuild.c
> @@ -3,9 +3,18 @@
> #include <unistd.h>
> #include <stdlib.h>
> #include <string.h>
> +#include <sys/stat.h>
> +#include <config.h>
> +#if defined(HAVE_SYS_SYSMACROS_H)
> +#include <sys/sysmacros.h>
> +#endif
> #include "erofs/print.h"
> #include "erofs/inode.h"
> #include "erofs/rebuild.h"
> +#include "erofs/io.h"
> +#include "erofs/dir.h"
> +#include "erofs/xattr.h"
> +#include "erofs/blobchunk.h"
> #include "erofs/internal.h"
>
> #ifdef HAVE_LINUX_AUFS_TYPE_H
> @@ -115,3 +124,265 @@ struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd,
> }
> return d;
> }
> +
> +static int erofs_rebuild_fill_inode_map(struct erofs_inode *inode,
> + struct erofs_inode *vi)
erofs_rebuilt_fixup_inode_indexes? Also could you use (dst, src)
instead of (inode, vi)?
> +{
> + int ret;
> + struct erofs_sb_info *sbi = vi->sbi;
> + unsigned int count, unit, chunkbits, i;
> + unsigned int deviceid = inode->dev;
> + erofs_off_t chunksize;
> + erofs_blk_t blkaddr;
> + struct erofs_inode_chunk_index *idx;
> +
> + /* TODO: fill data map in other layouts */
> + if (vi->datalayout != EROFS_INODE_CHUNK_BASED &&
> + vi->datalayout != EROFS_INODE_FLAT_PLAIN) {
> + erofs_err("unsupported datalayout %d", vi->datalayout);
> + return -EOPNOTSUPP;
> + }
> +
> + if (sbi->extra_devices) {
> + chunkbits = vi->u.chunkbits;
> + inode->u.chunkformat = vi->u.chunkformat;
> + } else {
> + chunkbits = ilog2(inode->i_size - 1) + 1;
> + if (chunkbits < sbi->blkszbits)
> + chunkbits = sbi->blkszbits;
> + if (chunkbits - sbi->blkszbits > EROFS_CHUNK_FORMAT_BLKBITS_MASK)
> + chunkbits = EROFS_CHUNK_FORMAT_BLKBITS_MASK + sbi->blkszbits;
> + inode->u.chunkformat = EROFS_CHUNK_FORMAT_INDEXES;
> + inode->u.chunkformat |= chunkbits - sbi->blkszbits;
> + }
> + chunksize = 1ULL << chunkbits;
> + count = DIV_ROUND_UP(inode->i_size, chunksize);
> +
> + unit = sizeof(struct erofs_inode_chunk_index);
> + inode->extent_isize = count * unit;
> + idx = malloc(max(sizeof(*idx), sizeof(void *)));
> + if (!idx)
> + return -ENOMEM;
> + inode->chunkindexes = idx;
> +
> + for (i = 0; i < count; i++) {
> + struct erofs_blobchunk *chunk;
> + struct erofs_map_blocks map = {
> + .index = UINT_MAX,
> + };
> +
> + map.m_la = i << chunkbits;
> + ret = erofs_map_blocks(vi, &map, 0);
> + if (ret)
> + goto err;
> +
> + blkaddr = erofs_blknr(sbi, map.m_pa);
> + chunk = erofs_get_unhashed_chunk(deviceid, blkaddr, 0);
> + if (IS_ERR(chunk)) {
> + ret = PTR_ERR(chunk);
> + goto err;
> + }
> + *(void **)idx++ = chunk;
> + erofs_dbg("\t%s: chunk %d (deviceid %u, blkaddr %u)",
> + inode->i_srcpath, i, deviceid, blkaddr);
> +
> + }
> + inode->datalayout = EROFS_INODE_CHUNK_BASED;
> + return 0;
> +err:
> + free(idx);
> + inode->chunkindexes = NULL;
> + return ret;
> +}
> +
> +static int erofs_rebuild_fill_inode(struct erofs_inode *inode,
> + struct erofs_inode *vi)
same here.
> +{
> + int ret = 0;
> +
> + inode->i_srcpath = strdup(vi->i_srcpath);
> + inode->i_mode = vi->i_mode;
> + inode->i_uid = vi->i_uid;
> + inode->i_gid = vi->i_gid;
> + inode->i_mtime = vi->i_mtime;
> + inode->i_ino[1] = vi->nid;
> + inode->i_nlink = 1;
> + inode->opaque = vi->opaque;
> + list_splice_tail(&vi->i_xattrs, &inode->i_xattrs);
> +
> + switch (inode->i_mode & S_IFMT) {
> + case S_IFCHR:
> + case S_IFBLK:
> + case S_IFIFO:
> + case S_IFSOCK:
> + inode->u.i_rdev = erofs_new_encode_dev(vi->u.i_rdev);
> + inode->i_size = 0;
> + erofs_dbg("\tdev: %d %d", major(vi->u.i_rdev),
> + minor(vi->u.i_rdev));
> + break;
> + case S_IFDIR:
> + inode->i_size = 0;
> + ret = erofs_init_empty_dir(inode);
> + break;
> + case S_IFLNK:
> + inode->i_size = vi->i_size;
> + inode->i_link = malloc(inode->i_size + 1);
> + ret = erofs_pread(vi, inode->i_link, inode->i_size, 0);
> + erofs_dbg("\tsymlink: %s -> %s", inode->i_srcpath, inode->i_link);
> + break;
> + case S_IFREG:
> + inode->i_size = vi->i_size;
> + if (inode->i_size)
> + ret = erofs_rebuild_fill_inode_map(inode, vi);
> + else
> + inode->u.i_blkaddr = NULL_ADDR;
> + break;
> + default:
> + ret = -EINVAL;
> + break;
> + }
> + return ret;
> +}
> +
> +struct erofs_rebuild_dir_context {
> + struct erofs_dir_context ctx;
> + struct erofs_inode *root;
> + dev_t dev;
> +};
> +
> +static int erofs_rebuild_parse_inode(struct erofs_rebuild_dir_context *rctx,
> + struct erofs_inode *vi)
> +{
> + struct erofs_inode *inode;
> + struct erofs_dentry *d;
> + int ret;
> +
> + d = erofs_rebuild_get_dentry(rctx->root, vi->i_srcpath,
> + false, NULL, NULL);
I'm not sure if it's effective, I think we should just record parent nid and lookup
subdir instead.
> + if (IS_ERR(d))
> + return PTR_ERR(d);
> +
> + if (d->type != EROFS_FT_UNKNOWN) {
> + inode = d->inode;
> + DBG_BUGON((inode->i_mode & S_IFMT) != (vi->i_mode & S_IFMT));
> + if (!S_ISDIR(inode->i_mode) || inode->opaque) {
> + erofs_dbg("file %s: %s (%d) exists",
> + vi->i_srcpath, inode->i_srcpath,
> + erofs_mode_to_ftype(inode->i_mode));
> + return 0;
> + }
> + erofs_dbg("dir %s: %s merging", vi->i_srcpath, inode->i_srcpath);
> + } else {
> + erofs_dbg("loading file: %s (%d) (nid %llu)", vi->i_srcpath,
> + erofs_mode_to_ftype(vi->i_mode), vi->nid);
> + if (S_ISREG(vi->i_mode) && vi->i_nlink > 1 &&
> + (inode = erofs_iget(rctx->dev, vi->nid))) {
> + /* hardlink file */
> + if (S_ISDIR(inode->i_mode)) {
why this can happen?
> + erofs_dbg("hardlink directory not supported");
> + ret = -EISDIR;
> + goto put_inode;
> + }
> + inode->i_nlink++;
> + erofs_dbg("\thardlink: %s -> %s",
> + vi->i_srcpath, inode->i_srcpath);
> + } else {
> + inode = erofs_new_inode();
> + if (IS_ERR(inode))
> + return PTR_ERR(inode);
> +
> + ret = erofs_read_xattrs_from_disk(vi);
> + if (ret)
> + goto put_inode;
> +
> + inode->i_parent = d->inode;
> + inode->dev = rctx->dev;
> + ret = erofs_rebuild_fill_inode(inode, vi);
> + if (ret)
> + goto put_inode;
> +
> + erofs_insert_ihash(inode, rctx->dev, vi->nid);
> + }
> +
> + d->inode = inode;
> + d->type = erofs_mode_to_ftype(inode->i_mode);
> + }
> +
> + ret = 0;
> + if (S_ISDIR(vi->i_mode)) {
> + struct erofs_rebuild_dir_context nctx = *rctx;
> + nctx.ctx.dir = vi;
> + ret = erofs_iterate_dir(&nctx.ctx, false);
> + }
> + return ret;
> +
> +put_inode:
> + erofs_iput(inode);
> + return ret;
> +}
> +
> +static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx)
> +{
> + struct erofs_rebuild_dir_context *rctx = (void *)ctx;
> + struct erofs_inode *dir = ctx->dir;
> + struct erofs_inode vi = {};
why we cannot reuse the new allocated inode -- erofs_new_inode()?
> + char *path;
> + int ret;
> +
> + if (ctx->dot_dotdot)
> + return 0;
> +
> + erofs_dbg("file %s/%.*s", dir->i_srcpath, ctx->de_namelen, ctx->dname);
> + vi.nid = ctx->de_nid;
> + vi.sbi = dir->sbi;
> + ret = erofs_read_inode_from_disk(&vi);
> + if (ret)
> + return ret;
> +
> + ret = asprintf(&path, "%s/%.*s", dir->i_srcpath, ctx->de_namelen,
> + ctx->dname);
> + if (ret < 0)
> + return ret;
> + vi.i_srcpath = path;
> +
> + ret = erofs_rebuild_parse_inode(rctx, &vi);
> + free(path);
> + return ret;
> +}
> +
> +int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi)
> +{
> + struct erofs_inode inode = {};
> + struct erofs_rebuild_dir_context ctx;
> + int ret;
> +
> + if (!sbi->devname) {
> + erofs_err("please open the device first");
erofs_err("failed to find a device for rebuilding");
> + return -EINVAL;
> + }
> +
> + ret = erofs_read_superblock(sbi);
> + if (ret) {
> + erofs_err("failed to read superblock of img %s", sbi->devname);
let's avoid using abbreviation "img"
"failed to read superblock of %s"
> + return ret;
> + }
> +
> + inode.nid = sbi->root_nid;
> + inode.sbi = sbi;
> + ret = erofs_read_inode_from_disk(&inode);
> + if (ret) {
> + erofs_err("failed to read root inode of img %s", sbi->devname);
"failed to read root inode of %s"
Thanks,
Gao Xiang
More information about the Linux-erofs
mailing list