[PATCH v2 08/10] erofs-utils: enable incremental builds

Gao Xiang hsiangkao at linux.alibaba.com
Tue Jun 18 18:24:13 AEST 2024


`--incremental=<data|rvsp>` are now supported for tarerofs but
only `--incremental=rvsp` works for the rebuild mode.

For example:
$ mkfs.erofs --tar=f --gzip --aufs --clean=data foo.erofs f0.tgz
$ mkfs.erofs --tar=f --gzip --aufs --incremental=data foo.erofs f1.tgz
...
$ mkfs.erofs --tar=f --gzip --aufs --incremental=data foo.erofs fn-1.tgz

Signed-off-by: Gao Xiang <hsiangkao at linux.alibaba.com>
---
 include/erofs/io.h  |   2 +
 include/erofs/tar.h |   3 +-
 lib/io.c            |  13 +++++
 lib/tar.c           |  15 ++++-
 mkfs/main.c         | 138 ++++++++++++++++++++++++++++++++++----------
 5 files changed, 137 insertions(+), 34 deletions(-)

diff --git a/include/erofs/io.h b/include/erofs/io.h
index f24a563..d3a487f 100644
--- a/include/erofs/io.h
+++ b/include/erofs/io.h
@@ -32,6 +32,7 @@ struct erofs_vfops {
 	int (*ftruncate)(struct erofs_vfile *vf, u64 length);
 	ssize_t (*read)(struct erofs_vfile *vf, void *buf, size_t len);
 	off_t (*lseek)(struct erofs_vfile *vf, u64 offset, int whence);
+	int (*fstat)(struct erofs_vfile *vf, struct stat *buf);
 };
 
 struct erofs_vfile {
@@ -40,6 +41,7 @@ struct erofs_vfile {
 	int fd;
 };
 
+int erofs_io_fstat(struct erofs_vfile *vf, struct stat *buf);
 ssize_t erofs_io_pwrite(struct erofs_vfile *vf, const void *buf, u64 pos, size_t len);
 int erofs_io_fsync(struct erofs_vfile *vf);
 ssize_t erofs_io_fallocate(struct erofs_vfile *vf, u64 offset, size_t len, bool pad);
diff --git a/include/erofs/tar.h b/include/erofs/tar.h
index e1de0df..403f3a8 100644
--- a/include/erofs/tar.h
+++ b/include/erofs/tar.h
@@ -57,9 +57,10 @@ struct erofs_tarfile {
 	struct erofs_iostream ios;
 	char *mapfile, *dumpfile;
 
+	u32 dev;
 	int fd;
 	u64 offset;
-	bool index_mode, headeronly_mode, aufs;
+	bool index_mode, headeronly_mode, rvsp_mode, aufs;
 };
 
 void erofs_iostream_close(struct erofs_iostream *ios);
diff --git a/lib/io.c b/lib/io.c
index c523f00..83c145c 100644
--- a/lib/io.c
+++ b/lib/io.c
@@ -26,6 +26,19 @@
 #define EROFS_MODNAME	"erofs_io"
 #include "erofs/print.h"
 
+int erofs_io_fstat(struct erofs_vfile *vf, struct stat *buf)
+{
+	if (unlikely(cfg.c_dry_run)) {
+		buf->st_size = 0;
+		buf->st_mode = S_IFREG | 0777;
+		return 0;
+	}
+
+	if (vf->ops)
+		return vf->ops->fstat(vf, buf);
+	return fstat(vf->fd, buf);
+}
+
 ssize_t erofs_io_pwrite(struct erofs_vfile *vf, const void *buf,
 			u64 pos, size_t len)
 {
diff --git a/lib/tar.c b/lib/tar.c
index 53a1188..8d2caa5 100644
--- a/lib/tar.c
+++ b/lib/tar.c
@@ -942,6 +942,7 @@ new_inode:
 			ret = PTR_ERR(inode);
 			goto out;
 		}
+		inode->dev = tar->dev;
 		inode->i_parent = d->inode;
 		d->inode = inode;
 		d->type = erofs_mode_to_ftype(st.st_mode);
@@ -981,13 +982,21 @@ new_inode:
 			inode->i_link = malloc(inode->i_size + 1);
 			memcpy(inode->i_link, eh.link, inode->i_size + 1);
 		} else if (inode->i_size) {
-			if (tar->headeronly_mode)
+			if (tar->headeronly_mode) {
 				ret = erofs_write_zero_inode(inode);
-			else if (tar->index_mode)
+			} else if (tar->rvsp_mode) {
+				inode->datasource = EROFS_INODE_DATA_SOURCE_RESVSP;
+				inode->i_ino[1] = data_offset;
+				if (erofs_iostream_lskip(&tar->ios, inode->i_size))
+					ret = -EIO;
+				else
+					ret = 0;
+			} else if (tar->index_mode) {
 				ret = tarerofs_write_file_index(inode, tar,
 								data_offset);
-			else
+			} else {
 				ret = tarerofs_write_file_data(inode, tar);
+			}
 			if (ret)
 				goto out;
 		}
diff --git a/mkfs/main.c b/mkfs/main.c
index 425fe00..6e9120f 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -79,6 +79,8 @@ static struct option long_options[] = {
 	{"workers", required_argument, NULL, 520},
 #endif
 	{"zfeature-bits", required_argument, NULL, 521},
+	{"clean", optional_argument, NULL, 522},
+	{"incremental", optional_argument, NULL, 523},
 	{0, 0, 0, 0},
 };
 
@@ -152,6 +154,10 @@ static void usage(int argc, char **argv)
 		" --all-root            make all files owned by root\n"
 		" --blobdev=X           specify an extra device X to store chunked data\n"
 		" --chunksize=#         generate chunk-based files with #-byte chunks\n"
+		" --clean=X             run full clean build (default) or:\n"
+		" --incremental=X       run incremental build\n"
+		"                       (X = data|rvsp; data=complete data, rvsp=space is allocated\n"
+		"                                       and filled with zeroes)\n"
 		" --compress-hints=X    specify a file to configure per-file compression strategy\n"
 		" --exclude-path=X      avoid including file X (X = exact literal path)\n"
 		" --exclude-regex=X     avoid including files that match X (X = regular expression)\n"
@@ -214,7 +220,14 @@ static unsigned int pclustersize_packed, pclustersize_max;
 static struct erofs_tarfile erofstar = {
 	.global.xattrs = LIST_HEAD_INIT(erofstar.global.xattrs)
 };
-static bool tar_mode, rebuild_mode;
+static bool tar_mode, rebuild_mode, incremental_mode;
+
+enum {
+	EROFS_MKFS_DATA_IMPORT_DEFAULT,
+	EROFS_MKFS_DATA_IMPORT_FULLDATA,
+	EROFS_MKFS_DATA_IMPORT_RVSP,
+	EROFS_MKFS_DATA_IMPORT_SPARSE,
+} dataimport_mode;
 
 static unsigned int rebuild_src_count;
 static LIST_HEAD(rebuild_src_list);
@@ -775,6 +788,22 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 			if (err)
 				return err;
 			break;
+		case 522:
+		case 523:
+			if (!optarg || !strcmp(optarg, "data")) {
+				dataimport_mode = EROFS_MKFS_DATA_IMPORT_FULLDATA;
+			} else if (!strcmp(optarg, "rvsp")) {
+				dataimport_mode = EROFS_MKFS_DATA_IMPORT_RVSP;
+			} else {
+				dataimport_mode = strtol(optarg, &endptr, 0);
+				if (errno || *endptr != '\0') {
+					erofs_err("invalid --%s=%s",
+						  opt == 523 ? "incremental" : "clean", optarg);
+					return -EINVAL;
+				}
+			}
+			incremental_mode = (opt == 523);
+			break;
 		case 'V':
 			version();
 			exit(0);
@@ -998,9 +1027,6 @@ static void erofs_mkfs_default_options(void)
 	sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_ZERO_PADDING;
 	sbi.feature_compat = EROFS_FEATURE_COMPAT_SB_CHKSUM |
 			     EROFS_FEATURE_COMPAT_MTIME;
-
-	/* generate a default uuid first */
-	erofs_uuid_generate(sbi.uuid);
 }
 
 /* https://reproducible-builds.org/specs/source-date-epoch/ for more details */
@@ -1052,16 +1078,30 @@ static struct erofs_inode *erofs_mkfs_alloc_root(struct erofs_sb_info *sbi)
 	return root;
 }
 
-static int erofs_rebuild_load_trees(struct erofs_inode *root)
+static int erofs_mkfs_rebuild_load_trees(struct erofs_inode *root)
 {
 	struct erofs_sb_info *src;
 	unsigned int extra_devices = 0;
 	erofs_blk_t nblocks;
 	int ret, idx;
+	enum erofs_rebuild_datamode datamode;
+
+	switch (dataimport_mode) {
+	case EROFS_MKFS_DATA_IMPORT_DEFAULT:
+		datamode = EROFS_REBUILD_DATA_BLOB_INDEX;
+		break;
+	case EROFS_MKFS_DATA_IMPORT_FULLDATA:
+		datamode = EROFS_REBUILD_DATA_FULL;
+		break;
+	case EROFS_MKFS_DATA_IMPORT_RVSP:
+		datamode = EROFS_REBUILD_DATA_RESVSP;
+		break;
+	default:
+		return -EINVAL;
+	}
 
 	list_for_each_entry(src, &rebuild_src_list, list) {
-		ret = erofs_rebuild_load_tree(root, src,
-					      EROFS_REBUILD_DATA_BLOB_INDEX);
+		ret = erofs_rebuild_load_tree(root, src, datamode);
 		if (ret) {
 			erofs_err("failed to load %s", src->devname);
 			return ret;
@@ -1074,7 +1114,10 @@ static int erofs_rebuild_load_trees(struct erofs_inode *root)
 		extra_devices += src->extra_devices;
 	}
 
-	if (extra_devices && extra_devices != rebuild_src_count) {
+	if (datamode != EROFS_REBUILD_DATA_BLOB_INDEX)
+		return 0;
+
+	if (extra_devices != rebuild_src_count) {
 		erofs_err("extra_devices(%u) is mismatched with source images(%u)",
 			  extra_devices, rebuild_src_count);
 		return -EOPNOTSUPP;
@@ -1117,6 +1160,7 @@ static int erofs_rebuild_load_trees(struct erofs_inode *root)
 static void erofs_mkfs_showsummaries(erofs_blk_t nblocks)
 {
 	char uuid_str[37] = {};
+	char *incr = incremental_mode ? "new" : "total";
 
 	if (!(cfg.c_dbg_lvl > EROFS_ERR && cfg.c_showprogress))
 		return;
@@ -1126,11 +1170,11 @@ static void erofs_mkfs_showsummaries(erofs_blk_t nblocks)
 	fprintf(stdout, "------\nFilesystem UUID: %s\n"
 		"Filesystem total blocks: %u (of %u-byte blocks)\n"
 		"Filesystem total inodes: %llu\n"
-		"Filesystem total metadata blocks: %u\n"
-		"Filesystem total deduplicated bytes (of source files): %llu\n",
+		"Filesystem %s metadata blocks: %u\n"
+		"Filesystem %s deduplicated bytes (of source files): %llu\n",
 		uuid_str, nblocks, 1U << sbi.blkszbits, sbi.inos | 0ULL,
-		erofs_total_metablocks(),
-		sbi.saved_by_deduplication | 0ULL);
+		incr, erofs_total_metablocks(),
+		incr, sbi.saved_by_deduplication | 0ULL);
 }
 
 int main(int argc, char **argv)
@@ -1167,20 +1211,13 @@ int main(int argc, char **argv)
 		sbi.build_time_nsec = t.tv_usec;
 	}
 
-	err = erofs_dev_open(&sbi, cfg.c_img_path, O_RDWR | O_TRUNC);
+	err = erofs_dev_open(&sbi, cfg.c_img_path, O_RDWR |
+				(incremental_mode ? 0 : O_TRUNC));
 	if (err) {
 		fprintf(stderr, "Try '%s --help' for more information.\n", argv[0]);
 		return 1;
 	}
 
-	if (tar_mode && !erofstar.index_mode) {
-		err = erofs_diskbuf_init(1);
-		if (err) {
-			erofs_err("failed to initialize diskbuf: %s",
-				   strerror(-err));
-			goto exit;
-		}
-	}
 #ifdef WITH_ANDROID
 	if (cfg.fs_config_file &&
 	    load_canned_fs_config(cfg.fs_config_file) < 0) {
@@ -1218,14 +1255,22 @@ int main(int argc, char **argv)
 	if (cfg.c_random_pclusterblks)
 		srand(time(NULL));
 #endif
-	if (tar_mode && erofstar.index_mode) {
+	if (tar_mode) {
+		if (dataimport_mode == EROFS_MKFS_DATA_IMPORT_RVSP)
+			erofstar.rvsp_mode = true;
+		erofstar.dev = rebuild_src_count + 1;
+
 		if (erofstar.mapfile) {
 			err = erofs_blocklist_open(erofstar.mapfile, true);
 			if (err) {
 				erofs_err("failed to open %s", erofstar.mapfile);
 				goto exit;
 			}
-		} else {
+		} else if (erofstar.index_mode) {
+			/*
+			 * If mapfile is unspecified for tarfs index mode,
+			 * 512-byte block size is enforced here.
+			 */
 			sbi.blkszbits = 9;
 		}
 	}
@@ -1246,10 +1291,43 @@ int main(int argc, char **argv)
 		sbi.blkszbits = src->blkszbits;
 	}
 
-	sb_bh = erofs_reserve_sb();
-	if (IS_ERR(sb_bh)) {
-		err = PTR_ERR(sb_bh);
-		goto exit;
+	if (!incremental_mode) {
+		sb_bh = erofs_reserve_sb();
+		if (IS_ERR(sb_bh)) {
+			err = PTR_ERR(sb_bh);
+			goto exit;
+		}
+		/* generate new UUIDs for clean builds */
+		erofs_uuid_generate(sbi.uuid);
+	} else {
+		union {
+			struct stat st;
+			erofs_blk_t startblk;
+		} u;
+
+		erofs_warn("EXPERIMENTAL incremental build in use. Use at your own risk!");
+		err = erofs_read_superblock(&sbi);
+		if (err) {
+			erofs_err("failed to read superblock of %s", sbi.devname);
+			goto exit;
+		}
+
+		err = erofs_io_fstat(&sbi.bdev, &u.st);
+		if (!err && S_ISREG(u.st.st_mode))
+			u.startblk = DIV_ROUND_UP(u.st.st_size, erofs_blksiz(&sbi));
+		else
+			u.startblk = sbi.primarydevice_blocks;
+		erofs_buffer_init(u.startblk);
+		sb_bh = NULL;
+	}
+
+	if (tar_mode && !erofstar.index_mode) {
+		err = erofs_diskbuf_init(1);
+		if (err) {
+			erofs_err("failed to initialize diskbuf: %s",
+				   strerror(-err));
+			goto exit;
+		}
 	}
 
 	err = erofs_load_compress_hints(&sbi);
@@ -1310,7 +1388,7 @@ int main(int argc, char **argv)
 		if (err < 0)
 			goto exit;
 
-		err = erofs_rebuild_dump_tree(root, false);
+		err = erofs_rebuild_dump_tree(root, incremental_mode);
 		if (err < 0)
 			goto exit;
 	} else if (rebuild_mode) {
@@ -1320,10 +1398,10 @@ int main(int argc, char **argv)
 			goto exit;
 		}
 
-		err = erofs_rebuild_load_trees(root);
+		err = erofs_mkfs_rebuild_load_trees(root);
 		if (err)
 			goto exit;
-		err = erofs_rebuild_dump_tree(root, false);
+		err = erofs_rebuild_dump_tree(root, incremental_mode);
 		if (err)
 			goto exit;
 	} else {
-- 
2.39.3



More information about the Linux-erofs mailing list