[PATCH v1 1/2] erofs-utils: mkfs,oci: support tarindex mode with zinfo for OCI
ChengyuZhu6
hudson at cyzhu.com
Thu Oct 9 18:41:45 AEDT 2025
From: Chengyu Zhu <hudsonzhu at tencent.com>
Introduce OCI tarindex mode and optional zinfo generation.
e.g.:
$ mkfs.erofs --oci=i,platform=linux/amd64,layer=3 \
--gzinfo=golang.zinfo golang.erofs golang:1.22.8
Signed-off-by: Chengyu Zhu <hudsonzhu at tencent.com>
Signed-off-by: Gao Xiang <hsiangkao at linux.alibaba.com>
---
lib/liberofs_oci.h | 2 ++
lib/remotes/oci.c | 57 ++++++++++++++++++++++++++++++++---
mkfs/main.c | 74 +++++++++++++++++++++++++++++-----------------
3 files changed, 102 insertions(+), 31 deletions(-)
diff --git a/lib/liberofs_oci.h b/lib/liberofs_oci.h
index 71c8879..5298f18 100644
--- a/lib/liberofs_oci.h
+++ b/lib/liberofs_oci.h
@@ -35,6 +35,8 @@ struct ocierofs_config {
char *password;
char *blob_digest;
int layer_index;
+ char *tarindex_path;
+ char *zinfo_path;
};
struct ocierofs_layer_info {
diff --git a/lib/remotes/oci.c b/lib/remotes/oci.c
index b2f1f59..d5ae4d8 100644
--- a/lib/remotes/oci.c
+++ b/lib/remotes/oci.c
@@ -27,6 +27,7 @@
#include "liberofs_base64.h"
#include "liberofs_oci.h"
#include "liberofs_private.h"
+#include "liberofs_gzran.h"
#ifdef OCIEROFS_ENABLED
@@ -840,14 +841,33 @@ out:
return ret;
}
-static int ocierofs_process_tar_stream(struct erofs_importer *importer, int fd)
+static int ocierofs_process_tar_stream(struct erofs_importer *importer, int fd,
+ const struct ocierofs_config *config,
+ u64 *tar_offset_out)
{
struct erofs_tarfile tarfile = {};
- int ret;
+ int ret, decoder, zinfo_fd;
+ struct erofs_vfile vf;
init_list_head(&tarfile.global.xattrs);
- ret = erofs_iostream_open(&tarfile.ios, fd, EROFS_IOS_DECODER_GZIP);
+ /*
+ * Choose decoder based on config:
+ * - tarindex + zinfo -> tar.gzip (GZRAN decoder)
+ * - tarindex only -> tar (no decoder, raw)
+ * - neither -> default gzip decoder
+ */
+ if (config && config->tarindex_path) {
+ tarfile.index_mode = true;
+ if (config->zinfo_path)
+ decoder = EROFS_IOS_DECODER_GZRAN;
+ else
+ decoder = EROFS_IOS_DECODER_NONE;
+ } else {
+ decoder = EROFS_IOS_DECODER_GZIP;
+ }
+
+ ret = erofs_iostream_open(&tarfile.ios, fd, decoder);
if (ret) {
erofs_err("failed to initialize tar stream: %s",
erofs_strerror(ret));
@@ -858,6 +878,25 @@ static int ocierofs_process_tar_stream(struct erofs_importer *importer, int fd)
ret = tarerofs_parse_tar(importer, &tarfile);
/* Continue parsing until end of archive */
} while (!ret);
+
+ if (decoder == EROFS_IOS_DECODER_GZRAN) {
+ zinfo_fd = open(config->zinfo_path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
+ if (zinfo_fd < 0) {
+ ret = -errno;
+ } else {
+ vf = (struct erofs_vfile){ .fd = zinfo_fd };
+ ret = erofs_gzran_builder_export_zinfo(tarfile.ios.gb, &vf);
+ close(zinfo_fd);
+ if (ret < 0) {
+ erofs_err("failed to export zinfo: %s",
+ erofs_strerror(ret));
+ }
+ }
+ }
+
+ if (tar_offset_out)
+ *tar_offset_out = tarfile.offset;
+
erofs_iostream_close(&tarfile.ios);
if (ret < 0 && ret != -ENODATA) {
@@ -1230,6 +1269,7 @@ int ocierofs_build_trees(struct erofs_importer *importer,
{
struct ocierofs_ctx ctx = {};
int ret, i, end, fd;
+ u64 tar_offset = 0;
ret = ocierofs_init(&ctx, config);
if (ret) {
@@ -1250,6 +1290,12 @@ int ocierofs_build_trees(struct erofs_importer *importer,
end = ctx.layer_count;
}
+ if (config->tarindex_path && (end - i) != 1) {
+ erofs_err("tarindex mode requires exactly one layer (use blob= or layer= option)");
+ ret = -EINVAL;
+ goto out;
+ }
+
while (i < end) {
char *trimmed = erofs_trim_for_progressinfo(ctx.layers[i]->digest,
sizeof("Extracting layer ...") - 1);
@@ -1263,7 +1309,7 @@ int ocierofs_build_trees(struct erofs_importer *importer,
ret = fd;
break;
}
- ret = ocierofs_process_tar_stream(importer, fd);
+ ret = ocierofs_process_tar_stream(importer, fd, config, &tar_offset);
close(fd);
if (ret) {
erofs_err("failed to process tar stream for layer %s: %s",
@@ -1273,6 +1319,9 @@ int ocierofs_build_trees(struct erofs_importer *importer,
i++;
}
out:
+ if (config->tarindex_path && importer->sbi)
+ importer->sbi->devs[0].blocks = BLK_ROUND_UP(importer->sbi, tar_offset);
+
ocierofs_ctx_cleanup(&ctx);
return ret;
}
diff --git a/mkfs/main.c b/mkfs/main.c
index 1c37576..5657b1d 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -213,7 +213,8 @@ static void usage(int argc, char **argv)
" [,sig=<2,4>] S3 API signature version (default: 2)\n"
#endif
#ifdef OCIEROFS_ENABLED
- " --oci[=platform=X] X=platform (default: linux/amd64)\n"
+ " --oci=[f|i] generate a full (f) or index-only (i) image from OCI remote source\n"
+ " [,=platform=X] X=platform (default: linux/amd64)\n"
" [,layer=#] #=layer index to extract (0-based; omit to extract all layers)\n"
" [,blob=Y] Y=blob digest to extract (omit to extract all layers)\n"
" [,username=Z] Z=username for authentication (optional)\n"
@@ -285,8 +286,8 @@ static struct erofs_s3 s3cfg;
#ifdef OCIEROFS_ENABLED
static struct ocierofs_config ocicfg;
-static char *mkfs_oci_options;
#endif
+static bool mkfs_oci_tarindex_mode;
enum {
EROFS_MKFS_DATA_IMPORT_DEFAULT,
@@ -727,7 +728,9 @@ static int mkfs_parse_s3_cfg(char *cfg_str)
* @options_str: comma-separated options string
*
* Parse OCI options string containing comma-separated key=value pairs.
- * Supported options include platform, blob, layer, username, and password.
+ *
+ * Supported options include f|i, platform, blob|layer, username, password,
+ * and zinfo.
*
* Return: 0 on success, negative errno on failure
*/
@@ -740,11 +743,22 @@ static int mkfs_parse_oci_options(struct ocierofs_config *oci_cfg, char *options
return 0;
opt = options_str;
+ q = strchr(opt, ',');
+ if (q)
+ *q = '\0';
+ if (!strcmp(opt, "i") || !strcmp(opt, "f")) {
+ mkfs_oci_tarindex_mode = (*opt == 'i');
+ opt = q ? q + 1 : NULL;
+ } else if (q) {
+ *q = ',';
+ }
+
while (opt) {
q = strchr(opt, ',');
if (q)
*q = '\0';
+
p = strstr(opt, "platform=");
if (p) {
p += strlen("platform=");
@@ -790,19 +804,19 @@ static int mkfs_parse_oci_options(struct ocierofs_config *oci_cfg, char *options
oci_cfg->username = strdup(p);
if (!oci_cfg->username)
return -ENOMEM;
+ } else {
+ p = strstr(opt, "password=");
+ if (p) {
+ p += strlen("password=");
+ free(oci_cfg->password);
+ oci_cfg->password = strdup(p);
+ if (!oci_cfg->password)
+ return -ENOMEM;
+ } else {
+ erofs_err("mkfs: invalid --oci value %s", opt);
+ return -EINVAL;
+ }
}
-
- p = strstr(opt, "password=");
- if (p) {
- p += strlen("password=");
- free(oci_cfg->password);
- oci_cfg->password = strdup(p);
- if (!oci_cfg->password)
- return -ENOMEM;
- }
-
- erofs_err("mkfs: invalid --oci value %s", opt);
- return -EINVAL;
}
}
}
@@ -1378,10 +1392,13 @@ static int mkfs_parse_options_cfg(struct erofs_importer_params *params,
break;
#endif
#ifdef OCIEROFS_ENABLED
- case 534:
- mkfs_oci_options = optarg;
+ case 534: {
source_mode = EROFS_MKFS_SOURCE_OCI;
+ err = mkfs_parse_oci_options(&ocicfg, optarg);
+ if (err)
+ return err;
break;
+ }
#endif
case 535:
if (optarg)
@@ -1757,6 +1774,9 @@ int main(int argc, char **argv)
goto exit;
}
mkfs_blkszbits = src->blkszbits;
+ } else if (mkfs_oci_tarindex_mode) {
+ mkfs_blkszbits = 9;
+ tar_index_512b = true;
}
if (!incremental_mode)
@@ -1883,13 +1903,11 @@ int main(int argc, char **argv)
#endif
#ifdef OCIEROFS_ENABLED
} else if (source_mode == EROFS_MKFS_SOURCE_OCI) {
- ocicfg.blob_digest = NULL;
- ocicfg.layer_index = -1;
-
- err = mkfs_parse_oci_options(&ocicfg, mkfs_oci_options);
- if (err)
- goto exit;
ocicfg.image_ref = cfg.c_src_path;
+ if (mkfs_oci_tarindex_mode)
+ ocicfg.tarindex_path = strdup(cfg.c_src_path);
+ if (!ocicfg.zinfo_path)
+ ocicfg.zinfo_path = mkfs_aws_zinfo_file;
if (incremental_mode ||
dataimport_mode == EROFS_MKFS_DATA_IMPORT_RVSP ||
@@ -1914,10 +1932,12 @@ int main(int argc, char **argv)
if (!g_sbi.extra_devices) {
DBG_BUGON(1);
} else {
- if (cfg.c_src_path)
- g_sbi.devs[0].src_path = strdup(cfg.c_src_path);
- g_sbi.devs[0].blocks =
- BLK_ROUND_UP(&g_sbi, erofstar.offset);
+ if (source_mode != EROFS_MKFS_SOURCE_OCI) {
+ if (cfg.c_src_path)
+ g_sbi.devs[0].src_path = strdup(cfg.c_src_path);
+ g_sbi.devs[0].blocks =
+ BLK_ROUND_UP(&g_sbi, erofstar.offset);
+ }
}
}
--
2.51.0
More information about the Linux-erofs
mailing list