[PATCH v6-rebased] erofs-utils: add OCI registry support
Gao Xiang
hsiangkao at linux.alibaba.com
Tue Aug 26 12:55:37 AEST 2025
Hi Chengyu,
On 2025/8/25 17:33, ChengyuZhu6 wrote:
> From: Chengyu Zhu <hudsonzhu at tencent.com>
>
> This patch adds support for building EROFS filesystems from
> OCI-compliant container registries, enabling users to create EROFS
> images directly from container images stored in registries like
> Docker Hub, Quay.io, etc.
>
> The implementation includes:
> - OCI remote backend with registry authentication support
> - Manifest parsing for Docker v2 and OCI v1 formats
> - Layer extraction and tar processing integration
> - Multi-platform image selection capability
> - Both anonymous and authenticated registry access
> - Comprehensive build system integration
>
> Configure: ./configure --enable-oci
> New mkfs.erofs option: --oci-options=[option] target source-image
>
> Supported options:
> - platform=os/arch (default: linux/amd64)
> - layer=N (extract specific layer, default: all layers)
> - username/password (basic authentication)
>
> e.g.:
> - mkfs.erofs \
> --oci-options=platform=linux/amd64, \
> username=1234,password=1234 \
> image.erofs ubuntu
>
> Signed-off-by: Changzhi Xie <sa.z at qq.com>
> Signed-off-by: Chengyu Zhu <hudsonzhu at tencent.com>
Can you apply the following diff to your v6-rebased first?
diff --git a/configure.ac b/configure.ac
index 85d055f..7db4489 100644
--- a/configure.ac
+++ b/configure.ac
@@ -651,7 +651,9 @@ AS_IF([test "x$with_json_c" != "xno"], [
LIBS="${saved_LIBS}"
CPPFLAGS="${saved_CPPFLAGS}"
], [
- AC_MSG_ERROR([Cannot find proper json-c])
+ AS_IF([test "x$with_json_c" = "xyes"], [
+ AC_MSG_ERROR([Cannot find proper json-c])
+ ])
])
])
diff --git a/lib/liberofs_oci.h b/lib/liberofs_oci.h
index c4a4c78..0cde24e 100644
--- a/lib/liberofs_oci.h
+++ b/lib/liberofs_oci.h
@@ -76,12 +76,10 @@ int erofs_oci_params_set_string(char **field, const char *value);
* ocierofs_build_trees - Build file trees from OCI container image layers
* @root: root inode to build the file tree under
* @oci: OCI client structure with configured parameters
- * @fillzero: if true, only create inodes without downloading actual data
*
* Return: 0 on success, negative errno on failure
*/
-int ocierofs_build_trees(struct erofs_importer *importer, struct erofs_oci *oci,
- bool fillzero);
+int ocierofs_build_trees(struct erofs_importer *importer, struct erofs_oci *oci);
#ifdef __cplusplus
}
diff --git a/lib/remotes/oci.c b/lib/remotes/oci.c
index 7f167e1..1a217d5 100644
--- a/lib/remotes/oci.c
+++ b/lib/remotes/oci.c
@@ -102,7 +102,6 @@ static int ocierofs_curl_setup_common_options(struct CURL *curl)
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 120L);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "ocierofs/" PACKAGE_VERSION);
-
return 0;
}
@@ -821,8 +820,7 @@ out:
return ret;
}
-int ocierofs_build_trees(struct erofs_importer *importer, struct erofs_oci *oci,
- bool fillzero)
+int ocierofs_build_trees(struct erofs_importer *importer, struct erofs_oci *oci)
{
char *auth_header = NULL;
char *manifest_digest = NULL;
@@ -881,54 +879,34 @@ int ocierofs_build_trees(struct erofs_importer *importer, struct erofs_oci *oci,
}
if (oci->params.layer_index >= 0) {
- char *trimmed;
-
if (oci->params.layer_index >= layer_count) {
erofs_err("layer index %d exceeds available layers (%d)",
oci->params.layer_index, layer_count);
ret = -EINVAL;
goto out_layers;
}
-
+ layer_count = 1;
i = oci->params.layer_index;
- trimmed = erofs_trim_for_progressinfo(layers_info[i],
+ } else {
+ i = 0;
+ }
+
+ while (i < layer_count) {
+ char *trimmed = erofs_trim_for_progressinfo(layers_info[i],
sizeof("Extracting layer ...") - 1);
- erofs_update_progressinfo("Extracting layer %d: %s ...", i,
- trimmed);
+
+ erofs_update_progressinfo("Extracting layer %s ...", trimmed);
free(trimmed);
- if (!fillzero) {
- ret = ocierofs_extract_layer(oci, importer, layers_info[i],
- auth_header, i);
- if (ret) {
- erofs_err("failed to extract layer %d: %s", i,
- erofs_strerror(ret));
- goto out_layers;
- }
- }
- } else {
- for (i = 0; i < layer_count; i++) {
- char *trimmed = erofs_trim_for_progressinfo(layers_info[i],
- sizeof("Extracting layer ...") - 1);
- erofs_update_progressinfo("Extracting layer %s ...",
- trimmed);
- free(trimmed);
-
- if (fillzero)
- continue;
-
- ret = ocierofs_extract_layer(oci, importer, layers_info[i],
- auth_header, i);
- if (ret) {
- erofs_err("failed to extract layer %d: %s", i,
- erofs_strerror(ret));
- goto out_layers;
- }
+ ret = ocierofs_extract_layer(oci, importer, layers_info[i],
+ auth_header, i);
+ if (ret) {
+ erofs_err("failed to extract layer %d: %s", i,
+ erofs_strerror(ret));
+ break;
}
}
- ret = 0;
-
out_layers:
for (i = 0; i < layer_count; i++)
free(layers_info[i]);
@@ -943,7 +921,6 @@ out_auth:
!auth_header) {
ocierofs_curl_clear_auth(oci->curl);
}
-
out:
return ret;
}
@@ -955,14 +932,9 @@ int ocierofs_init(struct erofs_oci *oci)
memset(oci, 0, sizeof(*oci));
- if (curl_global_init(CURL_GLOBAL_DEFAULT) != CURLE_OK)
- return -EIO;
-
oci->curl = curl_easy_init();
- if (!oci->curl) {
- curl_global_cleanup();
+ if (!oci->curl)
return -EIO;
- }
if (ocierofs_curl_setup_common_options(oci->curl)) {
ocierofs_cleanup(oci);
@@ -977,9 +949,7 @@ int ocierofs_init(struct erofs_oci *oci)
ocierofs_cleanup(oci);
return -ENOMEM;
}
-
oci->params.layer_index = -1; /* -1 means extract all layers */
-
return 0;
}
@@ -992,8 +962,6 @@ void ocierofs_cleanup(struct erofs_oci *oci)
curl_easy_cleanup(oci->curl);
oci->curl = NULL;
}
- curl_global_cleanup();
-
free(oci->params.registry);
free(oci->params.repository);
free(oci->params.tag);
diff --git a/mkfs/main.c b/mkfs/main.c
index 897b4ba..573adc3 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -2000,11 +2000,11 @@ int main(int argc, char **argv)
goto exit;
if (incremental_mode ||
- dataimport_mode == EROFS_MKFS_DATA_IMPORT_RVSP)
+ dataimport_mode == EROFS_MKFS_DATA_IMPORT_RVSP ||
+ dataimport_mode == EROFS_MKFS_DATA_IMPORT_ZEROFILL)
err = -EOPNOTSUPP;
else
- err = ocierofs_build_trees(&importer, &ocicfg,
- dataimport_mode == EROFS_MKFS_DATA_IMPORT_ZEROFILL);
+ err = ocierofs_build_trees(&importer, &ocicfg);
if (err)
goto exit;
#endif
--
2.43.5
Also I have some other comments when trying:
> ---
...
> diff --git a/mkfs/main.c b/mkfs/main.c
> index e0ba55d..403cd6a 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -32,6 +32,7 @@
> #include "../lib/liberofs_uuid.h"
> #include "../lib/liberofs_metabox.h"
> #include "../lib/liberofs_s3.h"
> +#include "../lib/liberofs_oci.h"
> #include "../lib/compressor.h"
>
> static struct option long_options[] = {
> @@ -95,6 +96,9 @@ static struct option long_options[] = {
> {"vmdk-desc", required_argument, NULL, 532},
> #ifdef S3EROFS_ENABLED
> {"s3", required_argument, NULL, 533},
> +#endif
> +#ifdef OCIEROFS_ENABLED
> + {"oci-options", required_argument, NULL, 534},
Can we just use `--oci` instead? and optional_argument.
> #endif
> {0, 0, 0, 0},
> };
> @@ -206,6 +210,14 @@ static void usage(int argc, char **argv)
> " [,passwd_file=Y] X=endpoint, Y=s3fs-compatible password file\n"
> " [,urlstyle=Z] S3 API calling style (Z = vhost|path) (default: vhost)\n"
> " [,sig=<2,4>] S3 API signature version (default: 2)\n"
> +#endif
> +#ifdef OCIEROFS_ENABLED
> + " --oci-options=X specify OCI options\n"
" --oci[=platform=X] X=platform (default: linux/amd64)\n"
" [,layer=Y] Y=layer index to extract (0-based; omit to extract all layers)\n"
" [,username=Z] Z="
....
> + " X=platform=Y,layer=Z,username=U,password=P\n"
> +
> +static int parse_oci_password_option(const char *opt, char *options_copy)
> +{
> + const char *p;
> +
> + p = strstr(opt, "password=");
> + if (p) {
> + if (erofs_oci_params_set_string(&ocicfg.params.password,
> + p + strlen("password="))) {
> + erofs_err("failed to set password");
> + free(options_copy);
> + return -ENOMEM;
> + }
> + return 0;
> + }
> + return -EINVAL;
> +}
> +
> +static int mkfs_parse_oci_options(const char *options_str)
> +{
> + char *opt, *q, *options_copy;
> + int ret;
> +
> + if (!options_str)
> + return 0;
> +
> + options_copy = strdup(options_str);
> + if (!options_copy)
> + return -ENOMEM;
Could we just avoid options_copy here? see:
mkfs_parse_s3_cfg() as an example.
> +
> + opt = options_copy;
> + while (opt) {
> + q = strchr(opt, ',');
> + if (q)
> + *q = '\0';
> +
> + ret = parse_oci_platform_option(opt, options_copy);
> + if (ret == 0) {
> + opt = q ? q + 1 : NULL;
> + continue;
> + }
What if ret == -ENOMEM here? it will just step down
to the rest, how about just follow mkfs_parse_s3_cfg() style?
Also I have a small question:
for example, I can pull image with:
$ nerdctl pull docker.xuanyuan.me/library/ubuntu:22.04
but with mkfs.erofs:
$mkfs/mkfs.erofs --oci-options=platform=linux/amd64 image.erofs docker.xuanyuan.me/library/ubuntu:22.04 -d9
mkfs.erofs 1.8.10-g15844856-dirty
<I> erofs_io: erofs_dev_open() Line[357] successfully to open image.erofs
c_version: [1.8.10-g15844856-dirty]
c_dbg_lvl: [ 9]
c_dry_run: [ 0]
<D> erofs: ocierofs_get_auth_token_with_url() Line[313] Requesting auth token from: https://docker.xuanyuan.me/token/auth.docker.io?service=docker.xuanyuan.me&scope=repository:library/ubuntu:pull
<E> erofs: ocierofs_build_trees() Line[866] failed to get manifest digest: [Error 5] Input/output error
<E> erofs: main() Line[2089] Could not format the device : [Error 5] Input/output error
It just fails, it seems it still has some issue with anonymous
access?
Thanks,
Gao Xiang
More information about the Linux-erofs
mailing list