[PATCH v7 3/4] erofs-utils: mkfs: introduce --s3=... option

Hongbo Li lihongbo22 at huawei.com
Fri Aug 8 16:17:41 AEST 2025



On 2025/8/8 11:15, Gao Xiang wrote:
> From: Yifan Zhao <zhaoyifan28 at huawei.com>
> 
> It introduces configuration options for the upcoming experimental S3
> support, including configuration parsing and `passwd_file` reading
> logic.
> 
> Users can specify the following options:
>   - S3 service endpoint (required);
>   - S3 credentials file in the format $ak:%sk (optional);
>   - S3 API calling style (optional);
>   - S3 API signature version (optional, only V2 is currently supported).
> 
> Signed-off-by: Yifan Zhao <zhaoyifan28 at huawei.com>
> Signed-off-by: Gao Xiang <hsiangkao at linux.alibaba.com>

Reviewed-by: Hongbo Li <lihongbo22 at huawei.com>

Thanks,
Hongbo

> ---
>   lib/liberofs_s3.h |  40 ++++++++
>   mkfs/main.c       | 226 +++++++++++++++++++++++++++++++++++++++-------
>   2 files changed, 234 insertions(+), 32 deletions(-)
>   create mode 100644 lib/liberofs_s3.h
> 
> diff --git a/lib/liberofs_s3.h b/lib/liberofs_s3.h
> new file mode 100644
> index 0000000..4d3555e
> --- /dev/null
> +++ b/lib/liberofs_s3.h
> @@ -0,0 +1,40 @@
> +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
> +/*
> + * Copyright (C) 2025 HUAWEI, Inc.
> + *             http://www.huawei.com/
> + * Created by Yifan Zhao <zhaoyifan28 at huawei.com>
> + */
> +#ifndef __EROFS_S3_H
> +#define __EROFS_S3_H
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +enum s3erofs_url_style {
> +	S3EROFS_URL_STYLE_PATH,          // Path style: https://s3.amazonaws.com/bucket/object
> +	S3EROFS_URL_STYLE_VIRTUAL_HOST,  // Virtual host style: https://bucket.s3.amazonaws.com/object
> +};
> +
> +enum s3erofs_signature_version {
> +	S3EROFS_SIGNATURE_VERSION_2,
> +	S3EROFS_SIGNATURE_VERSION_4,
> +};
> +
> +#define S3_ACCESS_KEY_LEN 256
> +#define S3_SECRET_KEY_LEN 256
> +
> +struct erofs_s3 {
> +	const char *endpoint;
> +	char access_key[S3_ACCESS_KEY_LEN + 1];
> +	char secret_key[S3_SECRET_KEY_LEN + 1];
> +
> +	enum s3erofs_url_style url_style;
> +	enum s3erofs_signature_version sig;
> +};
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif
> diff --git a/mkfs/main.c b/mkfs/main.c
> index ab27b77..a7d3f31 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -31,6 +31,7 @@
>   #include "../lib/liberofs_private.h"
>   #include "../lib/liberofs_uuid.h"
>   #include "../lib/liberofs_metabox.h"
> +#include "../lib/liberofs_s3.h"
>   #include "../lib/compressor.h"
>   
>   static struct option long_options[] = {
> @@ -92,6 +93,9 @@ static struct option long_options[] = {
>   #endif
>   	{"fsalignblks", required_argument, NULL, 531},
>   	{"vmdk-desc", required_argument, NULL, 532},
> +#ifdef S3EROFS_ENABLED
> +	{"s3", required_argument, NULL, 533},
> +#endif
>   	{0, 0, 0, 0},
>   };
>   
> @@ -174,8 +178,8 @@ static void usage(int argc, char **argv)
>   		" --chunksize=#         generate chunk-based files with #-byte chunks\n"
>   		" --clean=X             run full clean build (default) or:\n"
>   		" --incremental=X       run incremental build\n"
> -		"                       (X = data|rvsp; data=full data, rvsp=space is allocated\n"
> -		"                                       and filled with zeroes)\n"
> +		"                       X = data|rvsp|0 (data: full data, rvsp: space fallocated\n"
> +		"                                        0: inodes zeroed)\n"
>   		" --compress-hints=X    specify a file to configure per-file compression strategy\n"
>   		" --dsunit=#            align all data block addresses to multiples of #\n"
>   		" --exclude-path=X      avoid including file X (X = exact literal path)\n"
> @@ -197,6 +201,12 @@ static void usage(int argc, char **argv)
>   		" --root-xattr-isize=#  ensure the inline xattr size of the root directory is # bytes at least\n"
>   		" --aufs                replace aufs special files with overlayfs metadata\n"
>   		" --sort=<path,none>    data sorting order for tarballs as input (default: path)\n"
> +#ifdef S3EROFS_ENABLED
> +		" --s3=X                generate an image from S3-compatible object store\n"
> +		"   [,passwd_file=Y]    X=endpoint, Y=s3fs-compatible password file\n"
> +		"   [,urlstyle=Z]       S3 API calling style (Z = vhost|path) (default: vhost)\n"
> +		"   [,sig=<2,4>]        S3 API signature version (default: 2)\n"
> +#endif
>   		" --tar=X               generate a full or index-only image from a tarball(-ish) source\n"
>   		"                       (X = f|i|headerball; f=full mode, i=index mode,\n"
>   		"                                            headerball=file data is omited in the source stream)\n"
> @@ -247,16 +257,23 @@ static struct erofs_tarfile erofstar = {
>   static bool incremental_mode;
>   static u8 metabox_algorithmid;
>   
> +#ifdef S3EROFS_ENABLED
> +static struct erofs_s3 s3cfg;
> +#endif
> +
>   enum {
>   	EROFS_MKFS_DATA_IMPORT_DEFAULT,
>   	EROFS_MKFS_DATA_IMPORT_FULLDATA,
>   	EROFS_MKFS_DATA_IMPORT_RVSP,
> -	EROFS_MKFS_DATA_IMPORT_SPARSE,
> +	EROFS_MKFS_DATA_IMPORT_ZEROFILL,
>   } dataimport_mode;
>   
>   static enum {
>   	EROFS_MKFS_SOURCE_LOCALDIR,
>   	EROFS_MKFS_SOURCE_TAR,
> +#ifdef S3EROFS_ENABLED
> +	EROFS_MKFS_SOURCE_S3,
> +#endif
>   	EROFS_MKFS_SOURCE_REBUILD,
>   } source_mode;
>   
> @@ -522,6 +539,137 @@ static void mkfs_parse_tar_cfg(char *cfg)
>   		erofstar.index_mode = true;
>   }
>   
> +#ifdef S3EROFS_ENABLED
> +static int mkfs_parse_s3_cfg_passwd(const char *filepath, char *ak, char *sk)
> +{
> +	struct stat st;
> +	int fd, n, ret;
> +	char buf[S3_ACCESS_KEY_LEN + S3_SECRET_KEY_LEN + 3];
> +	char *colon;
> +
> +	fd = open(filepath, O_RDONLY);
> +	if (fd < 0) {
> +		erofs_err("failed to open passwd_file %s", filepath);
> +		return -errno;
> +	}
> +
> +	ret = fstat(fd, &st);
> +	if (ret) {
> +		ret = -errno;
> +		goto err;
> +	}
> +
> +	if (!S_ISREG(st.st_mode)) {
> +		erofs_err("%s is not a regular file", filepath);
> +		ret = -EINVAL;
> +		goto err;
> +	}
> +
> +	if ((st.st_mode & 077) != 0)
> +		erofs_warn("passwd_file %s should not be accessible by group or others",
> +			   filepath);
> +
> +	if (st.st_size > S3_ACCESS_KEY_LEN + S3_SECRET_KEY_LEN + 3) {
> +		erofs_err("passwd_file %s is too large (size: %llu)", filepath,
> +			  st.st_size | 0ULL);
> +		ret = -EINVAL;
> +		goto err;
> +	}
> +
> +	n = read(fd, buf, st.st_size);
> +	if (n < 0) {
> +		ret = -errno;
> +		goto err;
> +	}
> +	buf[n] = '\0';
> +
> +	while (n > 0 && (buf[n - 1] == '\n' || buf[n - 1] == '\r'))
> +		buf[--n] = '\0';
> +
> +	colon = strchr(buf, ':');
> +	if (!colon) {
> +		ret = -EINVAL;
> +		goto err;
> +	}
> +	*colon = '\0';
> +
> +	strcpy(ak, buf);
> +	strcpy(sk, colon + 1);
> +
> +err:
> +	close(fd);
> +	return ret;
> +}
> +
> +static int mkfs_parse_s3_cfg(char *cfg_str)
> +{
> +	char *p, *q, *opt;
> +	int ret = 0;
> +
> +	if (source_mode != EROFS_MKFS_SOURCE_LOCALDIR)
> +		return -EINVAL;
> +	source_mode = EROFS_MKFS_SOURCE_S3;
> +
> +	if (!cfg_str) {
> +		erofs_err("s3: missing parameter");
> +		return -EINVAL;
> +	}
> +
> +	s3cfg.url_style = S3EROFS_URL_STYLE_VIRTUAL_HOST;
> +	s3cfg.sig = S3EROFS_SIGNATURE_VERSION_2;
> +
> +	p = strchr(cfg_str, ',');
> +	if (p) {
> +		s3cfg.endpoint = strndup(cfg_str, p - cfg_str);
> +	} else {
> +		s3cfg.endpoint = strdup(cfg_str);
> +		return 0;
> +	}
> +
> +	opt = p + 1;
> +	while (opt) {
> +		q = strchr(opt, ',');
> +		if (q)
> +			*q = '\0';
> +
> +		if ((p = strstr(opt, "passwd_file="))) {
> +			p += sizeof("passwd_file=") - 1;
> +			ret = mkfs_parse_s3_cfg_passwd(p, s3cfg.access_key,
> +						       s3cfg.secret_key);
> +			if (ret)
> +				return ret;
> +		} else if ((p = strstr(opt, "urlstyle="))) {
> +			p += sizeof("urlstyle=") - 1;
> +			if (strncmp(p, "vhost", 5) == 0) {
> +				s3cfg.url_style = S3EROFS_URL_STYLE_VIRTUAL_HOST;
> +			} else if (strncmp(p, "path", 4) == 0) {
> +				s3cfg.url_style = S3EROFS_URL_STYLE_PATH;
> +			} else {
> +				erofs_err("invalid S3 addressing style %s", p);
> +				return -EINVAL;
> +			}
> +		} else if ((p = strstr(opt, "sig="))) {
> +			p += strlen("sig=");
> +			if (strncmp(p, "4", 1) == 0) {
> +				erofs_warn("AWS Signature Version 4 is not supported yet, using Version 2");
> +			} else if (strncmp(p, "2", 1) == 0) {
> +				s3cfg.sig = S3EROFS_SIGNATURE_VERSION_2;
> +			} else {
> +				erofs_err("Invalid AWS Signature Version %s", p);
> +				return -EINVAL;
> +			}
> +		} else {
> +			erofs_err("invalid --s3 option %s", opt);
> +			return -EINVAL;
> +		}
> +
> +		opt = q ? q + 1 : NULL;
> +	}
> +
> +	return 0;
> +}
> +#endif
> +
>   static int mkfs_parse_one_compress_alg(char *alg,
>   				       struct erofs_compr_opts *copts)
>   {
> @@ -670,6 +818,13 @@ static int mkfs_parse_sources(int argc, char *argv[], int optind)
>   			erofstar.ios.dumpfd = fd;
>   		}
>   		break;
> +#ifdef S3EROFS_ENABLED
> +	case EROFS_MKFS_SOURCE_S3:
> +		cfg.c_src_path = strdup(argv[optind++]);
> +		if (!cfg.c_src_path)
> +			return -ENOMEM;
> +		break;
> +#endif
>   	default:
>   		erofs_err("unexpected source_mode: %d", source_mode);
>   		return -EINVAL;
> @@ -997,6 +1152,8 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
>   				dataimport_mode = EROFS_MKFS_DATA_IMPORT_FULLDATA;
>   			} else if (!strcmp(optarg, "rvsp")) {
>   				dataimport_mode = EROFS_MKFS_DATA_IMPORT_RVSP;
> +			} else if (!strcmp(optarg, "0")) {
> +				dataimport_mode = EROFS_MKFS_DATA_IMPORT_ZEROFILL;
>   			} else {
>   				errno = 0;
>   				dataimport_mode = strtol(optarg, &endptr, 0);
> @@ -1058,6 +1215,13 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
>   				return -EINVAL;
>   			}
>   			break;
> +#ifdef S3EROFS_ENABLED
> +		case 533:
> +			err = mkfs_parse_s3_cfg(optarg);
> +			if (err)
> +				return err;
> +			break;
> +#endif
>   		case 'V':
>   			version();
>   			exit(0);
> @@ -1538,35 +1702,7 @@ int main(int argc, char **argv)
>   
>   	erofs_inode_manager_init();
>   
> -	if (source_mode == EROFS_MKFS_SOURCE_TAR) {
> -		root = erofs_rebuild_make_root(&g_sbi);
> -		if (IS_ERR(root)) {
> -			err = PTR_ERR(root);
> -			goto exit;
> -		}
> -
> -		while (!(err = tarerofs_parse_tar(root, &erofstar)));
> -
> -		if (err < 0)
> -			goto exit;
> -
> -		err = erofs_rebuild_dump_tree(root, incremental_mode);
> -		if (err < 0)
> -			goto exit;
> -	} else if (source_mode == EROFS_MKFS_SOURCE_REBUILD) {
> -		root = erofs_rebuild_make_root(&g_sbi);
> -		if (IS_ERR(root)) {
> -			err = PTR_ERR(root);
> -			goto exit;
> -		}
> -
> -		err = erofs_mkfs_rebuild_load_trees(root);
> -		if (err)
> -			goto exit;
> -		err = erofs_rebuild_dump_tree(root, incremental_mode);
> -		if (err)
> -			goto exit;
> -	} else {
> +	if (source_mode == EROFS_MKFS_SOURCE_LOCALDIR) {
>   		err = erofs_build_shared_xattrs_from_path(&g_sbi, cfg.c_src_path);
>   		if (err) {
>   			erofs_err("failed to build shared xattrs: %s",
> @@ -1583,6 +1719,32 @@ int main(int argc, char **argv)
>   			root = NULL;
>   			goto exit;
>   		}
> +	} else {
> +		root = erofs_rebuild_make_root(&g_sbi);
> +		if (IS_ERR(root)) {
> +			err = PTR_ERR(root);
> +			goto exit;
> +		}
> +
> +		if (source_mode == EROFS_MKFS_SOURCE_TAR) {
> +			while (!(err = tarerofs_parse_tar(root, &erofstar)))
> +				;
> +			if (err < 0)
> +				goto exit;
> +		} else if (source_mode == EROFS_MKFS_SOURCE_REBUILD) {
> +			err = erofs_mkfs_rebuild_load_trees(root);
> +			if (err)
> +				goto exit;
> +#ifdef S3EROFS_ENABLED
> +		} else if (source_mode == EROFS_MKFS_SOURCE_S3) {
> +			err = -EOPNOTSUPP;
> +			goto exit;
> +#endif
> +		}
> +
> +		err = erofs_rebuild_dump_tree(root, incremental_mode);
> +		if (err)
> +			goto exit;
>   	}
>   
>   	if (tar_index_512b) {


More information about the Linux-erofs mailing list