[PATCH v2 3/4] erofs-utils: mkfs: introduce `--s3=...` option

[Gao Xiang]

On 2025/8/1 15:30, Yifan Zhao wrote:
> From: zhaoyifan <zhaoyifan28 at huawei.com>
> 
> This patch introduces configuration options for the upcoming experimental S3
> support, including configuration parsing and passwd_file reading logic.
> 
> User could specify the following options:
> - S3 service endpoint (Compulsory)
> - S3 credentials file, in the format of "$ak:%sk" (Optional)
> - S3 API calling style (Optional)
> - S3 API signature version, only sigV2 supported yet (Optional)
> 
> Signed-off-by: Yifan Zhao <zhaoyifan28 at huawei.com>
> ---
> change since v1:
> - rename: include/erofs/s3.h => lib/liberofs_s3.h
> - add liberofs_s3.h in this patch rather than previous one
> 
>   lib/liberofs_s3.h |  40 +++++++++
>   lib/remotes/s3.c  |   3 +-
>   mkfs/main.c       | 220 ++++++++++++++++++++++++++++++++++++++++------
>   3 files changed, 233 insertions(+), 30 deletions(-)
>   create mode 100644 lib/liberofs_s3.h
> 
> diff --git a/lib/liberofs_s3.h b/lib/liberofs_s3.h
> new file mode 100644
> index 0000000..16a06c9
> --- /dev/null
> +++ b/lib/liberofs_s3.h
> @@ -0,0 +1,40 @@
> +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
> +/*
> + * Copyright (C) 2025 HUAWEI, Inc.
> + *             http://www.huawei.com/
> + * Created by Yifan Zhao <zhaoyifan28 at huawei.com>
> + */
> +#ifndef __EROFS_S3_H
> +#define __EROFS_S3_H
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +enum s3erofs_url_style {
> +    S3EROFS_URL_STYLE_PATH,          // Path style: https://s3.amazonaws.com/bucket/object
> +    S3EROFS_URL_STYLE_VIRTUAL_HOST,  // Virtual host style: https://bucket.s3.amazonaws.com/object
> +};
> +
> +enum s3erofs_signature_version {
> +	S3EROFS_SIGNATURE_VERSION_2,
> +	S3EROFS_SIGNATURE_VERSION_4,
> +};
> +
> +#define S3_ACCESS_KEY_LEN 256
> +#define S3_SECRET_KEY_LEN 256
> +
> +struct erofs_s3 {
> +	const char *endpoint, *bucket;
> +	char access_key[S3_ACCESS_KEY_LEN + 1];
> +	char secret_key[S3_SECRET_KEY_LEN + 1];
> +
> +	enum s3erofs_url_style url_style;
> +	enum s3erofs_signature_version sig;
> +};
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif
> \ No newline at end of file
> diff --git a/lib/remotes/s3.c b/lib/remotes/s3.c
> index ed2b023..358ee91 100644
> --- a/lib/remotes/s3.c
> +++ b/lib/remotes/s3.c
> @@ -3,4 +3,5 @@
>    * Copyright (C) 2025 HUAWEI, Inc.
>    *             http://www.huawei.com/
>    * Created by Yifan Zhao <zhaoyifan28 at huawei.com>
> - */
> \ No newline at end of file
> + */
> +#include "liberofs_s3.h"
> \ No newline at end of file
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 3aa1421..f524f45 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -31,6 +31,7 @@
>   #include "../lib/liberofs_private.h"
>   #include "../lib/liberofs_uuid.h"
>   #include "../lib/liberofs_metabox.h"
> +#include "../lib/liberofs_s3.h"
>   #include "../lib/compressor.h"
>   
>   static struct option long_options[] = {
> @@ -59,6 +60,9 @@ static struct option long_options[] = {
>   	{"gid-offset", required_argument, NULL, 17},
>   	{"tar", optional_argument, NULL, 20},
>   	{"aufs", no_argument, NULL, 21},
> +#ifdef HAVE_S3
> +	{"s3", required_argument, NULL, 22},
> +#endif
>   	{"mount-point", required_argument, NULL, 512},
>   	{"xattr-prefix", required_argument, NULL, 19},
>   #ifdef WITH_ANDROID
> @@ -197,6 +201,12 @@ static void usage(int argc, char **argv)
>   		" --root-xattr-isize=#  ensure the inline xattr size of the root directory is # bytes at least\n"
>   		" --aufs                replace aufs special files with overlayfs metadata\n"
>   		" --sort=<path,none>    data sorting order for tarballs as input (default: path)\n"
> +#ifdef HAVE_S3

HAVE_S3 is a bit odd, how about using
S3_ENABLED (like LZ4_ENABLED?)


> +		" --s3=X                generate an index-only image from s3-compatible object store backend\n"
> +		"   [,passwd_file=Y]    X=endpoint, Y=s3 credentials file\n"

What's s3 credentials file? Is it documented
somewhere? Why is it named as passwd_file?

Can we have an option to pass in accesskey
too?


> +		"   [,style=Z]          S3 API calling style (Z = vhost|path) (default: vhost)\n"
> +		"   [,sig=<2,4>]        S3 API signature version (default: 2)\n"
> +#endif
>   		" --tar=X               generate a full or index-only image from a tarball(-ish) source\n"
>   		"                       (X = f|i|headerball; f=full mode, i=index mode,\n"
>   		"                                            headerball=file data is omited in the source stream)\n"
> @@ -247,6 +257,10 @@ static struct erofs_tarfile erofstar = {
>   static bool incremental_mode;
>   static u8 metabox_algorithmid;
>   
> +#ifdef HAVE_S3
> +static struct erofs_s3 s3cfg;
> +#endif
> +
>   enum {
>   	EROFS_MKFS_DATA_IMPORT_DEFAULT,
>   	EROFS_MKFS_DATA_IMPORT_FULLDATA,
> @@ -257,6 +271,9 @@ enum {
>   enum {
>   	EROFS_MKFS_SOURCE_DEFAULT,

EROFS_MKFS_SOURCE_LOCALDIR,

Thanks,
Gao Xiang