[PATCH v2 3/4] erofs-utils: mkfs: introduce `--s3=...` option

[Hongbo Li]

On 2025/8/1 15:46, Gao Xiang wrote:
> 
> 
> On 2025/8/1 15:30, Yifan Zhao wrote:
>> From: zhaoyifan <zhaoyifan28 at huawei.com>
>>
>> This patch introduces configuration options for the upcoming 
>> experimental S3
>> support, including configuration parsing and passwd_file reading logic.
>>
>> User could specify the following options:
>> - S3 service endpoint (Compulsory)
>> - S3 credentials file, in the format of "$ak:%sk" (Optional)
>> - S3 API calling style (Optional)
>> - S3 API signature version, only sigV2 supported yet (Optional)
>>
>> Signed-off-by: Yifan Zhao <zhaoyifan28 at huawei.com>
>> ---
>> change since v1:
>> - rename: include/erofs/s3.h => lib/liberofs_s3.h
>> - add liberofs_s3.h in this patch rather than previous one
>>
>>   lib/liberofs_s3.h |  40 +++++++++
>>   lib/remotes/s3.c  |   3 +-
>>   mkfs/main.c       | 220 ++++++++++++++++++++++++++++++++++++++++------
>>   3 files changed, 233 insertions(+), 30 deletions(-)
>>   create mode 100644 lib/liberofs_s3.h
>>
>> diff --git a/lib/liberofs_s3.h b/lib/liberofs_s3.h
>> new file mode 100644
>> index 0000000..16a06c9
>> --- /dev/null
>> +++ b/lib/liberofs_s3.h
>> @@ -0,0 +1,40 @@
>> +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
>> +/*
>> + * Copyright (C) 2025 HUAWEI, Inc.
>> + *             http://www.huawei.com/
>> + * Created by Yifan Zhao <zhaoyifan28 at huawei.com>
>> + */
>> +#ifndef __EROFS_S3_H
>> +#define __EROFS_S3_H
>> +
>> +#ifdef __cplusplus
>> +extern "C" {
>> +#endif
>> +
>> +enum s3erofs_url_style {
>> +    S3EROFS_URL_STYLE_PATH,          // Path style: 
>> https://s3.amazonaws.com/bucket/object
>> +    S3EROFS_URL_STYLE_VIRTUAL_HOST,  // Virtual host style: 
>> https://bucket.s3.amazonaws.com/object
>> +};
>> +
>> +enum s3erofs_signature_version {
>> +    S3EROFS_SIGNATURE_VERSION_2,
>> +    S3EROFS_SIGNATURE_VERSION_4,
>> +};
>> +
>> +#define S3_ACCESS_KEY_LEN 256
>> +#define S3_SECRET_KEY_LEN 256
>> +
>> +struct erofs_s3 {
>> +    const char *endpoint, *bucket;
>> +    char access_key[S3_ACCESS_KEY_LEN + 1];
>> +    char secret_key[S3_SECRET_KEY_LEN + 1];
>> +
>> +    enum s3erofs_url_style url_style;
>> +    enum s3erofs_signature_version sig;
>> +};
>> +
>> +#ifdef __cplusplus
>> +}
>> +#endif
>> +
>> +#endif
>> \ No newline at end of file
>> diff --git a/lib/remotes/s3.c b/lib/remotes/s3.c
>> index ed2b023..358ee91 100644
>> --- a/lib/remotes/s3.c
>> +++ b/lib/remotes/s3.c
>> @@ -3,4 +3,5 @@
>>    * Copyright (C) 2025 HUAWEI, Inc.
>>    *             http://www.huawei.com/
>>    * Created by Yifan Zhao <zhaoyifan28 at huawei.com>
>> - */
>> \ No newline at end of file
>> + */
>> +#include "liberofs_s3.h"
>> \ No newline at end of file
>> diff --git a/mkfs/main.c b/mkfs/main.c
>> index 3aa1421..f524f45 100644
>> --- a/mkfs/main.c
>> +++ b/mkfs/main.c
>> @@ -31,6 +31,7 @@
>>   #include "../lib/liberofs_private.h"
>>   #include "../lib/liberofs_uuid.h"
>>   #include "../lib/liberofs_metabox.h"
>> +#include "../lib/liberofs_s3.h"
>>   #include "../lib/compressor.h"
>>   static struct option long_options[] = {
>> @@ -59,6 +60,9 @@ static struct option long_options[] = {
>>       {"gid-offset", required_argument, NULL, 17},
>>       {"tar", optional_argument, NULL, 20},
>>       {"aufs", no_argument, NULL, 21},
>> +#ifdef HAVE_S3
>> +    {"s3", required_argument, NULL, 22},
>> +#endif
>>       {"mount-point", required_argument, NULL, 512},
>>       {"xattr-prefix", required_argument, NULL, 19},
>>   #ifdef WITH_ANDROID
>> @@ -197,6 +201,12 @@ static void usage(int argc, char **argv)
>>           " --root-xattr-isize=#  ensure the inline xattr size of the 
>> root directory is # bytes at least\n"
>>           " --aufs                replace aufs special files with 
>> overlayfs metadata\n"
>>           " --sort=<path,none>    data sorting order for tarballs as 
>> input (default: path)\n"
>> +#ifdef HAVE_S3
> 
> HAVE_S3 is a bit odd, how about using
> S3_ENABLED (like LZ4_ENABLED?)
> 
> 
>> +        " --s3=X                generate an index-only image from 
>> s3-compatible object store backend\n"
>> +        "   [,passwd_file=Y]    X=endpoint, Y=s3 credentials file\n"
> 
> What's s3 credentials file? Is it documented
> somewhere? Why is it named as passwd_file?
> 
> Can we have an option to pass in accesskey
> too?

This follows the format of s3fs-fuse. Storing the ak/sk in a file is for 
security purposes. The file permission is set to 600 to prevent non-root 
users from accessing the ak/sk.

[1] https://github.com/s3fs-fuse/s3fs-fuse

Thanks,
Hongbo

> 
> 
>> +        "   [,style=Z]          S3 API calling style (Z = vhost|path) 
>> (default: vhost)\n"
>> +        "   [,sig=<2,4>]        S3 API signature version (default: 2)\n"
>> +#endif
>>           " --tar=X               generate a full or index-only image 
>> from a tarball(-ish) source\n"
>>           "                       (X = f|i|headerball; f=full mode, 
>> i=index mode,\n"
>>           "                                            headerball=file 
>> data is omited in the source stream)\n"
>> @@ -247,6 +257,10 @@ static struct erofs_tarfile erofstar = {
>>   static bool incremental_mode;
>>   static u8 metabox_algorithmid;
>> +#ifdef HAVE_S3
>> +static struct erofs_s3 s3cfg;
>> +#endif
>> +
>>   enum {
>>       EROFS_MKFS_DATA_IMPORT_DEFAULT,
>>       EROFS_MKFS_DATA_IMPORT_FULLDATA,
>> @@ -257,6 +271,9 @@ enum {
>>   enum {
>>       EROFS_MKFS_SOURCE_DEFAULT,
> 
> EROFS_MKFS_SOURCE_LOCALDIR,
> 
> Thanks,
> Gao Xiang