[PATCH 2/2] erofs-utils: lib: support AWS SigV4 for S3 backend

[Gao Xiang]

Hi Yifan,

On 2025/11/20 17:22, Yifan Zhao wrote:
> This patch introduces support for AWS Signature Version 4 for s3erofs
> remote backend.
> 
> Now users can specify the folowing options:
>   - passwd_file=Y, S3 credentials file in the format $ak:$sk (optional);
>   - urlstyle=<vhost, path>, S3 API calling style (optional);
>   - sig=<2,4>, S3 API signature version (optional);
>   - region=W, region code for S3 endpoint (required for sig=4).
> 
> e.g.:
> mkfs.erofs \
>      --s3=s3.us-east-1.amazonaws.com,sig=4,region=us-east-1 \
>      output.img some_bucket/path/to/object

Thanks for the effort!

Could we find a public s3 bucket and post here as an example?

> 
> Signed-off-by: Yifan Zhao <zhaoyifan28 at huawei.com>
> ---
>   lib/liberofs_s3.h |   1 +
>   lib/remotes/s3.c  | 567 +++++++++++++++++++++++++++++++++++++---------
>   mkfs/main.c       |  14 +-
>   3 files changed, 471 insertions(+), 111 deletions(-)
> 
> diff --git a/lib/liberofs_s3.h b/lib/liberofs_s3.h
> index f2ec822..f4886cd 100644
> --- a/lib/liberofs_s3.h
> +++ b/lib/liberofs_s3.h
> @@ -27,6 +27,7 @@ enum s3erofs_signature_version {
>   struct erofs_s3 {
>   	void *easy_curl;
>   	const char *endpoint;
> +	const char *region;
>   	char access_key[S3_ACCESS_KEY_LEN + 1];
>   	char secret_key[S3_SECRET_KEY_LEN + 1];
>   
> diff --git a/lib/remotes/s3.c b/lib/remotes/s3.c
> index 0f7e1a9..3263dd7 100644
> --- a/lib/remotes/s3.c
> +++ b/lib/remotes/s3.c
> @@ -23,7 +23,8 @@
>   #define S3EROFS_PATH_MAX		1024
>   #define S3EROFS_MAX_QUERY_PARAMS	16
>   #define S3EROFS_URL_LEN			8192
> -#define S3EROFS_CANONICAL_QUERY_LEN	2048
> +#define S3EROFS_CANONICAL_URI_LEN	1024

Is there a spec to document that?

> +#define S3EROFS_CANONICAL_QUERY_LEN	S3EROFS_URL_LEN
>   
>   #define BASE64_ENCODE_LEN(len)	(((len + 2) / 3) * 4)
>   
> @@ -34,52 +35,142 @@ struct s3erofs_query_params {
>   };
>   
>   struct s3erofs_curl_request {
> -	const char *method;

It seems it's removed... S3 only allows `GET` method?

>   	char url[S3EROFS_URL_LEN];
> +	char canonical_uri[S3EROFS_CANONICAL_URI_LEN];
>   	char canonical_query[S3EROFS_CANONICAL_QUERY_LEN];
>   };
>   
> +static const char *s3erofs_parse_host(const char *endpoint, const char **schema) {

K&R style is:

static const char *s3erofs_parse_host()
{
	if (!tmp) {
		...
	} else {
		...
	}

}

> +	const char *tmp = strstr(endpoint, "://");
> +	const char *host;
> +
> +	if (!tmp) {
> +		host = endpoint;
> +		if (schema)
> +			*schema = NULL;
> +	} else {
> +		host = tmp + sizeof("://") - 1;
> +		if (schema) {
> +			*schema = strndup(endpoint, host - endpoint);
> +			if (!*schema)
> +				return ERR_PTR(-ENOMEM);
> +		}
> +	}
> +
> +	return host;
> +}
> +
> +static int s3erofs_urlencode(const char *input, char **output)
> +{

static void *s3erofs_urlencode(const char *input)
{
	char *output;

	output = malloc(strlen(input) * 3 + 1);
	if (!output)
		return ERR_PTR(-ENOMEM);

	...

	return output;

}

> +	static const char hex[] = "0123456789ABCDEF";
> +	int i;
> +	char c, *p;
> +
> +	*output = malloc(strlen(input) * 3 + 1);
> +	if (!*output)
> +		return -ENOMEM;
> +
> +	p = *output;
> +	for (i = 0; i < strlen(input); ++i) {
> +		c = (unsigned char)input[i];
> +
> +		// Unreserved characters: A-Z a-z 0-9 - . _ ~
> +		if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
> +		    (c >= '0' && c <= '9') || c == '-' || c == '.' || c == '_' ||
> +		    c == '~') {
> +			*p++ = c;
> +		} else {
> +			*p++ = '%';
> +			*p++ = hex[c >> 4];
> +			*p++ = hex[c & 0x0F];
> +		}
> +	}
> +	*p = '\0';
> +
> +	return 0;
> +}
> +
> +struct kv_pair {
> +	char *key;
> +	char *value;
> +};
> +
> +static int compare_kv_pair(const void *a, const void *b)
> +{
> +	return strcmp(((const struct kv_pair *)a)->key, ((const struct kv_pair *)b)->key);
> +}
> +
> +static int s3erofs_prepare_canonical_query(struct s3erofs_curl_request *req,
> +					   struct s3erofs_query_params *params)
> +{
> +	struct kv_pair *pairs;
> +	int i, pos = 0, ret = 0;
> +
> +	if (params->num == 0)

	if (!params->num) {
	}

> +		return 0;
> +
> +	pairs = malloc(sizeof(struct kv_pair) * params->num);
> +	for (i = 0; i < params->num; i++) {
> +		ret = s3erofs_urlencode(params->key[i], &pairs[i].key);
> +		if (ret < 0)
> +			goto out;
> +		ret = s3erofs_urlencode(params->value[i], &pairs[i].value);

Why we use urlencoding now?

Thanks,
Gao Xiang