[PATCH v2 4/4] erofs-utils: mkfs: support EROFS index-only image generation from S3

Gao Xiang hsiangkao at linux.alibaba.com
Fri Aug 1 17:52:06 AEST 2025



On 2025/8/1 15:30, Yifan Zhao wrote:
> From: zhaoyifan <zhaoyifan28 at huawei.com>
> 
> This patch introduces experimental S3 support for mkfs.erofs, allowing EROFS
> images to be generated from AWS S3 (and other S3 API compatible services).
> 
> Currently the functionality is limited:
> - only index-only EROFS image generation are supported
> - only AWS Signature Version 2 is supported
> - only S3 object name and size are respected during image generation
> 
> Signed-off-by: Yifan Zhao <zhaoyifan28 at huawei.com>
> ---
> change since v1:
> - get rid of erofs_init_empty_dir() as described in commit cea8581
> 
>   lib/liberofs_s3.h |   2 +
>   lib/remotes/s3.c  | 605 +++++++++++++++++++++++++++++++++++++++++++++-
>   mkfs/main.c       |   5 +-
>   3 files changed, 609 insertions(+), 3 deletions(-)
> 
> diff --git a/lib/liberofs_s3.h b/lib/liberofs_s3.h
> index 16a06c9..a975fbb 100644
> --- a/lib/liberofs_s3.h
> +++ b/lib/liberofs_s3.h
> @@ -33,6 +33,8 @@ struct erofs_s3 {
>   	enum s3erofs_signature_version sig;
>   };
>   
> +int s3erofs_build_trees(struct erofs_inode *root, struct erofs_s3 *s3cfg);
> +
>   #ifdef __cplusplus
>   }
>   #endif
> diff --git a/lib/remotes/s3.c b/lib/remotes/s3.c
> index 358ee91..a062a50 100644
> --- a/lib/remotes/s3.c
> +++ b/lib/remotes/s3.c
> @@ -4,4 +4,607 @@
>    *             http://www.huawei.com/
>    * Created by Yifan Zhao <zhaoyifan28 at huawei.com>
>    */
> -#include "liberofs_s3.h"
> \ No newline at end of file

"\ No newline at end of file"

Let's check the editor first, in principle the patches
shouldn't have this.

> +#include <stdlib.h>
> +#include <time.h>
> +#include <curl/curl.h>
> +#include <libxml/parser.h>
> +#include <openssl/hmac.h>
> +#include "erofs/internal.h"
> +#include "erofs/print.h"
> +#include "erofs/inode.h"
> +#include "erofs/blobchunk.h"
> +#include "erofs/rebuild.h"
> +#include "liberofs_s3.h"
> +
> +#define S3EROFS_MAX_QUERY_PARAMS    16
> +#define S3EROFS_URL_LEN		    8192
> +#define S3EROFS_CANONICAL_QUERY_LEN 2048
> +
> +#define BASE64_ENCODE_LEN(len) (((len + 2) / 3) * 4)
> +
> +static CURL *easy_curl;
> +
> +struct s3erofs_query_params {
> +	int num;
> +	const char *key[S3EROFS_MAX_QUERY_PARAMS];
> +	const char *value[S3EROFS_MAX_QUERY_PARAMS];
> +};
> +
> +static void s3erofs_prepare_url(const char *endpoint, const char *bucket, const char *object,
> +				struct s3erofs_query_params *params, char *url,
> +				enum s3erofs_url_style url_style)
> +{
> +	const char *schema = NULL;
> +	const char *host = NULL;
> +	size_t pos = 0;
> +	int i;
> +
> +	if (!endpoint || !bucket || !url) {
> +		return;
> +	}
> +
> +	if (strncmp(endpoint, "https://", 8) == 0) {
> +		schema = "https://";
> +		host = endpoint + 8;
> +	} else if (strncmp(endpoint, "http://", 7) == 0) {
> +		schema = "http://";
> +		host = endpoint + 7;
> +	} else {
> +		schema = "https://";
> +		host = endpoint;
> +	}
> +
> +	if (url_style == S3EROFS_URL_STYLE_VIRTUAL_HOST) {
> +		pos += snprintf(url, S3EROFS_URL_LEN, "%s%s.%s", schema, bucket, host);
> +	} else {
> +		pos += snprintf(url, S3EROFS_URL_LEN, "%s%s/%s", schema, host, bucket);
> +	}
> +
> +	if (object) {
> +		pos += snprintf(url + pos, S3EROFS_URL_LEN - pos, "/%s", object);
> +	}
> +
> +	for (i = 0; i < params->num; i++) {
> +		pos += snprintf(url + pos, S3EROFS_URL_LEN - pos, "%c%s=%s", (i == 0 ? '?' : '&'),
> +				params->key[i], params->value[i]);
> +	}
> +}
> +
> +static char *get_canonical_headers(const struct curl_slist *list) { return ""; }
> +
> +static char *s3erofs_sigv2_header(const struct curl_slist *headers, const char *method,
> +				  const char *content_md5, const char *content_type,
> +				  const char *date, const char *canonical_query, const char *ak,
> +				  const char *sk)
> +{
> +	char *str, *output = NULL;
> +	u8 *hmac_signature;
> +	unsigned int len = 0, pos = 0, output_len;
> +	const char *canonical_headers = get_canonical_headers(headers);
> +	const char *prefix = "Authorization: AWS ";
> +
> +	if (!method || !date || !ak || !sk) {
> +		return ERR_PTR(-EINVAL);
> +	}
> +
> +	if (!content_md5)
> +		content_md5 = "";
> +	if (!content_type)
> +		content_type = "";
> +	if (!canonical_query)
> +		canonical_query = "/";
> +
> +	len += strlen(method) + 1;
> +	len += strlen(content_md5) + 1;
> +	len += strlen(content_type) + 1;
> +	len += strlen(date) + 1;
> +	len += strlen(canonical_headers);
> +	len += strlen(canonical_query);
> +
> +	str = (char *)malloc(++len);
> +	if (!str)
> +		return ERR_PTR(-ENOMEM);
> +
> +	pos += snprintf(str + pos, len - pos, "%s\n", method);
> +	pos += snprintf(str + pos, len - pos, "%s\n", content_md5);
> +	pos += snprintf(str + pos, len - pos, "%s\n", content_type);
> +	pos += snprintf(str + pos, len - pos, "%s\n", date);
> +	pos += snprintf(str + pos, len - pos, "%s", canonical_headers);
> +	pos += snprintf(str + pos, len - pos, "%s", canonical_query);
> +
> +	hmac_signature = (u8 *)malloc(EVP_MAX_MD_SIZE);
> +	if (!hmac_signature)
> +		goto free_string;
> +
> +	if (!HMAC(EVP_sha1(), sk, strlen(sk), (u8 *)str, strlen(str), hmac_signature, &len))
> +		goto free_hmac;
> +
> +	output_len = BASE64_ENCODE_LEN(len);
> +	output_len += strlen(prefix);
> +	output_len += strlen(ak);
> +	/* for ':' between ak and signature*/
> +	output_len += 1;
> +
> +	output = (char *)malloc(output_len);
> +	if (!output)
> +		goto free_hmac;
> +
> +	memcpy(output, prefix, strlen(prefix));
> +	memcpy(output + strlen(prefix), ak, strlen(ak));
> +	output[strlen(prefix) + strlen(ak)] = ':';
> +
> +	EVP_EncodeBlock((unsigned char *)output + strlen(prefix) + strlen(ak) + 1, hmac_signature,
> +			len);
> +
> +free_hmac:
> +	free(hmac_signature);
> +free_string:
> +	free(str);
> +	return output ?: ERR_PTR(-ENOMEM);
> +}
> +
> +static void s3erofs_now_rfc1123(char *buf, size_t maxlen)
> +{
> +	time_t now = time(NULL);
> +	struct tm *ptm = gmtime(&now);
> +	strftime(buf, maxlen, "%a, %d %b %Y %H:%M:%S GMT", ptm);
> +}
> +
> +struct s3erofs_curl_request {
> +	const char *method;
> +	char url[S3EROFS_URL_LEN];
> +	char canonical_query[S3EROFS_CANONICAL_QUERY_LEN];
> +};
> +
> +struct s3erofs_curl_response {
> +	char *data;
> +	size_t size;
> +};
> +
> +static size_t s3erofs_request_write_memory_cb(void *contents, size_t size, size_t nmemb,
> +					      void *userp)
> +{
> +	size_t realsize = size * nmemb;
> +	struct s3erofs_curl_response *response = (struct s3erofs_curl_response *)userp;
> +	void *tmp;
> +
> +	tmp = realloc(response->data, response->size + realsize + 1);
> +	if (tmp == NULL)
> +		return 0;
> +
> +	response->data = tmp;
> +
> +	memcpy(response->data + response->size, contents, realsize);
> +	response->size += realsize;
> +	response->data[response->size] = '\0';
> +
> +	return realsize;
> +}
> +
> +static int s3erofs_request_insert_auth(struct curl_slist **request_headers, const char *method,
> +				       const char *canonical_query, const char *ak, const char *sk)
> +{
> +	char date_header[40];
> +	const char *date_prefix = "Date: ";
> +	size_t date_prefix_len = strlen(date_prefix);
> +	char *sigv2_header;
> +
> +	memcpy(date_header, date_prefix, date_prefix_len);
> +	s3erofs_now_rfc1123(date_header + date_prefix_len, sizeof(date_header) - date_prefix_len);
> +
> +	sigv2_header = s3erofs_sigv2_header(*request_headers, method, NULL, NULL,
> +					    date_header + date_prefix_len, canonical_query, ak, sk);
> +	if (IS_ERR(sigv2_header))
> +		return PTR_ERR(sigv2_header);
> +
> +	*request_headers = curl_slist_append(*request_headers, date_header);
> +	*request_headers = curl_slist_append(*request_headers, sigv2_header);
> +
> +	free(sigv2_header);
> +	return 0;
> +}
> +
> +static int s3erofs_request_perform(struct erofs_s3 *s3cfg, struct s3erofs_curl_request *req,
> +				   struct s3erofs_curl_response *resp)
> +{
> +	struct curl_slist *request_headers = NULL;
> +	long http_code = 0;
> +	int ret;
> +
> +	ret = s3erofs_request_insert_auth(&request_headers, req->method, req->canonical_query,
> +					  s3cfg->access_key, s3cfg->secret_key);
> +	if (ret < 0) {
> +		erofs_err("failed to insert auth headers\n");
> +		return ret;
> +	}
> +
> +	curl_easy_setopt(easy_curl, CURLOPT_URL, req->url);
> +	curl_easy_setopt(easy_curl, CURLOPT_WRITEDATA, resp);
> +	curl_easy_setopt(easy_curl, CURLOPT_HTTPHEADER, request_headers);
> +
> +	ret = curl_easy_perform(easy_curl);
> +	if (ret != CURLE_OK) {
> +		erofs_err("curl_easy_perform() failed: %s\n", curl_easy_strerror(ret));
> +		ret = -EIO;
> +		goto err_header;
> +	}
> +
> +	ret = curl_easy_getinfo(easy_curl, CURLINFO_RESPONSE_CODE, &http_code);
> +	if (ret != CURLE_OK) {
> +		erofs_err("curl_easy_getinfo() failed: %s\n", curl_easy_strerror(ret));
> +		ret = -EIO;
> +		goto err_header;
> +	}
> +
> +	if (!(http_code >= 200 && http_code < 300)) {
> +		erofs_err("request failed with HTTP code %ld\n", http_code);
> +		ret = -EIO;
> +	}
> +
> +err_header:
> +	curl_slist_free_all(request_headers);
> +	return ret;
> +}
> +
> +struct s3erofs_object_info {
> +	char *key;
> +	u64 size;
> +};
> +
> +struct s3erofs_object_iterator {
> +	struct s3erofs_object_info *objects;
> +	int cur, total;
> +
> +	struct erofs_s3 *s3cfg;
> +	const char *prefix, *delimiter;
> +
> +	char *next_marker;
> +	bool is_truncated;
> +};
> +
> +static int s3erofs_parse_list_objects_one(xmlNodePtr node, struct s3erofs_object_info *info)
> +{
> +	xmlNodePtr child;
> +	xmlChar *str;
> +
> +	for (child = node->children; child; child = child->next) {
> +		if (child->type == XML_ELEMENT_NODE) {
> +			str = xmlNodeGetContent(child);
> +			if (!str) {
> +				return -ENOMEM;
> +			}
> +
> +			if (xmlStrEqual(child->name, (const xmlChar *)"Key")) {
> +				info->key = strdup((char *)str);
> +			} else if (xmlStrEqual(child->name, (const xmlChar *)"Size")) {
> +				info->size = atoll((char *)str);
> +			}
> +
> +			xmlFree(str);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
> +static int s3erofs_parse_list_objects_result(const char *data, int len,
> +					     struct s3erofs_object_iterator *it)
> +{
> +	xmlDocPtr doc = NULL;
> +	xmlNodePtr root = NULL, node;
> +	xmlChar *str;
> +	xmlNodePtr contents_nodes[1000];
> +	int contents_count = 0;
> +	int i = 0;
> +	int ret = 0;
> +	void *tmp;
> +
> +	doc = xmlReadMemory(data, len, NULL, NULL, 0);
> +	if (!doc) {
> +		erofs_err("failed to parse XML data\n");
> +		return -EINVAL;
> +	}
> +
> +	root = xmlDocGetRootElement(doc);
> +	if (!root) {
> +		erofs_err("failed to get root element\n");
> +		ret = -EINVAL;
> +		goto out;
> +	}
> +
> +	if (!xmlStrEqual(root->name, (const xmlChar *)"ListBucketResult")) {
> +		erofs_err("invalid root element: expected ListBucketResult, got %s", root->name);
> +		ret = -EINVAL;
> +		goto out;
> +	}
> +
> +	for (node = root->children; node; node = node->next) {
> +		if (node->type == XML_ELEMENT_NODE) {
> +			if (xmlStrEqual(node->name, (const xmlChar *)"Contents")) {
> +				if (contents_count < 1000)
> +					contents_nodes[contents_count++] = node;
> +			} else if (xmlStrEqual(node->name, (const xmlChar *)"IsTruncated")) {
> +				str = xmlNodeGetContent(node);
> +				if (str) {
> +					it->is_truncated =
> +					    xmlStrEqual(str, (const xmlChar *)"true") ? true
> +										      : false;
> +					xmlFree(str);
> +				}
> +			} else if (xmlStrEqual(node->name, (const xmlChar *)"NextMarker")) {
> +				str = xmlNodeGetContent(node);
> +				if (str) {
> +					if (it->next_marker)
> +						free(it->next_marker);
> +					it->next_marker = strdup((char *)str);
> +					if (!it->next_marker) {
> +						xmlFree(str);
> +						ret = -ENOMEM;
> +						goto out;
> +					}
> +					xmlFree(str);
> +				}
> +			}
> +		}
> +	}
> +
> +	if (contents_count == 0)
> +		goto out;
> +
> +	tmp = realloc(it->objects, contents_count * sizeof(struct s3erofs_object_info));
> +	if (!tmp) {
> +		ret = -ENOMEM;
> +		goto out;
> +	}
> +	memset(tmp, 0, contents_count * sizeof(struct s3erofs_object_info));
> +	it->objects = tmp;
> +
> +	for (i = 0; i < contents_count; i++) {
> +		ret = s3erofs_parse_list_objects_one(contents_nodes[i], &it->objects[i]);
> +		if (ret < 0) {
> +			erofs_err("failed to parse contents node #%d", i);
> +			while (--i >= 0)
> +				free(it->objects[i].key);
> +			free(it->objects);
> +			goto out;
> +		}
> +	}
> +
> +	it->total = contents_count;
> +
> +out:
> +	xmlFreeDoc(doc);
> +	return ret;
> +}
> +
> +static int s3erofs_list_objects(struct s3erofs_object_iterator *it)
> +{
> +	struct s3erofs_curl_request req = {0};
> +	struct s3erofs_curl_response resp = {0};
> +	struct s3erofs_query_params params = {0};
> +	int ret = 0;
> +
> +	struct erofs_s3 *s3cfg = it->s3cfg;
> +
> +	if (it->prefix && strlen(it->prefix ) > 1024) {
> +		erofs_err("prefix is too long");
> +		return -EINVAL;
> +	}
> +
> +	if (it->delimiter && strlen(it->delimiter) > 1024) {
> +		erofs_err("delimiter is too long");
> +		return -EINVAL;
> +	}
> +
> +	if (it->prefix) {
> +		params.key[params.num] = "prefix";
> +		params.value[params.num] = it->prefix;
> +		++params.num;
> +	}
> +
> +	if (it->delimiter) {
> +		params.key[params.num] = "delimiter";
> +		params.value[params.num] = it->delimiter;
> +		++params.num;
> +	}
> +
> +	if (it->next_marker) {
> +		params.key[params.num] = "marker";
> +		params.value[params.num] = it->next_marker;
> +		++params.num;
> +	}
> +
> +	params.key[params.num] = "max-keys";
> +	params.value[params.num] = "1000";
> +	++params.num;
> +
> +	req.method = "GET";
> +	s3erofs_prepare_url(s3cfg->endpoint, s3cfg->bucket, NULL, &params, req.url,
> +				s3cfg->url_style);
> +	snprintf(req.canonical_query, S3EROFS_CANONICAL_QUERY_LEN, "/%s", s3cfg->bucket);
> +
> +	ret = s3erofs_request_perform(s3cfg, &req, &resp);
> +	if (ret < 0)
> +		return ret;
> +
> +	ret = s3erofs_parse_list_objects_result(resp.data, resp.size, it);
> +	if (ret < 0)
> +		return ret;
> +
> +	return 0;
> +}
> +
> +static struct s3erofs_object_iterator *
> +s3erofs_create_object_iterator(struct erofs_s3 *s3cfg, const char *prefix,
> +			       const char *delimiter)
> +{
> +	struct s3erofs_object_iterator *it = calloc(1, sizeof(*it));
> +
> +	it->s3cfg = s3cfg;
> +	it->prefix = prefix;
> +	it->delimiter = delimiter;
> +
> +	it->is_truncated = true;
> +
> +	return it;
> +}
> +
> +static void s3erofs_destroy_object_iterator(struct s3erofs_object_iterator *it)
> +{
> +	if (it->next_marker)
> +		free(it->next_marker);
> +	if (it->objects) {
> +		for (int i = 0; i < it->total; i++)
> +			free(it->objects[i].key);
> +		free(it->objects);
> +	}
> +
> +	free(it);
> +}
> +
> +static struct s3erofs_object_info *
> +s3erofs_get_next_object(struct s3erofs_object_iterator *it)
> +{
> +	int ret = 0;
> +
> +	if (it->cur < it->total) {
> +		return &it->objects[it->cur++];
> +	}
> +
> +	if (it->is_truncated) {
> +		ret = s3erofs_list_objects(it);
> +		if (ret < 0)
> +			return ERR_PTR(ret);
> +
> +		it->cur = 0;
> +		return &it->objects[it->cur++];
> +	}
> +
> +	return NULL;
> +}
> +
> +static int s3erofs_global_init(void)
> +{
> +	int ret;

Does it support multiple remotes?

Use a mutex to protect this for multiple instances?

> +
> +	ret = curl_global_init(CURL_GLOBAL_DEFAULT);
> +	if (ret != CURLE_OK)
> +		return -EIO;
> +
> +	easy_curl = curl_easy_init();
> +	if (!easy_curl) {
> +		curl_global_cleanup();
> +		return -EIO;
> +	}
> +
> +	curl_easy_setopt(easy_curl, CURLOPT_WRITEFUNCTION, s3erofs_request_write_memory_cb);
> +	curl_easy_setopt(easy_curl, CURLOPT_FOLLOWLOCATION, 1L);
> +	curl_easy_setopt(easy_curl, CURLOPT_TIMEOUT, 30L);
> +
> +	xmlInitParser();
> +
> +	return ret;
> +}
> +
> +static void s3erofs_global_exit(void)
> +{
> +	if (!easy_curl)
> +		return;
> +
> +	xmlCleanupParser();
> +
> +	curl_easy_cleanup(easy_curl);
> +	easy_curl = NULL;
> +
> +	curl_global_cleanup();
> +}
> +
> +int s3erofs_build_trees(struct erofs_inode *root, struct erofs_s3 *s3cfg)
> +{
> +	struct erofs_sb_info *sbi = root->sbi;
> +	struct s3erofs_object_iterator *iter;
> +	struct s3erofs_object_info *obj;
> +	struct erofs_dentry *d;
> +	struct erofs_inode *inode;
> +	struct stat st;
> +	bool dumb;
> +	int ret = 0;
> +
> +	st.st_uid = getuid();
> +	st.st_gid = getgid();
> +
> +	/* XXX */
> +	st.st_mtime = sbi->epoch;


	st.st_mtime = sbi->epoch + sbi->build_time?

build_time can be specified by users too.

Thanks,
Gao Xiang


More information about the Linux-erofs mailing list