[PATCH 1/2] erofs-utils: mount: support mounting EROFS stored as an AWS S3 object
Yifan Zhao
stopire at gmail.com
Sat Apr 18 00:15:30 AEST 2026
On 4/17/2026 6:18 PM, Gao Xiang wrote:
> Allow mount.erofs to directly mount an EROFS filesystem stored as
> an AWS S3 object without downloading it first: (meta)data is fetched
> on demand via HTTP range requests.
>
> The source argument takes the form "bucket/key", e.g.:
>
> $ mount.erofs -t erofs.nbd \
> -o s3.endpoint=s3.amazonaws.com,s3.passwd_file=/path/to/passwd \
> my-bucket/dir/foo.erofs mnt
>
> In addition, AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment
> variables are also honored as fallback credentials.
>
> Assisted-by: qoder:(unknown)
> Cc: Yuxuan Liu <cdjddzy at foxmail.com>
> Signed-off-by: Gao Xiang <hsiangkao at linux.alibaba.com>
> ---
> lib/liberofs_s3.h | 5 +
> lib/remotes/s3.c | 303 ++++++++++++++++++++++++++++++++++++++++++++--
> mkfs/main.c | 71 +----------
> mount/main.c | 237 ++++++++++++++++++++++++++++++------
> 4 files changed, 495 insertions(+), 121 deletions(-)
>
> diff --git a/lib/liberofs_s3.h b/lib/liberofs_s3.h
> index c81834785c5f..3d2b2727b3b6 100644
> --- a/lib/liberofs_s3.h
> +++ b/lib/liberofs_s3.h
> @@ -35,8 +35,13 @@ struct erofs_s3 {
> enum s3erofs_signature_version sig;
> };
>
> +struct erofs_vfile;
> +
> int s3erofs_build_trees(struct erofs_importer *im, struct erofs_s3 *s3,
> const char *path, bool fillzero);
> +struct erofs_vfile *s3erofs_io_open(struct erofs_s3 *s3, const char *bucket,
> + const char *key);
> +int s3erofs_parse_s3fs_passwd(const char *filepath, char *ak, char *sk);
>
> #ifdef __cplusplus
> }
> diff --git a/lib/remotes/s3.c b/lib/remotes/s3.c
> index 964555d38432..35df935f8328 100644
> --- a/lib/remotes/s3.c
> +++ b/lib/remotes/s3.c
> @@ -20,6 +20,7 @@
> #include "erofs/importer.h"
> #include "liberofs_rebuild.h"
> #include "liberofs_s3.h"
> +#include "liberofs_base64.h"
>
> #define S3EROFS_PATH_MAX 1024
> #define S3EROFS_MAX_QUERY_PARAMS 16
> @@ -39,6 +40,7 @@ struct s3erofs_curl_request {
> char url[S3EROFS_URL_LEN];
> char canonical_uri[S3EROFS_CANONICAL_URI_LEN];
> char canonical_query[S3EROFS_CANONICAL_QUERY_LEN];
> + const char *method;
> };
>
> static const char *s3erofs_parse_host(const char *endpoint, const char **schema)
> @@ -353,6 +355,7 @@ static void s3erofs_to_hex(const u8 *data, size_t len, char *output)
>
> // See: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTAuthentication.html#ConstructingTheAuthenticationHeader
> static char *s3erofs_sigv2_header(const struct curl_slist *headers,
> + const char *request_method,
> const char *content_md5,
> const char *content_type, const char *date,
> const char *canonical_uri, const char *ak,
> @@ -373,8 +376,8 @@ static char *s3erofs_sigv2_header(const struct curl_slist *headers,
> if (!canonical_uri)
> canonical_uri = "/";
>
> - pos = asprintf(&str, "GET\n%s\n%s\n%s\n%s%s", content_md5, content_type,
> - date, "", canonical_uri);
> + pos = asprintf(&str, "%s\n%s\n%s\n%s\n%s%s", request_method,
> + content_md5, content_type, date, "", canonical_uri);
> if (pos < 0)
> return ERR_PTR(-ENOMEM);
>
> @@ -401,6 +404,7 @@ free_string:
>
> // See: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
> static char *s3erofs_sigv4_header(const struct curl_slist *headers,
> + const char *request_method,
> time_t request_time, const char *canonical_uri,
> const char *canonical_query, const char *region,
> const char *ak, const char *sk)
> @@ -430,13 +434,11 @@ static char *s3erofs_sigv4_header(const struct curl_slist *headers,
>
> // Task 1: Create canonical request
> if (asprintf(&canonical_request,
> - "GET\n"
> - "%s\n"
> - "%s\n"
> - "%s\n"
> + "%s\n%s\n%s\n%s\n"
> "host;x-amz-content-sha256;x-amz-date\n"
> "UNSIGNED-PAYLOAD",
> - canonical_uri, canonical_query, canonical_headers) < 0) {
> + request_method, canonical_uri, canonical_query,
> + canonical_headers) < 0) {
> err = -ENOMEM;
> goto err_canonical_headers;
> }
> @@ -533,7 +535,7 @@ static int s3erofs_request_insert_auth_v2(struct curl_slist **request_headers,
> s3erofs_format_time(time(NULL), date + sizeof(date_prefix) - 1,
> sizeof(date) - sizeof(date_prefix) + 1, S3EROFS_DATE_RFC1123);
>
> - sigv2 = s3erofs_sigv2_header(*request_headers, NULL, NULL,
> + sigv2 = s3erofs_sigv2_header(*request_headers, req->method, NULL, NULL,
> date + sizeof(date_prefix) - 1, req->canonical_uri,
> s3->access_key, s3->secret_key);
> if (IS_ERR(sigv2))
> @@ -576,7 +578,7 @@ static int s3erofs_request_insert_auth_v4(struct curl_slist **request_headers,
> *request_headers = curl_slist_append(*request_headers, tmp);
> free(tmp);
>
> - sigv4 = s3erofs_sigv4_header(*request_headers, request_time,
> + sigv4 = s3erofs_sigv4_header(*request_headers, req->method, request_time,
> req->canonical_uri, req->canonical_query,
> s3->region, s3->access_key, s3->secret_key);
> if (IS_ERR(sigv4))
> @@ -619,6 +621,13 @@ static int s3erofs_request_perform(struct erofs_s3 *s3,
> long http_code = 0;
> int ret;
>
> + if (!strcmp(req->method, "HEAD")) {
> + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
> + } else {
> + curl_easy_setopt(curl, CURLOPT_NOBODY, 0L);
> + curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);
> + }
> +
> if (s3->access_key[0]) {
> if (s3->sig == S3EROFS_SIGNATURE_VERSION_4)
> ret = s3erofs_request_insert_auth_v4(&request_headers, req, s3);
> @@ -846,7 +855,7 @@ out:
>
> static int s3erofs_list_objects(struct s3erofs_object_iterator *it)
> {
> - struct s3erofs_curl_request req = {};
> + struct s3erofs_curl_request req = { .method = "GET", };
> struct s3erofs_curl_response resp = {};
> struct s3erofs_query_params params;
> struct erofs_s3 *s3 = it->s3;
> @@ -1014,7 +1023,7 @@ static int s3erofs_remote_getobject(struct erofs_importer *im,
> const char *bucket, const char *key)
> {
> struct erofs_sb_info *sbi = inode->sbi;
> - struct s3erofs_curl_request req = {};
> + struct s3erofs_curl_request req = { .method = "GET", };
> struct s3erofs_curl_getobject_resp resp;
> struct erofs_vfile vf;
> u64 diskbuf_off;
> @@ -1170,6 +1179,276 @@ err_global:
> return ret;
> }
>
> +struct s3erofs_vfile {
> + struct erofs_vfile vf;
> + struct erofs_s3 *s3;
> + char *bucket, *key;
> + u64 offset, size;
> +};
> +
> +struct s3erofs_range_resp {
> + void *buf;
> + size_t len;
> +};
> +
> +static size_t s3erofs_range_write_cb(void *contents, size_t size,
> + size_t nmemb, void *userp)
> +{
> + struct s3erofs_range_resp *resp = userp;
> + size_t realsize = size * nmemb;
> +
> + if (realsize > resp->len)
> + return 0;
> +
> + memcpy(resp->buf, contents, realsize);
> + resp->buf = (char *)resp->buf + realsize;
> + resp->len -= realsize;
> + return realsize;
> +}
> +
> +static int s3erofs_get_object_range(struct s3erofs_vfile *s3vf,
> + void *buf, size_t len, u64 offset)
> +{
> + struct s3erofs_curl_request req = { .method = "GET", };
> + struct erofs_s3 *s3 = s3vf->s3;
> + struct s3erofs_range_resp resp;
> + CURL *curl = s3->easy_curl;
> + u64 end = offset + len;
> + long http_code = 0;
> + char range[64];
> + int ret;
> +
> + if (end > s3vf->size)
> + end = s3vf->size;
> + if (__erofs_unlikely(end <= offset))
> + return 0;
> + resp.buf = buf;
> + resp.len = end - offset;
> +
> + ret = s3erofs_prepare_url(&req, s3->endpoint, s3vf->bucket,
> + s3vf->key, NULL, s3->url_style, s3->sig);
> + if (ret < 0)
> + return ret;
> +
> + /* Add Range header for partial content */
> + snprintf(range, sizeof(range), "%llu-%llu", offset | 0ULL, (end - 1) | 0ULL);
> +
> + curl_easy_setopt(curl, CURLOPT_RANGE, range);
> + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, s3erofs_range_write_cb);
> +
> + ret = s3erofs_request_perform(s3, &req, &resp);
> + if (ret)
> + return ret;
> +
> + ret = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
> + if (ret != CURLE_OK) {
> + erofs_err("curl_easy_getinfo() failed: %s",
> + curl_easy_strerror(ret));
> + return -EIO;
> + }
> +
> + if (http_code != 206 && http_code != 200) {
> + erofs_err("S3 range request failed with HTTP code %ld", http_code);
> + return -EIO;
> + }
> + return len - resp.len; /* actual bytes read */
> +}
> +
> +static ssize_t s3erofs_io_pread(struct erofs_vfile *vf, void *buf,
> + size_t len, u64 offset)
> +{
> + struct s3erofs_vfile *s3vf = (struct s3erofs_vfile *)vf;
> + int ret;
> +
> + if (offset >= s3vf->size) {
> + memset(buf, 0, len);
> + return len;
> + }
> + ret = s3erofs_get_object_range(s3vf, buf, len, offset);
> + if (ret >= 0 && ret < len) {
> + memset(buf + ret, 0, len - ret);
> + return len;
> + }
> + return ret;
> +}
> +
> +static ssize_t s3erofs_io_read(struct erofs_vfile *vf, void *buf, size_t len)
> +{
> + struct s3erofs_vfile *s3vf = (struct s3erofs_vfile *)vf;
> + ssize_t ret;
> +
> + ret = s3erofs_io_pread(vf, buf, len, s3vf->offset);
> + if (ret > 0)
> + s3vf->offset += ret;
> + return ret;
> +}
> +
> +static void s3erofs_io_close(struct erofs_vfile *vf)
> +{
> + struct s3erofs_vfile *s3vf = (struct s3erofs_vfile *)vf;
> +
> + if (!s3vf)
> + return;
> +
> + s3erofs_curl_easy_exit(s3vf->s3);
> + free(s3vf->bucket);
> + free(s3vf->key);
> + free(s3vf);
> +}
> +
> +static struct erofs_vfops s3erofs_io_vfops = {
> + .pread = s3erofs_io_pread,
> + .read = s3erofs_io_read,
> + .close = s3erofs_io_close,
> +};
> +
> +static int s3erofs_get_object_size(struct s3erofs_vfile *s3vf)
> +{
> + struct s3erofs_curl_request req = { .method = "HEAD", };
> + struct erofs_s3 *s3 = s3vf->s3;
> + CURL *curl = s3->easy_curl;
> + long http_code = 0;
> + double content_length = 0;
> + int ret;
> +
> + ret = s3erofs_prepare_url(&req, s3->endpoint, s3vf->bucket,
> + s3vf->key, NULL, s3->url_style, s3->sig);
> + if (ret < 0)
> + return ret;
> +
> + ret = s3erofs_request_perform(s3, &req, NULL);
> + if (ret)
> + return ret;
> +
> + ret = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
> + if (ret != CURLE_OK) {
> + erofs_err("curl_easy_getinfo() failed: %s",
> + curl_easy_strerror(ret));
> + return -EIO;
> + }
> +
> + if (http_code != 200) {
> + erofs_err("HEAD request failed with HTTP code %ld", http_code);
> + return -EIO;
> + }
> +
> + ret = curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD,
> + &content_length);
> + if (ret != CURLE_OK)
> + return -EIO;
> + s3vf->size = (u64)content_length;
> + return 0;
> +}
> +
> +struct erofs_vfile *s3erofs_io_open(struct erofs_s3 *s3, const char *bucket,
> + const char *key)
> +{
> + struct s3erofs_vfile *s3vf;
> + int ret = -ENOMEM;
> +
> + s3vf = calloc(1, sizeof(*s3vf));
> + if (!s3vf)
> + return ERR_PTR(-ENOMEM);
> +
> + s3vf->vf = (struct erofs_vfile){.ops = &s3erofs_io_vfops};
> + s3vf->bucket = strdup(bucket);
> + if (!s3vf->bucket)
> + goto err_free;
> + s3vf->key = strdup(key);
> + if (!s3vf->key)
> + goto err_free;
> + s3vf->s3 = s3;
> +
> + ret = s3erofs_curl_easy_init(s3vf->s3);
> + if (ret)
> + goto err_free;
> +
> + /* Get object size via HEAD request */
> + ret = s3erofs_get_object_size(s3vf);
> + if (ret) {
> + erofs_err("failed to get S3 object size");
> + goto err_curl;
> + }
> +
> + erofs_dbg("S3 object (%s) size: %llu", s3vf->key, s3vf->size);
> + return &s3vf->vf;
> +
> +err_curl:
> + s3erofs_curl_easy_exit(s3);
> +err_free:
> + free(s3vf->key);
> + free(s3vf->bucket);
> + free(s3vf);
> + return ERR_PTR(ret);
> +}
> +
> +int s3erofs_parse_s3fs_passwd(const char *filepath, char *ak, char *sk)
> +{
> + char buf[S3_ACCESS_KEY_LEN + S3_SECRET_KEY_LEN + 3];
> + struct stat st;
> + int fd, n, ret;
> + char *colon;
> +
> + fd = open(filepath, O_RDONLY);
> + if (fd < 0) {
> + erofs_err("failed to open passwd_file %s", filepath);
> + return -errno;
> + }
> +
> + ret = fstat(fd, &st);
> + if (ret) {
> + ret = -errno;
> + goto err;
> + }
> +
> + if (!S_ISREG(st.st_mode)) {
> + erofs_err("%s is not a regular file", filepath);
> + ret = -EINVAL;
> + goto err;
> + }
> +
> + if ((st.st_mode & 077) != 0)
> + erofs_warn("passwd_file %s should not be accessible by group or others",
> + filepath);
> +
> + if (st.st_size >= sizeof(buf)) {
> + erofs_err("passwd_file %s is too large (size: %llu)", filepath,
> + st.st_size | 0ULL);
> + ret = -EINVAL;
> + goto err;
> + }
> +
> + n = read(fd, buf, st.st_size);
> + if (n < 0) {
> + ret = -errno;
> + goto err;
> + }
> + buf[n] = '\0';
> +
> + while (n > 0 && (buf[n - 1] == '\n' || buf[n - 1] == '\r'))
> + buf[--n] = '\0';
> +
> + colon = strchr(buf, ':');
> + if (!colon) {
> + ret = -EINVAL;
> + goto err;
> + }
> + *colon = '\0';
> +
> + if (strlen(buf) > S3_ACCESS_KEY_LEN ||
> + strlen(colon + 1) > S3_SECRET_KEY_LEN) {
> + ret = -EINVAL;
> + goto err;
> + }
> +
> + strcpy(ak, buf);
> + strcpy(sk, colon + 1);
> +
> +err:
> + close(fd);
> + return ret;
> +}
> +
> #ifdef TEST
> struct s3erofs_prepare_url_testcase {
> const char *name;
> @@ -1186,7 +1465,7 @@ struct s3erofs_prepare_url_testcase {
> static bool run_s3erofs_prepare_url_test(const struct s3erofs_prepare_url_testcase *tc,
> enum s3erofs_signature_version sig)
> {
> - struct s3erofs_curl_request req = {};
> + struct s3erofs_curl_request req = { .method = "GET", };
> int ret;
> const char *expected_canonical;
>
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 5006f76fa73b..5de5fbe0c961 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -634,73 +634,6 @@ static void mkfs_parse_tar_cfg(char *cfg)
> }
>
> #ifdef S3EROFS_ENABLED
> -static int mkfs_parse_s3_cfg_passwd(const char *filepath, char *ak, char *sk)
> -{
> - struct stat st;
> - int fd, n, ret;
> - char buf[S3_ACCESS_KEY_LEN + S3_SECRET_KEY_LEN + 3];
> - char *colon;
> -
> - fd = open(filepath, O_RDONLY);
> - if (fd < 0) {
> - erofs_err("failed to open passwd_file %s", filepath);
> - return -errno;
> - }
> -
> - ret = fstat(fd, &st);
> - if (ret) {
> - ret = -errno;
> - goto err;
> - }
> -
> - if (!S_ISREG(st.st_mode)) {
> - erofs_err("%s is not a regular file", filepath);
> - ret = -EINVAL;
> - goto err;
> - }
> -
> - if ((st.st_mode & 077) != 0)
> - erofs_warn("passwd_file %s should not be accessible by group or others",
> - filepath);
> -
> - if (st.st_size >= sizeof(buf)) {
> - erofs_err("passwd_file %s is too large (size: %llu)", filepath,
> - st.st_size | 0ULL);
> - ret = -EINVAL;
> - goto err;
> - }
> -
> - n = read(fd, buf, st.st_size);
> - if (n < 0) {
> - ret = -errno;
> - goto err;
> - }
> - buf[n] = '\0';
> -
> - while (n > 0 && (buf[n - 1] == '\n' || buf[n - 1] == '\r'))
> - buf[--n] = '\0';
> -
> - colon = strchr(buf, ':');
> - if (!colon) {
> - ret = -EINVAL;
> - goto err;
> - }
> - *colon = '\0';
> -
> - if (strlen(buf) > S3_ACCESS_KEY_LEN ||
> - strlen(colon + 1) > S3_SECRET_KEY_LEN) {
> - ret = -EINVAL;
> - goto err;
> - }
> -
> - strcpy(ak, buf);
> - strcpy(sk, colon + 1);
> -
> -err:
> - close(fd);
> - return ret;
> -}
> -
> static int mkfs_parse_s3_cfg(char *cfg_str)
> {
> char *p, *q, *opt;
> @@ -734,8 +667,8 @@ static int mkfs_parse_s3_cfg(char *cfg_str)
>
> if ((p = strstr(opt, "passwd_file="))) {
> p += sizeof("passwd_file=") - 1;
> - ret = mkfs_parse_s3_cfg_passwd(p, s3cfg.access_key,
> - s3cfg.secret_key);
> + ret = s3erofs_parse_s3fs_passwd(p, s3cfg.access_key,
> + s3cfg.secret_key);
> if (ret)
> return ret;
> } else if ((p = strstr(opt, "urlstyle="))) {
> diff --git a/mount/main.c b/mount/main.c
> index e09e58533ecc..bd7beb1fbb13 100644
> --- a/mount/main.c
> +++ b/mount/main.c
> @@ -22,6 +22,7 @@
> #ifdef EROFS_FANOTIFY_ENABLED
> #include "../lib/liberofs_fanotify.h"
> #endif
> +#include "../lib/liberofs_s3.h"
>
> #ifdef HAVE_LINUX_LOOP_H
> #include <linux/loop.h>
> @@ -88,13 +89,17 @@ static struct erofsmount_cfg {
> enum erofsmount_source_type {
> EROFSMOUNT_SOURCE_LOCAL,
> EROFSMOUNT_SOURCE_OCI,
> + EROFSMOUNT_SOURCE_S3_OBJECT,
> };
>
> static struct erofsmount_source {
> enum erofsmount_source_type type;
> union {
> - const char *device_path;
> struct ocierofs_config ocicfg;
> + struct {
> + const char *device_path;
> + struct erofs_s3 s3cfg;
> + };
> };
> } mountsrc;
>
> @@ -104,27 +109,36 @@ static void usage(int argc, char **argv)
> "Manage EROFS filesystem.\n"
> "\n"
> "General options:\n"
> - " -V, --version print the version number of mount.erofs and exit\n"
> - " -h, --help display this help and exit\n"
> - " -d <0-9> set output verbosity; 0=quiet, 9=verbose (default=%i)\n"
> - " -o options comma-separated list of mount options\n"
> - " -t type[.subtype] filesystem type (and optional subtype)\n"
> - " subtypes: fuse, local, nbd" EROFSMOUNT_FANOTIFY_HELP "\n"
> - " -u unmount the filesystem\n"
> - " --disconnect abort an existing NBD device forcibly\n"
> - " --reattach reattach to an existing NBD device\n"
> + " -V, --version print the version number of mount.erofs and exit\n"
> + " -h, --help display this help and exit\n"
> + " -d <0-9> set output verbosity; 0=quiet, 9=verbose (default=%i)\n"
> + " -o options comma-separated list of mount options\n"
> + " -t type[.subtype] filesystem type (and optional subtype)\n"
> + " subtypes: fuse, local, nbd" EROFSMOUNT_FANOTIFY_HELP "\n"
> + " -u unmount the filesystem\n"
> + " --disconnect abort an existing NBD device forcibly\n"
> + " --reattach reattach to an existing NBD device\n"
> #ifdef OCIEROFS_ENABLED
> "\n"
> "OCI-specific options (EXPERIMENTAL, with -o):\n"
> - " oci.blob=<digest> specify OCI blob digest (sha256:...)\n"
> - " oci.layer=<index> specify OCI layer index\n"
> - " oci.platform=<name> specify platform (default: linux/amd64)\n"
> - " oci.username=<user> username for authentication (optional)\n"
> - " oci.password=<pass> password for authentication (optional)\n"
> - " oci.tarindex=<path> path to tarball index file (optional)\n"
> - " oci.zinfo=<path> path to gzip zinfo file (optional)\n"
> - " oci.insecure use HTTP instead of HTTPS (optional)\n"
> + " oci.blob=<digest> specify OCI blob digest (sha256:...)\n"
> + " oci.layer=<index> specify OCI layer index\n"
> + " oci.platform=<name> specify platform (default: linux/amd64)\n"
> + " oci.username=<user> username for authentication (optional)\n"
> + " oci.password=<pass> password for authentication (optional)\n"
> + " oci.tarindex=<path> path to tarball index file (optional)\n"
> + " oci.zinfo=<path> path to gzip zinfo file (optional)\n"
> + " oci.insecure use HTTP instead of HTTPS (optional)\n"
> #endif
> +#ifdef S3EROFS_ENABLED
> + "\n"
> + "S3-specific options (EXPERIMENTAL, with -o):\n"
> + " s3.endpoint=<url> S3 endpoint URL (e.g., s3.amazonaws.com)\n"
> + " s3.passwd_file=<path> specify a s3fs-compatible password file\n"
> + " s3.region=<region> region code in which endpoint belongs to (required for sigv4)\n"
> + " s3.sig=<2,4> S3 API signature version (default: 2)\n"
> + " s3.urlstyle=<vhost|path> S3 API calling URL (default: vhost)\n"
> + #endif
> , argv[0], EROFS_WARN);
> }
>
> @@ -210,6 +224,85 @@ static int erofsmount_parse_oci_option(const char *option)
> }
> #endif
>
> +#ifdef S3EROFS_ENABLED
> +static int erofsmount_parse_s3_option(const char *option, struct erofs_s3 *s3cfg)
> +{
> + const char *p;
> + int ret;
> +
> + if ((p = strstr(option, "s3.endpoint=")) != NULL) {
> + p += sizeof("s3.endpoint=") - 1;
> + s3cfg->endpoint = strdup(p);
> + if (!s3cfg->endpoint)
> + return -ENOMEM;
> + } else if ((p = strstr(option, "s3.passwd_file=")) != NULL) {
> + p += sizeof("s3.passwd_file=") - 1;
> + ret = s3erofs_parse_s3fs_passwd(p, s3cfg->access_key,
> + s3cfg->secret_key);
> + if (ret)
> + return ret;
> + } else if ((p = strstr(option, "s3.region=")) != NULL) {
> + p += sizeof("s3.region=") - 1;
> + s3cfg->region = strdup(p);
> + if (!s3cfg->region)
> + return -ENOMEM;
> + } else if ((p = strstr(option, "s3.urlstyle=")) != NULL) {
> + p += sizeof("s3.urlstyle=") - 1;
> + if (!strcmp(p, "vhost"))
> + s3cfg->url_style = S3EROFS_URL_STYLE_VIRTUAL_HOST;
> + else if (!strcmp(p, "path"))
> + s3cfg->url_style = S3EROFS_URL_STYLE_PATH;
> + else {
> + erofs_err("invalid S3 URL style %s", p);
> + return -EINVAL;
> + }
> + } else if ((p = strstr(option, "s3.sig=")) != NULL) {
> + p += sizeof("s3.sig=") - 1;
> + if (!strcmp(p, "2"))
> + s3cfg->sig = S3EROFS_SIGNATURE_VERSION_2;
> + else if (!strcmp(p, "4"))
> + s3cfg->sig = S3EROFS_SIGNATURE_VERSION_4;
> + else {
> + erofs_err("invalid S3 signature version %s", p);
> + return -EINVAL;
> + }
> + } else {
> + return -EINVAL;
> + }
> + return 0;
> +}
> +
> +static int erofsmount_parse_s3_source(struct erofs_s3 *s3cfg, const char *source,
> + char **bucket, char **key)
> +{
> + const char *slash;
> +
> + if (!source || !*source)
> + return -EINVAL;
> +
> + slash = strchr(source, '/');
> + if (!slash) {
> + /* No slash: treat entire source as bucket, empty key */
> + *bucket = strdup(source);
> + *key = strdup("");
> + } else {
> + *bucket = strndup(source, slash - source);
> + *key = strdup(slash + 1);
> + }
> + if (!*bucket || !*key) {
> + free(*bucket);
> + free(*key);
> + return -ENOMEM;
> + }
> + return 0;
> +}
> +#else
> +static int erofsmount_parse_s3_option(const char *option, void *s3cfg)
> +{
> + return -EINVAL;
> +}
> +#endif
> +
> static long erofsmount_parse_flagopts(char *s, long flags, char **more)
> {
> static const struct {
> @@ -253,6 +346,33 @@ static long erofsmount_parse_flagopts(char *s, long flags, char **more)
> err = erofsmount_parse_oci_option(s);
> if (err < 0)
> return err;
> +#ifdef S3EROFS_ENABLED
> + } else if (strncmp(s, "s3.", 3) == 0) {
> + /* Initialize s3cfg here iff != EROFSMOUNT_SOURCE_S3_OBJECT */
> + if (mountsrc.type != EROFSMOUNT_SOURCE_S3_OBJECT) {
> + erofs_warn("EXPERIMENTAL S3 mount support in use, use at your own risk.");
> + mountsrc.type = EROFSMOUNT_SOURCE_S3_OBJECT;
> + mountsrc.s3cfg.url_style = S3EROFS_URL_STYLE_VIRTUAL_HOST;
> + mountsrc.s3cfg.sig = S3EROFS_SIGNATURE_VERSION_2;
> + mountsrc.s3cfg.access_key[0] = '\0';
> + mountsrc.s3cfg.secret_key[0] = '\0';
> + if (getenv("AWS_ACCESS_KEY_ID")) {
> + strncpy(mountsrc.s3cfg.access_key,
> + getenv("AWS_ACCESS_KEY_ID"),
> + S3_ACCESS_KEY_LEN);
> + mountsrc.s3cfg.access_key[S3_ACCESS_KEY_LEN] = '\0';
> + }
> + if (getenv("AWS_SECRET_ACCESS_KEY")) {
> + strncpy(mountsrc.s3cfg.secret_key,
> + getenv("AWS_SECRET_ACCESS_KEY"),
> + S3_SECRET_KEY_LEN);
> + mountsrc.s3cfg.secret_key[S3_SECRET_KEY_LEN] = '\0';
> + }
> + }
> + err = erofsmount_parse_s3_option(s, &mountsrc.s3cfg);
> + if (err < 0)
> + return err;
> +#endif
> } else {
> for (i = 0; i < ARRAY_SIZE(opts); ++i) {
> if (!strcasecmp(s, opts[i].name)) {
> @@ -635,8 +755,9 @@ err_out:
> }
>
> struct erofsmount_nbd_ctx {
> - struct erofs_vfile vd; /* virtual device */
> + struct erofs_vfile _vd; /* virtual device */
> struct erofs_vfile sk; /* socket file */
> + struct erofs_vfile *vd;
> };
>
> static void *erofsmount_nbd_loopfn(void *arg)
> @@ -666,7 +787,7 @@ static void *erofsmount_nbd_loopfn(void *arg)
> erofs_nbd_send_reply_header(ctx->sk.fd, rq.cookie, 0);
> pos = rq.from;
> do {
> - written = erofs_io_sendfile(&ctx->sk, &ctx->vd, &pos, rq.len);
> + written = erofs_io_sendfile(&ctx->sk, ctx->vd, &pos, rq.len);
> if (written == -EINTR) {
> err = written;
> goto out;
> @@ -680,49 +801,68 @@ static void *erofsmount_nbd_loopfn(void *arg)
> }
> }
> out:
> - erofs_io_close(&ctx->vd);
> + erofs_io_close(ctx->vd);
> erofs_io_close(&ctx->sk);
> return (void *)(uintptr_t)err;
> }
>
> static int erofsmount_startnbd(int nbdfd, struct erofsmount_source *source)
> {
> - struct erofsmount_nbd_ctx ctx = {};
> + struct erofsmount_nbd_ctx ctx = {.vd = &ctx._vd};
> uintptr_t retcode;
> pthread_t th;
> int err, err2;
>
> if (source->type == EROFSMOUNT_SOURCE_OCI) {
> if (source->ocicfg.tarindex_path || source->ocicfg.zinfo_path) {
> - err = erofsmount_tarindex_open(&ctx.vd, &source->ocicfg,
> + err = erofsmount_tarindex_open(ctx.vd, &source->ocicfg,
> source->ocicfg.tarindex_path,
> source->ocicfg.zinfo_path);
> if (err)
> goto out_closefd;
> } else {
> - err = ocierofs_io_open(&ctx.vd, &source->ocicfg);
> + err = ocierofs_io_open(ctx.vd, &source->ocicfg);
> if (err)
> goto out_closefd;
> }
> +#ifdef S3EROFS_ENABLED
> + } else if (source->type == EROFSMOUNT_SOURCE_S3_OBJECT) {
> + char *bucket = NULL, *key = NULL;
> + struct erofs_vfile *s3vf;
> +
> + err = erofsmount_parse_s3_source(&source->s3cfg, source->device_path,
> + &bucket, &key);
> + if (err)
> + goto out_closefd;
> +
> + s3vf = s3erofs_io_open(&source->s3cfg, bucket, key);
> + free(bucket);
> + free(key);
> + if (IS_ERR(s3vf)) {
> + err = PTR_ERR(s3vf);
> + goto out_closefd;
> + }
> + ctx.vd = s3vf;
> +#endif
> } else {
> err = open(source->device_path, O_RDONLY);
> if (err < 0) {
> err = -errno;
> goto out_closefd;
> }
> - ctx.vd.fd = err;
> + ctx._vd.fd = err;
> }
>
> err = erofs_nbd_connect(nbdfd, 9, EROFSMOUNT_NBD_DISK_SIZE);
> if (err < 0) {
> - erofs_io_close(&ctx.vd);
> + erofs_io_close(ctx.vd);
> goto out_closefd;
> }
> ctx.sk.fd = err;
>
> err = -pthread_create(&th, NULL, erofsmount_nbd_loopfn, &ctx);
> if (err) {
> - erofs_io_close(&ctx.vd);
> + erofs_io_close(ctx.vd);
> erofs_io_close(&ctx.sk);
> goto out_closefd;
> }
> @@ -840,7 +980,7 @@ static char *erofsmount_write_recovery_info(struct erofsmount_source *source)
>
> if (source->type == EROFSMOUNT_SOURCE_OCI)
> err = erofsmount_write_recovery_oci(f, source);
> - else
> + else if (source->type == EROFSMOUNT_SOURCE_LOCAL)
> err = erofsmount_write_recovery_local(f, source);
>
> fclose(f);
> @@ -996,12 +1136,12 @@ static int erofsmount_reattach_gzran_oci(struct erofsmount_nbd_ctx *ctx,
> if (err < 0)
> return -ENOMEM;
>
> - err = erofsmount_reattach_oci(&ctx->vd, "OCI_NATIVE_BLOB", oci_source);
> + err = erofsmount_reattach_oci(ctx->vd, "OCI_NATIVE_BLOB", oci_source);
> free(oci_source);
> if (err)
> return err;
>
> - temp_vd = ctx->vd;
> + temp_vd = *ctx->vd;
> oci_cfg.image_ref = strdup(source);
> if (!oci_cfg.image_ref) {
> erofs_io_close(&temp_vd);
> @@ -1013,7 +1153,7 @@ static int erofsmount_reattach_gzran_oci(struct erofsmount_nbd_ctx *ctx,
> if (token_count > 4 && tokens[4] && *tokens[4])
> zinfo_path = tokens[4];
>
> - err = erofsmount_tarindex_open(&ctx->vd, &oci_cfg,
> + err = erofsmount_tarindex_open(ctx->vd, &oci_cfg,
> meta_path, zinfo_path);
> free(oci_cfg.image_ref);
> erofs_io_close(&temp_vd);
> @@ -1056,7 +1196,7 @@ static int erofsmount_startnbd_nl(pid_t *pid, struct erofsmount_source *source)
> return -errno;
>
> if ((*pid = fork()) == 0) {
> - struct erofsmount_nbd_ctx ctx = {};
> + struct erofsmount_nbd_ctx ctx = {.vd = &ctx._vd};
> char *recp;
>
> /* Otherwise, NBD disconnect sends SIGPIPE, skipping cleanup */
> @@ -1065,25 +1205,42 @@ static int erofsmount_startnbd_nl(pid_t *pid, struct erofsmount_source *source)
>
> if (source->type == EROFSMOUNT_SOURCE_OCI) {
> if (source->ocicfg.tarindex_path || source->ocicfg.zinfo_path) {
> - err = erofsmount_tarindex_open(&ctx.vd, &source->ocicfg,
> + err = erofsmount_tarindex_open(ctx.vd, &source->ocicfg,
> source->ocicfg.tarindex_path,
> source->ocicfg.zinfo_path);
> if (err)
> exit(EXIT_FAILURE);
> } else {
> - err = ocierofs_io_open(&ctx.vd, &source->ocicfg);
> + err = ocierofs_io_open(ctx.vd, &source->ocicfg);
> if (err)
> exit(EXIT_FAILURE);
> }
> +#ifdef S3EROFS_ENABLED
> + } else if (source->type == EROFSMOUNT_SOURCE_S3_OBJECT) {
> + char *bucket = NULL, *key = NULL;
> + struct erofs_vfile *s3vf;
> +
> + err = erofsmount_parse_s3_source(&source->s3cfg, source->device_path,
> + &bucket, &key);
> + if (err)
> + exit(EXIT_FAILURE);
> +
> + s3vf = s3erofs_io_open(&source->s3cfg, bucket, key);
> + free(bucket);
> + free(key);
> + if (IS_ERR(s3vf))
> + exit(EXIT_FAILURE);
> + ctx.vd = s3vf;
> +#endif
> } else {
> err = open(source->device_path, O_RDONLY);
> if (err < 0)
> exit(EXIT_FAILURE);
> - ctx.vd.fd = err;
> + ctx._vd.fd = err;
> }
> recp = erofsmount_write_recovery_info(source);
> if (IS_ERR(recp)) {
> - erofs_io_close(&ctx.vd);
> + erofs_io_close(ctx.vd);
> exit(EXIT_FAILURE);
> }
>
> @@ -1106,7 +1263,7 @@ static int erofsmount_startnbd_nl(pid_t *pid, struct erofsmount_source *source)
> }
> }
> }
> - erofs_io_close(&ctx.vd);
> + erofs_io_close(ctx.vd);
> out_fork:
> (void)unlink(recp);
> free(recp);
> @@ -1186,13 +1343,13 @@ static int erofsmount_reattach(const char *target)
In erofsmount_reattach() we should:
`struct erofsmount_nbd_ctx ctx = {};` => `struct erofsmount_nbd_ctx
ctx = {.vd = &ctx._vd};`
otherwise uninitialized ctx.vd leads to segfault.
Thanks,
Yifan Zhao
> err = -errno;
> goto err_line;
> }
> - ctx.vd.fd = err;
> + ctx.vd->fd = err;
> } else if (!strcmp(line, "TARINDEX_OCI_BLOB")) {
> err = erofsmount_reattach_gzran_oci(&ctx, source);
> if (err)
> goto err_line;
> } else if (!strcmp(line, "OCI_LAYER") || !strcmp(line, "OCI_NATIVE_BLOB")) {
> - err = erofsmount_reattach_oci(&ctx.vd, line, source);
> + err = erofsmount_reattach_oci(ctx.vd, line, source);
> if (err)
> goto err_line;
> } else {
> @@ -1214,7 +1371,7 @@ static int erofsmount_reattach(const char *target)
> erofs_io_close(&ctx.sk);
> err = 0;
> }
> - erofs_io_close(&ctx.vd);
> + erofs_io_close(ctx.vd);
> err_line:
> free(line);
> err_identifier:
More information about the Linux-erofs
mailing list