[PATCH v2] erofs-utils: add OCI registry support

[Gao Xiang]

On 2025/8/22 10:45, ChengyuZhu6 wrote:
> From: Chengyu Zhu <hudson at cyzhu.com>
> 
> This patch adds support for building EROFS filesystems from
> OCI-compliant container registries, enabling users to create EROFS
> images directly from container images stored in registries like
> Docker Hub, Quay.io, etc.
> 
> The implementation includes:
> - OCI remote backend with registry authentication support
> - Manifest parsing for Docker v2 and OCI v1 formats
> - Layer extraction and tar processing integration
> - Multi-platform image selection capability
> - Both anonymous and authenticated registry access
> - Comprehensive build system integration
> 
> New mkfs.erofs option: --oci=registry/repo:tag[,options]

Could you just write down the full command line? e,g.

mkfs.erofs --oci=registry/repo:tag[,options] <IMAGE> <?>

what's the meaning of <?>? since users already pass in
  registry/repo:tag and layer=N

> 
> Supported options:
> - platform=os/arch (default: linux/amd64)
> - layer=N (extract specific layer, default: all layers)
> - anonymous (use anonymous access)
> - username/password (basic authentication)
> 
> Signed-off-by: Changzhi Xie <sa.z at qq.com>
> Signed-off-by: Chengyu Zhu <hudson at cyzhu.com>

..

> +
> +#endif /* __EROFS_OCI_H */
> diff --git a/lib/remotes/oci.c b/lib/remotes/oci.c
> new file mode 100644
> index 0000000..0c14f90
> --- /dev/null
> +++ b/lib/remotes/oci.c
> @@ -0,0 +1,826 @@
> +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
> +/*
> + * Copyright (C) 2025 Tencent, Inc.
> + *             http://www.tencent.com/
> + */
> +#define _GNU_SOURCE
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +#include <sys/types.h>
> +#include <sys/stat.h>
> +#include <errno.h>
> +#include <curl/curl.h>
> +#include <json-c/json.h>
> +#include "erofs/internal.h"
> +#include "erofs/print.h"
> +#include "erofs/inode.h"
> +#include "erofs/blobchunk.h"
> +#include "erofs/diskbuf.h"
> +#include "erofs/rebuild.h"
> +#include "erofs/tar.h"
> +#include "liberofs_oci.h"
> +
> +#define OCI_AUTH_HEADER_MAX_LEN 1024
> +#define OCI_TEMP_FILENAME_MAX_LEN 256
> +
> +#define DOCKER_MEDIATYPE_MANIFEST_V2 "application/vnd.docker.distribution.manifest.v2+json"
> +#define DOCKER_MEDIATYPE_MANIFEST_V1 "application/vnd.docker.distribution.manifest.v1+json"
> +#define DOCKER_MEDIATYPE_MANIFEST_LIST "application/vnd.docker.distribution.manifest.list.v2+json"
> +#define OCI_MEDIATYPE_MANIFEST "application/vnd.oci.image.manifest.v1+json"
> +#define OCI_MEDIATYPE_INDEX "application/vnd.oci.image.index.v1+json"
> +
> +#define DOCKER_REGISTRY "docker.io"
> +#define DOCKER_API_REGISTRY "registry-1.docker.io"
> +#define QUAY_REGISTRY "quay.io"
> +
> +struct erofs_oci_request {
> +	char *url;
> +	struct curl_slist *headers;
> +};
> +
> +struct erofs_oci_response {
> +	char *data;
> +	size_t size;
> +	long http_code;
> +};
> +
> +struct oci_stream {

struct erofs_oci_stream {

It's prefered to use `erofs_` prefix even the structure
is internal-only or static.

> +	struct erofs_tarfile tarfile;
> +	FILE *temp_file;
> +	char temp_filename[OCI_TEMP_FILENAME_MAX_LEN];

I think you could just leave a fd for this?

and use
erofs_tmpfile().

> +	int layer_index;
> +};
> +
> +/* Callback for writing response data to memory */
> +static size_t oci_write_callback(void *contents, size_t size, size_t nmemb, void *userp)

same here.
ocierofs_write_callback

> +{
> +	size_t realsize = size * nmemb;
> +	struct erofs_oci_response *resp = userp;
> +	char *ptr;
> +
> +	if (!resp || !contents)
> +		return 0;
> +
> +	ptr = realloc(resp->data, resp->size + realsize + 1);
> +	if (!ptr) {
> +		erofs_err("failed to allocate memory for response data");
> +		return 0;
> +	}
> +
> +	resp->data = ptr;
> +	memcpy(&resp->data[resp->size], contents, realsize);
> +	resp->size += realsize;
> +	resp->data[resp->size] = '\0';
> +	return realsize;
> +}
> +
> +/* Callback for writing layer data to file */
> +static size_t oci_layer_write_callback(void *contents, size_t size, size_t nmemb, void *userp)

ocierofs_

> +{
> +	struct oci_stream *stream = userp;
> +	size_t realsize = size * nmemb;
> +
> +	if (!stream->temp_file)
> +		return 0;
> +
> +	if (fwrite(contents, 1, realsize, stream->temp_file) != realsize) {
> +		erofs_err("failed to write layer data for layer %d", stream->layer_index);
> +		return 0;
> +	}
> +
> +	return realsize;
> +}
> +
> +static int oci_curl_setup_common_options(CURL *curl)


ocierofs_

> +{
> +	if (!curl)
> +		return -EINVAL;

Since it's an internal helper, I think it's unneeded to
check the invalid argument.

> +
> +	curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
> +	curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 30L);
> +	curl_easy_setopt(curl, CURLOPT_TIMEOUT, 120L);
> +	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
> +	curl_easy_setopt(curl, CURLOPT_USERAGENT, "ocierofs/" PACKAGE_VERSION);
> +
> +	return 0;
> +}
> +
> +static int oci_curl_setup_basic_auth(CURL *curl, const char *username, const char *password)

ocierofs_

> +{
> +	char *userpwd = NULL;
> +
> +	if (!curl || !username || !password)
> +		return -EINVAL;

Same here too.

> +
> +	if (asprintf(&userpwd, "%s:%s", username, password) == -1)
> +		return -ENOMEM;
> +
> +	curl_easy_setopt(curl, CURLOPT_USERPWD, userpwd);
> +	curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
> +
> +	free(userpwd);
> +	return 0;
> +}
> +
> +static int oci_curl_clear_auth(CURL *curl)
> +{
> +	if (!curl)
> +		return -EINVAL;

Same here too.

> +
> +	curl_easy_setopt(curl, CURLOPT_USERPWD, NULL);
> +	curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_NONE);
> +
> +	return 0;
> +}
> +
> +static int oci_curl_setup_request(CURL *curl, const char *url, size_t (*write_func)(void *, size_t, size_t, void *),
> +                                  void *write_data, struct curl_slist *headers)
> +{
> +	if (!curl || !url || !write_func)
> +		return -EINVAL;

Same here too.

> +
> +	curl_easy_setopt(curl, CURLOPT_URL, url);
> +	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_func);
> +	curl_easy_setopt(curl, CURLOPT_WRITEDATA, write_data);
> +
> +	if (headers)
> +		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
> +
> +	return 0;
> +}
> +
> +static int oci_request_perform(struct erofs_oci *oci, struct erofs_oci_request *req,
> +							   struct erofs_oci_response *resp)
> +{
> +	CURLcode res;
> +	int ret;
> +
> +	ret = oci_curl_setup_request(oci->curl, req->url, oci_write_callback, resp, req->headers);
> +	if (ret)
> +		return ret;
> +
> +	res = curl_easy_perform(oci->curl);
> +	if (res != CURLE_OK) {
> +		erofs_err("curl request failed: %s", curl_easy_strerror(res));
> +		return -EIO;
> +	}
> +
> +	res = curl_easy_getinfo(oci->curl, CURLINFO_RESPONSE_CODE, &resp->http_code);
> +	if (res != CURLE_OK) {
> +		erofs_err("failed to get HTTP response code: %s", curl_easy_strerror(res));
> +		return -EIO;
> +	}
> +
> +	if (resp->http_code < 200 || resp->http_code >= 300) {
> +		erofs_err("HTTP request failed with code %ld", resp->http_code);
> +		return -EIO;
> +	}
> +
> +	return 0;
> +}
> +
> +static char *oci_get_auth_token(struct erofs_oci *oci, const char *registry,
> +								const char *repository, const char *username,
> +								const char *password)

It seems your tab style is incorrect: one tab should be 8 spaces.

> +{


...

> +
> +	if (!resp.data)
> +	{

brace should follow `if (!resp.data)`

	if (!resp.data) {

erofs-utils follows linux kernel style: K&R C style.

> +		erofs_err("empty response from auth server");
> +		ret = -EINVAL;
> +		goto out_url;
> +	}
> +
> +	root = json_tokener_parse(resp.data);
> +	if (!root)
> +	{

here.

> +		erofs_err("failed to parse auth response");
> +		ret = -EINVAL;
> +		goto out_url;
> +	}
> +
> +	if (!json_object_object_get_ex(root, "token", &token_obj))
> +	{

here.

There are still many broken braces below, I think let's
just fix the coding style first.

> +		erofs_err("no token found in auth response");
> +		ret = -EINVAL;
> +		goto out_json;
> +	}
> +
> +	token = json_object_get_string(token_obj);
> +	if (!token)
> +	{
> +		erofs_err("invalid token in auth response");
> +		ret = -EINVAL;
> +		goto out_json;
> +	}
> +
> +	if (asprintf(&auth_header, "Authorization: Bearer %s", token) == -1)
> +	{
> +		ret = -ENOMEM;
> +		goto out_json;
> +	}
> +
> +out_json:
> +	json_object_put(root);
> +out_url:
> +	free(req.url);
> +	free(resp.data);
> +	return ret ? ERR_PTR(ret) : auth_header;
> +}
> +
Thanks,
Gao Xiang