[PATCH v4 3/5] erofs-utils: mkfs: add --worker=# parameter

[Gao Xiang]

On 2024/2/29 00:16, Yifan Zhao wrote:
> This patch introduces a --worker=# parameter for the incoming
> multi-threaded compression support. It also introduces a segment size
> used in multi-threaded compression, which has the default value 16MB
> and cannot be modified.
> 
> It also introduces a concept called `segment size` to split large files
> for multi-threading, which has the default value 16MB for now.
> 
> Signed-off-by: Yifan Zhao <zhaoyifan at sjtu.edu.cn>
> ---
>   include/erofs/config.h |  4 ++++
>   lib/config.c           |  4 ++++
>   mkfs/main.c            | 38 ++++++++++++++++++++++++++++++++++++++
>   3 files changed, 46 insertions(+)
> 
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index 73e3ac2..d2f91ff 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -75,6 +75,10 @@ struct erofs_configure {
>   	char c_force_chunkformat;
>   	/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
>   	int c_inline_xattr_tolerance;
> +#ifdef EROFS_MT_ENABLED
> +	u64 c_segment_size;
> +	u32 c_mt_workers;
> +#endif
>   
>   	u32 c_pclusterblks_max, c_pclusterblks_def, c_pclusterblks_packed;
>   	u32 c_max_decompressed_extent_bytes;
> diff --git a/lib/config.c b/lib/config.c
> index 947a183..2530274 100644
> --- a/lib/config.c
> +++ b/lib/config.c
> @@ -38,6 +38,10 @@ void erofs_init_configure(void)
>   	cfg.c_pclusterblks_max = 1;
>   	cfg.c_pclusterblks_def = 1;
>   	cfg.c_max_decompressed_extent_bytes = -1;
> +#ifdef EROFS_MT_ENABLED
> +	cfg.c_segment_size = 16ULL * 1024 * 1024;
> +	cfg.c_mt_workers = 1;
> +#endif
>   
>   	erofs_stdout_tty = isatty(STDOUT_FILENO);
>   }
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 258c1ce..ce9c28b 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -74,6 +74,9 @@ static struct option long_options[] = {
>   	{"ungzip", optional_argument, NULL, 517},
>   #endif
>   	{"offset", required_argument, NULL, 518},
> +#ifdef EROFS_MT_ENABLED
> +	{"workers", required_argument, NULL, 519},
> +#endif
>   	{0, 0, 0, 0},
>   };
>   
> @@ -179,6 +182,9 @@ static void usage(int argc, char **argv)
>   		" --product-out=X       X=product_out directory\n"
>   		" --fs-config-file=X    X=fs_config file\n"
>   		" --block-list-file=X   X=block_list file\n"
> +#endif
> +#ifdef EROFS_MT_ENABLED
> +		" --workers=#            set the number of worker threads to # (default=1)\n"
>   #endif
>   		);
>   }
> @@ -408,6 +414,13 @@ static void erofs_rebuild_cleanup(void)
>   	rebuild_src_count = 0;
>   }
>   
> +#ifdef EROFS_MT_ENABLED
> +static u32 mkfs_max_worker_num() {

static unsigned int erofs_mkfs_max_worker_num()
{
	return erofs_get_available_processors() ? : 16;
}

> +	u32 ncpu = erofs_get_available_processors();
> +	return ncpu ? ncpu : 16;
> +}
> +#endif
> +
>   static int mkfs_parse_options_cfg(int argc, char *argv[])
>   {
>   	char *endptr;
> @@ -650,6 +663,21 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
>   				return -EINVAL;
>   			}
>   			break;
> +#ifdef EROFS_MT_ENABLED
> +		case 519:
> +			cfg.c_mt_workers = strtoul(optarg, &endptr, 0);
> +			if (errno || *endptr != '\0') {
> +				erofs_err("invalid worker number %s", optarg);
> +				return -EINVAL;
> +			}
> +			if (cfg.c_mt_workers > mkfs_max_worker_num()) {
> +				erofs_warn(
> +					"worker number %s is too large, setting to %ud",
> +					optarg, mkfs_max_worker_num());
let's not break erofs_{err,warn,...} print line, it means:

				cfg.c_mt_workers = mkfs_max_worker_num();
				erofs_warn("worker number %s is too large, reseting to %ud",
					   optarg, cfg.c_mt_workers);

> +				cfg.c_mt_workers = mkfs_max_worker_num();
> +			}
> +			break;
> +#endif
>   		case 'V':
>   			version();
>   			exit(0);
> @@ -803,6 +831,16 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
>   		}
>   		cfg.c_pclusterblks_packed = pclustersize_packed >> sbi.blkszbits;
>   	}
> +
> +#ifdef EROFS_MT_ENABLED
> +	if (cfg.c_mt_workers > 1 &&
> +	    (cfg.c_dedupe || cfg.c_fragments || cfg.c_ztailpacking)) {
> +		cfg.c_mt_workers = 1;
> +		erofs_warn("Please note that dedupe/fragments/ztailpacking"
> +			   "is NOT supported in multi-threaded mode now, using worker=1.");
> +	}
> +#endif

This part would be better to go with the next patch.

Thanks,
Gao Xiang