[PATCH 3/3] erofs-utils: mkfs: add LZMA algorithm support
Gao Xiang
xiang at kernel.org
Sat Oct 30 13:01:18 AEDT 2021
This patch adds LZMA compression algorithm support to erofs-utils
compression framework with upstream liblzma.
Cc: Lasse Collin <lasse.collin at tukaani.org>
Signed-off-by: Gao Xiang <xiang at kernel.org>
---
include/erofs/config.h | 1 +
include/erofs_fs.h | 6 +++
lib/Makefile.am | 3 ++
lib/compress.c | 37 ++++++++++++--
lib/compressor.c | 3 ++
lib/compressor.h | 1 +
lib/compressor_liblzma.c | 105 +++++++++++++++++++++++++++++++++++++++
mkfs/main.c | 1 -
8 files changed, 152 insertions(+), 5 deletions(-)
create mode 100644 lib/compressor_liblzma.c
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 574dd52be12d..a18c88301279 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -60,6 +60,7 @@ struct erofs_configure {
u32 c_pclusterblks_max, c_pclusterblks_def;
u32 c_max_decompressed_extent_bytes;
+ u32 c_dict_size;
u64 c_unix_timestamp;
u32 c_uid, c_gid;
#ifdef WITH_ANDROID
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index 86ad6f5fd86c..4291970753a8 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -261,6 +261,12 @@ struct z_erofs_lz4_cfgs {
u8 reserved[10];
} __packed;
+/* 14 bytes (+ length field = 16 bytes) */
+struct z_erofs_lzma_cfgs {
+ __le32 dict_size;
+ __le16 format;
+ u8 reserved[8];
+} __packed;
#define Z_EROFS_LZMA_MAX_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE)
/*
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 370de844146f..58ad192c51b3 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -31,4 +31,7 @@ if ENABLE_LZ4HC
liberofs_la_SOURCES += compressor_lz4hc.c
endif
endif
+if ENABLE_LIBLZMA
liberofs_la_CFLAGS += ${liblzma_CFLAGS}
+liberofs_la_SOURCES += compressor_liblzma.c
+endif
diff --git a/lib/compress.c b/lib/compress.c
index 6ca5bedaf596..98be7a26383b 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -588,6 +588,8 @@ static int erofs_get_compress_algorithm_id(const char *name)
{
if (!strcmp(name, "lz4") || !strcmp(name, "lz4hc"))
return Z_EROFS_COMPRESSION_LZ4;
+ if (!strcmp(name, "lzma"))
+ return Z_EROFS_COMPRESSION_LZMA;
return -ENOTSUP;
}
@@ -619,6 +621,29 @@ int z_erofs_build_compr_cfgs(struct erofs_buffer_head *sb_bh)
sizeof(lz4alg));
bh->op = &erofs_drop_directly_bhops;
}
+#ifdef HAVE_LIBLZMA
+ if (sbi.available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZMA)) {
+ struct {
+ __le16 size;
+ struct z_erofs_lzma_cfgs lzma;
+ } __packed lzmaalg = {
+ .size = cpu_to_le16(sizeof(struct z_erofs_lzma_cfgs)),
+ .lzma = {
+ .dict_size = cpu_to_le32(cfg.c_dict_size),
+ }
+ };
+
+ bh = erofs_battach(bh, META, sizeof(lzmaalg));
+ if (IS_ERR(bh)) {
+ DBG_BUGON(1);
+ return PTR_ERR(bh);
+ }
+ erofs_mapbh(bh->block);
+ ret = dev_write(&lzmaalg, erofs_btell(bh, false),
+ sizeof(lzmaalg));
+ bh->op = &erofs_drop_directly_bhops;
+ }
+#endif
return ret;
}
@@ -632,17 +657,18 @@ int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)
return ret;
/*
- * if primary algorithm is not lz4* (e.g. compression off),
- * clear LZ4_0PADDING feature for old kernel compatibility.
+ * if primary algorithm is empty (e.g. compression off),
+ * clear 0PADDING feature for old kernel compatibility.
*/
if (!cfg.c_compr_alg_master ||
- strncmp(cfg.c_compr_alg_master, "lz4", 3))
+ (cfg.c_legacy_compress && !strcmp(cfg.c_compr_alg_master, "lz4")))
erofs_sb_clear_lz4_0padding();
if (!cfg.c_compr_alg_master)
return 0;
- ret = erofs_compressor_setlevel(&compresshandle, cfg.c_compr_level_master);
+ ret = erofs_compressor_setlevel(&compresshandle,
+ cfg.c_compr_level_master);
if (ret)
return ret;
@@ -668,6 +694,9 @@ int z_erofs_compress_init(struct erofs_buffer_head *sb_bh)
erofs_warn("EXPERIMENTAL big pcluster feature in use. Use at your own risk!");
}
+ if (ret != Z_EROFS_COMPRESSION_LZ4)
+ erofs_sb_set_compr_cfgs();
+
if (erofs_sb_has_compr_cfgs()) {
sbi.available_compr_algs |= 1 << ret;
return z_erofs_build_compr_cfgs(sb_bh);
diff --git a/lib/compressor.c b/lib/compressor.c
index 89c1be10dd0c..ad12cdf2ceed 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -17,6 +17,9 @@ static struct erofs_compressor *compressors[] = {
#endif
&erofs_compressor_lz4,
#endif
+#if HAVE_LIBLZMA
+ &erofs_compressor_lzma,
+#endif
};
int erofs_compress_destsize(struct erofs_compress *c,
diff --git a/lib/compressor.h b/lib/compressor.h
index d1b43c87291f..aa85ae0bdc2f 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -43,6 +43,7 @@ struct erofs_compress {
/* list of compression algorithms */
extern struct erofs_compressor erofs_compressor_lz4;
extern struct erofs_compressor erofs_compressor_lz4hc;
+extern struct erofs_compressor erofs_compressor_lzma;
int erofs_compress_destsize(struct erofs_compress *c,
void *src, unsigned int *srcsize,
diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
new file mode 100644
index 000000000000..e9bfcc556c54
--- /dev/null
+++ b/lib/compressor_liblzma.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/compressor_liblzma.c
+ *
+ * Copyright (C) 2021 Gao Xiang <xiang at kernel.org>
+ */
+#include <stdlib.h>
+#include <lzma.h>
+#include "erofs/config.h"
+#include "erofs/print.h"
+#include "erofs/internal.h"
+#include "compressor.h"
+
+struct erofs_liblzma_context {
+ lzma_options_lzma opt;
+ lzma_stream strm;
+};
+
+static int erofs_liblzma_compress_destsize(struct erofs_compress *c,
+ void *src, unsigned int *srcsize,
+ void *dst, unsigned int dstsize)
+{
+ struct erofs_liblzma_context *ctx = c->private_data;
+ lzma_stream *strm = &ctx->strm;
+
+ lzma_ret ret = lzma_microlzma_encoder(strm, &ctx->opt);
+ if (ret != LZMA_OK)
+ return -EFAULT;
+
+ strm->next_in = src;
+ strm->avail_in = *srcsize;
+ strm->next_out = dst;
+ strm->avail_out = dstsize;
+
+ ret = lzma_code(strm, LZMA_FINISH);
+ if (ret != LZMA_STREAM_END)
+ return -EBADMSG;
+
+ *srcsize = strm->total_in;
+ return strm->total_out;
+}
+
+static int erofs_compressor_liblzma_exit(struct erofs_compress *c)
+{
+ struct erofs_liblzma_context *ctx = c->private_data;
+
+ if (!ctx)
+ return -EINVAL;
+
+ lzma_end(&ctx->strm);
+ free(ctx);
+ return 0;
+}
+
+static int erofs_compressor_liblzma_setlevel(struct erofs_compress *c,
+ int compression_level)
+{
+ struct erofs_liblzma_context *ctx = c->private_data;
+
+ if (compression_level < 0)
+ compression_level = LZMA_PRESET_DEFAULT;
+
+ if (lzma_lzma_preset(&ctx->opt, compression_level))
+ return -EINVAL;
+
+ /* XXX: temporary hack */
+ if (cfg.c_dict_size) {
+ if (cfg.c_dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE) {
+ erofs_err("dict size %u is too large", cfg.c_dict_size);
+ return -EINVAL;
+ }
+ ctx->opt.dict_size = cfg.c_dict_size;
+ } else {
+ if (ctx->opt.dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE)
+ ctx->opt.dict_size = Z_EROFS_LZMA_MAX_DICT_SIZE;
+ cfg.c_dict_size = ctx->opt.dict_size;
+ }
+ c->compression_level = compression_level;
+ return 0;
+}
+
+static int erofs_compressor_liblzma_init(struct erofs_compress *c)
+{
+ struct erofs_liblzma_context *ctx;
+
+ c->alg = &erofs_compressor_lzma;
+ ctx = malloc(sizeof(*ctx));
+ if (!ctx)
+ return -ENOMEM;
+ ctx->strm = (lzma_stream)LZMA_STREAM_INIT;
+ c->private_data = ctx;
+ erofs_warn("EXPERIMENTAL MicroLZMA feature in use. Use at your own risk!");
+ erofs_warn("Note that it may take more time since the compressor is still single-threaded for now.");
+ return 0;
+}
+
+struct erofs_compressor erofs_compressor_lzma = {
+ .name = "lzma",
+ .default_level = LZMA_PRESET_DEFAULT,
+ .best_level = LZMA_PRESET_EXTREME,
+ .init = erofs_compressor_liblzma_init,
+ .exit = erofs_compressor_liblzma_exit,
+ .setlevel = erofs_compressor_liblzma_setlevel,
+ .compress_destsize = erofs_liblzma_compress_destsize,
+};
diff --git a/mkfs/main.c b/mkfs/main.c
index 055d077988e9..028cf5a8911c 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -147,7 +147,6 @@ static int parse_extended_opts(const char *opts)
return -EINVAL;
/* disable compacted indexes and 0padding */
cfg.c_legacy_compress = true;
- erofs_sb_clear_lz4_0padding();
}
if (MATCH_EXTENTED_OPT("force-inode-compact", token, keylen)) {
--
2.20.1
More information about the Linux-erofs
mailing list