[PATCH v1] erofs-utils: mkfs: Add 'no-deduplication' option
Friendy Su
friendy.su at sony.com
Thu Oct 23 19:01:42 AEDT 2025
Add an option to disable dedupliation at format.
This option is mandatory if mount erofs with DAX.
Deduplicated chunks are shared among multiple files or different parts of
one file. Kernel DAX map got wrong when map them. The following
kernel backtrace is printed.
[ 2.031496] WARNING: CPU: 0 PID: 1 at fs/dax.c:460 dax_insert_entry+0x36e/0x380
[ 2.031978] Modules linked in:
[ 2.032173] CPU: 0 UID: 0 PID: 1 Comm: init Not tainted 6.17.0-rc2+ #111 PREEMPT(voluntary)
[ 2.032688] Hardware name: QEMU Ubuntu 24.04 PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
[ 2.033291] RIP: 0010:dax_insert_entry+0x36e/0x380
[ 2.033591] Code: 59 fe ff ff 48 8b 30 b9 09 00 00 00 83 e6 40 0f 85 70 ff ff ff e9 77 ff ff ff 31 f6 90 0f 0b 90 85 f6 75 ae e9 34 fe ff
ff 90 <0f> 0b 90 e9 02 fe ff ff be 09 00 00 00 eb e3 0f 1f 00 90 90 90 90
[ 2.034654] RSP: 0000:ffffb93fc0013b88 EFLAGS: 00010086
[ 2.034948] RAX: ffffe124441dc140 RBX: ffffb93fc0013c78 RCX: 0000000000000000
[ 2.035339] RDX: 00007f310337c000 RSI: 0000000000000000 RDI: ffffe124441dc140
[ 2.035730] RBP: 00000000020ee0a1 R08: 0000000001077050 R09: 0000000000000000
[ 2.036120] R10: ffffb93fc0013cd8 R11: 0000000000001000 R12: 0000000000000011
[ 2.036513] R13: 0000000000000000 R14: fffffffffffff000 R15: 00000000020ee0a1
[ 2.036912] FS: 00007f31026ad940(0000) GS:ffff90436812c000(0000) knlGS:0000000000000000
[ 2.037352] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 2.037669] CR2: 00007f31033c2216 CR3: 0000000001d17006 CR4: 0000000000770ef0
[ 2.038062] PKRU: 55555554
[ 2.038216] Call Trace:
[ 2.038363] <TASK>
[ 2.038486] dax_fault_iter+0x286/0x6a0
[ 2.038704] dax_iomap_pte_fault+0x17f/0x370
[ 2.038950] __do_fault+0x30/0xc0
[ 2.039153] __handle_mm_fault+0x90a/0x15a0
[ 2.039391] handle_mm_fault+0xde/0x240
[ 2.039607] do_user_addr_fault+0x166/0x640
[ 2.039853] exc_page_fault+0x74/0x170
[ 2.040087] asm_exc_page_fault+0x26/0x30
[ 2.040319] RIP: 0033:0x7f31039389bd
[ 2.040519] Code: 08 48 8b 85 68 ff ff ff 48 8b bd 60 ff ff ff 48 8b b5 58 ff ff ff 4c 89 f2 48 03 33 45 89 f7 48 c1 ea 20 48 89 b5 70 ff
ff ff <0f> b7 04 50 48 8d 14 52 4c 8d 24 d7 25 ff 7f 00 00 4c 89 65 80 48
Signed-off-by: Friendy Su <friendy.su at sony.com>
Reviewed-by: Yuezhang Mo <Yuezhang.Mo at sony.com>
Reviewed-by: Daniel Palmer <daniel.palmer at sony.com>
---
include/erofs/internal.h | 1 +
lib/blobchunk.c | 29 +++++++++++++++++------------
man/mkfs.erofs.1 | 6 ++++++
mkfs/main.c | 8 ++++++++
4 files changed, 32 insertions(+), 12 deletions(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 6106501..3b404af 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -156,6 +156,7 @@ struct erofs_sb_info {
struct erofs_buffer_head *bh_devt;
bool useqpl;
bool sb_valid;
+ bool no_deduplication;
};
/* make sure that any user of the erofs headers has atleast 64bit off_t type */
diff --git a/lib/blobchunk.c b/lib/blobchunk.c
index a5945f8..c1c93ba 100644
--- a/lib/blobchunk.c
+++ b/lib/blobchunk.c
@@ -64,19 +64,24 @@ static struct erofs_blobchunk *erofs_blob_getchunk(struct erofs_sb_info *sbi,
erofs_sha256(buf, chunksize, sha256);
hash = memhash(sha256, sizeof(sha256));
- chunk = hashmap_get_from_hash(&blob_hashmap, hash, sha256);
- if (chunk) {
- DBG_BUGON(chunksize != chunk->chunksize);
-
- sbi->saved_by_deduplication += chunksize;
- if (chunk->blkaddr == erofs_holechunk.blkaddr) {
- chunk = &erofs_holechunk;
- erofs_dbg("Found duplicated hole chunk");
- } else {
- erofs_dbg("Found duplicated chunk at %llu",
- chunk->blkaddr | 0ULL);
+ /*
+ * if 'no-deduplication' is set, no need check hash
+ */
+ if (!sbi->no_deduplication) {
+ chunk = hashmap_get_from_hash(&blob_hashmap, hash, sha256);
+ if (chunk) {
+ DBG_BUGON(chunksize != chunk->chunksize);
+
+ sbi->saved_by_deduplication += chunksize;
+ if (chunk->blkaddr == erofs_holechunk.blkaddr) {
+ chunk = &erofs_holechunk;
+ erofs_dbg("Found duplicated hole chunk");
+ } else {
+ erofs_dbg("Found duplicated chunk at %llu",
+ chunk->blkaddr | 0ULL);
+ }
+ return chunk;
}
- return chunk;
}
chunk = malloc(sizeof(struct erofs_blobchunk));
diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index cc5a310..59b74be 100644
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -235,6 +235,12 @@ Specify maximum decompressed extent size in bytes.
(used together with \fB-T\fR) the given timestamp is only applied to the build
time.
.TP
+.B "\-\-no-deduplication"
+Disable deduplication. Set this option if mount with \fBdax\fR.
+
+The deduplicated chunks are shared by multiple files or by different parts within one file.
+As for now, kernel can not handle DAX map well for the deduplicated chunks.
+.TP
.B "\-\-preserve-mtime"
Use extended inodes instead of compact inodes if the file modification time
would overflow compact inodes. This is the default. Overrides
diff --git a/mkfs/main.c b/mkfs/main.c
index f1ea7df..d521e52 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -101,6 +101,7 @@ static struct option long_options[] = {
{"oci", optional_argument, NULL, 534},
#endif
{"zD", optional_argument, NULL, 536},
+ {"no-deduplication", no_argument, NULL, 537},
{0, 0, 0, 0},
};
@@ -207,6 +208,7 @@ static void usage(int argc, char **argv)
" --root-xattr-isize=# ensure the inline xattr size of the root directory is # bytes at least\n"
" --aufs replace aufs special files with overlayfs metadata\n"
" --sort=<path,none> data sorting order for tarballs as input (default: path)\n"
+ " --no-deduplication disable deduplication\n"
#ifdef S3EROFS_ENABLED
" --s3=X generate an image from S3-compatible object store\n"
" [,passwd_file=Y] X=endpoint, Y=s3fs-compatible password file\n"
@@ -310,6 +312,7 @@ static LIST_HEAD(rebuild_src_list);
static u8 fixeduuid[16];
static bool valid_fixeduuid;
static unsigned int dsunit;
+static bool no_deduplication;
static int tarerofs_decoder;
static FILE *vmdk_dcf;
static char *mkfs_aws_zinfo_file;
@@ -1412,6 +1415,9 @@ static int mkfs_parse_options_cfg(struct erofs_importer_params *params,
else
params->compress_dir = false;
break;
+ case 537:
+ no_deduplication = true;
+ break;
case 'V':
version();
exit(0);
@@ -1793,6 +1799,8 @@ int main(int argc, char **argv)
if (err)
goto exit;
+ g_sbi.no_deduplication = no_deduplication;
+
/* Use the user-defined UUID or generate one for clean builds */
if (valid_fixeduuid)
memcpy(g_sbi.uuid, fixeduuid, sizeof(g_sbi.uuid));
--
2.34.1
More information about the Linux-erofs
mailing list