[PATCH] erofs-utils: use external xxhash library if possible
Gao Xiang
hsiangkao at linux.alibaba.com
Fri Dec 20 17:53:31 AEDT 2024
It's expected to be faster than the internal one.
Signed-off-by: Gao Xiang <hsiangkao at linux.alibaba.com>
---
configure.ac | 34 +++++++++++++++++++++++++++++
dump/Makefile.am | 2 +-
fsck/Makefile.am | 4 ++--
fuse/Makefile.am | 2 +-
lib/Makefile.am | 9 ++++++--
lib/dedupe.c | 2 +-
lib/{xxhash.h => liberofs_xxhash.h} | 15 +++++++++++++
lib/xattr.c | 2 +-
lib/xxhash.c | 2 +-
mkfs/Makefile.am | 2 +-
10 files changed, 64 insertions(+), 10 deletions(-)
rename lib/{xxhash.h => liberofs_xxhash.h} (74%)
diff --git a/configure.ac b/configure.ac
index 45a7d33..70b5363 100644
--- a/configure.ac
+++ b/configure.ac
@@ -148,6 +148,10 @@ AC_ARG_WITH(qpl,
[Enable and build with Intel QPL support @<:@default=disabled@:>@])], [],
[with_qpl="no"])
+AC_ARG_WITH(xxhash,
+ [AS_HELP_STRING([--with-xxhash],
+ [Enable and build with libxxhash support @<:@default=auto@:>@])])
+
AC_ARG_ENABLE(fuse,
[AS_HELP_STRING([--enable-fuse], [enable erofsfuse @<:@default=no@:>@])],
[enable_fuse="$enableval"], [enable_fuse="no"])
@@ -531,6 +535,31 @@ AS_IF([test "x$with_qpl" != "xno"], [
])
])
+# Configure Intel QPL
+have_xxhash="no"
+AS_IF([test "x$with_xxhash" != "xno"], [
+ PKG_CHECK_MODULES([libxxhash], [libxxhash], [
+ # Paranoia: don't trust the result reported by pkgconfig before trying out
+ saved_LIBS="$LIBS"
+ saved_CPPFLAGS=${CPPFLAGS}
+ CPPFLAGS="${libxxhash_CFLAGS} ${CPPFLAGS}"
+ LIBS="${libxxhash_LIBS} $LIBS"
+ AC_CHECK_HEADERS([xxhash.h],[
+ AC_CHECK_LIB(xxhash, XXH32, [], [
+ AC_MSG_ERROR([libxxhash doesn't work properly])])
+ AC_CHECK_DECL(XXH32, [have_xxhash="yes"],
+ [AC_MSG_ERROR([libxxhash doesn't work properly])], [[
+#include <xxhash.h>
+ ]])
+ ])
+ LIBS="${saved_LIBS}"
+ CPPFLAGS="${saved_CPPFLAGS}"], [
+ AS_IF([test "x$with_xxhash" = "xyes"], [
+ AC_MSG_ERROR([Cannot find proper libxxhash])
+ ])
+ ])
+])
+
# Enable 64-bit off_t
CFLAGS+=" -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64"
@@ -553,6 +582,7 @@ AM_CONDITIONAL([ENABLE_LIBLZMA], [test "x${have_liblzma}" = "xyes"])
AM_CONDITIONAL([ENABLE_LIBDEFLATE], [test "x${have_libdeflate}" = "xyes"])
AM_CONDITIONAL([ENABLE_LIBZSTD], [test "x${have_libzstd}" = "xyes"])
AM_CONDITIONAL([ENABLE_QPL], [test "x${have_qpl}" = "xyes"])
+AM_CONDITIONAL([ENABLE_XXHASH], [test "x${have_xxhash}" = "xyes"])
AM_CONDITIONAL([ENABLE_STATIC_FUSE], [test "x${enable_static_fuse}" = "xyes"])
if test "x$have_uuid" = "xyes"; then
@@ -600,6 +630,10 @@ if test "x$have_qpl" = "xyes"; then
AC_SUBST([libqpl_CFLAGS])
fi
+if test "x$have_xxhash" = "xyes"; then
+ AC_DEFINE([HAVE_XXHASH], 1, [Define to 1 if xxhash is found])
+fi
+
# Dump maximum block size
AS_IF([test "x$erofs_cv_max_block_size" = "x"],
[$erofs_cv_max_block_size = 4096], [])
diff --git a/dump/Makefile.am b/dump/Makefile.am
index 2a4f67a..2cf7fe8 100644
--- a/dump/Makefile.am
+++ b/dump/Makefile.am
@@ -8,4 +8,4 @@ dump_erofs_SOURCES = main.c
dump_erofs_CFLAGS = -Wall -I$(top_srcdir)/include
dump_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \
${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} \
- ${libzstd_LIBS} ${libqpl_LIBS}
+ ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS}
diff --git a/fsck/Makefile.am b/fsck/Makefile.am
index 5bdee4d..3b7b591 100644
--- a/fsck/Makefile.am
+++ b/fsck/Makefile.am
@@ -8,7 +8,7 @@ fsck_erofs_SOURCES = main.c
fsck_erofs_CFLAGS = -Wall -I$(top_srcdir)/include
fsck_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \
${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} \
- ${libzstd_LIBS} ${libqpl_LIBS}
+ ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS}
if ENABLE_FUZZING
noinst_PROGRAMS = fuzz_erofsfsck
@@ -17,5 +17,5 @@ fuzz_erofsfsck_CFLAGS = -Wall -I$(top_srcdir)/include -DFUZZING
fuzz_erofsfsck_LDFLAGS = -fsanitize=address,fuzzer
fuzz_erofsfsck_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \
${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} \
- ${libzstd_LIBS} ${libqpl_LIBS}
+ ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS}
endif
diff --git a/fuse/Makefile.am b/fuse/Makefile.am
index 50186da..55fb61f 100644
--- a/fuse/Makefile.am
+++ b/fuse/Makefile.am
@@ -8,7 +8,7 @@ erofsfuse_CFLAGS = -Wall -I$(top_srcdir)/include
erofsfuse_CFLAGS += ${libfuse2_CFLAGS} ${libfuse3_CFLAGS} ${libselinux_CFLAGS}
erofsfuse_LDADD = $(top_builddir)/lib/liberofs.la ${libfuse2_LIBS} ${libfuse3_LIBS} ${liblz4_LIBS} \
${libselinux_LIBS} ${liblzma_LIBS} ${zlib_LIBS} ${libdeflate_LIBS} ${libzstd_LIBS} \
- ${libqpl_LIBS}
+ ${libqpl_LIBS} ${libxxhash_LIBS}
if ENABLE_STATIC_FUSE
lib_LTLIBRARIES = liberofsfuse.la
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 9c0604d..ef98377 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -27,14 +27,14 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
$(top_srcdir)/include/erofs/fragments.h \
$(top_srcdir)/include/erofs/rebuild.h \
$(top_srcdir)/lib/liberofs_private.h \
- $(top_srcdir)/lib/xxhash.h
+ $(top_srcdir)/lib/liberofs_xxhash.h
noinst_HEADERS += compressor.h
liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
namei.c data.c compress.c compressor.c zmap.c decompress.c \
compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
fragments.c dedupe.c uuid_unparse.c uuid.c tar.c \
- block_list.c xxhash.c rebuild.c diskbuf.c
+ block_list.c rebuild.c diskbuf.c
liberofs_la_CFLAGS = -Wall ${libuuid_CFLAGS} -I$(top_srcdir)/include
if ENABLE_LZ4
@@ -58,6 +58,11 @@ if ENABLE_LIBZSTD
liberofs_la_CFLAGS += ${libzstd_CFLAGS}
liberofs_la_SOURCES += compressor_libzstd.c
endif
+if ENABLE_XXHASH
+liberofs_la_CFLAGS += ${libxxhash_CFLAGS}
+else
+liberofs_la_SOURCES += xxhash.c
+endif
if ENABLE_EROFS_MT
liberofs_la_LDFLAGS = -lpthread
liberofs_la_SOURCES += workqueue.c
diff --git a/lib/dedupe.c b/lib/dedupe.c
index e475635..bc62841 100644
--- a/lib/dedupe.c
+++ b/lib/dedupe.c
@@ -6,7 +6,7 @@
#include "erofs/dedupe.h"
#include "erofs/print.h"
#include "rolling_hash.h"
-#include "xxhash.h"
+#include "liberofs_xxhash.h"
#include "sha256.h"
unsigned long erofs_memcmp2(const u8 *s1, const u8 *s2,
diff --git a/lib/xxhash.h b/lib/liberofs_xxhash.h
similarity index 74%
rename from lib/xxhash.h
rename to lib/liberofs_xxhash.h
index 723c3a5..a0f8367 100644
--- a/lib/xxhash.h
+++ b/lib/liberofs_xxhash.h
@@ -8,7 +8,21 @@ extern "C"
#endif
#include <stdint.h>
+#ifdef HAVE_XXHASH_H
+#include <xxhash.h>
+#endif
+
+#ifdef HAVE_XXHASH
+static inline uint32_t xxh32(const void *input, size_t length, uint32_t seed)
+{
+ return XXH32(input, length, seed);
+}
+static inline uint64_t xxh64(const void *input, const size_t len, const uint64_t seed)
+{
+ return XXH64(input, len, seed);
+}
+#else
/*
* xxh32() - calculate the 32-bit hash of the input with a given seed.
*
@@ -32,6 +46,7 @@ uint32_t xxh32(const void *input, size_t length, uint32_t seed);
* Return: The 64-bit hash of the data.
*/
uint64_t xxh64(const void *input, const size_t len, const uint64_t seed);
+#endif
#ifdef __cplusplus
}
diff --git a/lib/xattr.c b/lib/xattr.c
index c95928e..c20c9a2 100644
--- a/lib/xattr.c
+++ b/lib/xattr.c
@@ -17,7 +17,7 @@
#include "erofs/xattr.h"
#include "erofs/cache.h"
#include "erofs/fragments.h"
-#include "xxhash.h"
+#include "liberofs_xxhash.h"
#include "liberofs_private.h"
#ifndef XATTR_SYSTEM_PREFIX
diff --git a/lib/xxhash.c b/lib/xxhash.c
index 2768375..ee78ebf 100644
--- a/lib/xxhash.c
+++ b/lib/xxhash.c
@@ -44,7 +44,7 @@
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
#include "erofs/defs.h"
-#include "xxhash.h"
+#include "liberofs_xxhash.h"
/*-*************************************
* Macros
diff --git a/mkfs/Makefile.am b/mkfs/Makefile.am
index 6354712..2499242 100644
--- a/mkfs/Makefile.am
+++ b/mkfs/Makefile.am
@@ -7,4 +7,4 @@ mkfs_erofs_SOURCES = main.c
mkfs_erofs_CFLAGS = -Wall -I$(top_srcdir)/include
mkfs_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \
${libuuid_LIBS} ${liblz4_LIBS} ${liblzma_LIBS} ${zlib_LIBS} \
- ${libdeflate_LIBS} ${libzstd_LIBS} ${libqpl_LIBS}
+ ${libdeflate_LIBS} ${libzstd_LIBS} ${libqpl_LIBS} ${libxxhash_LIBS}
--
2.39.5
More information about the Linux-erofs
mailing list