[PATCH v2] erofs-utils: introduce exclude dirs and files
Gao Xiang
hsiangkao at aol.com
Tue Feb 11 03:34:19 AEDT 2020
On Mon, Feb 10, 2020 at 09:41:04PM +0800, Li Guifu wrote:
> Gao Xiang,
> It would make a long string match on each path sprintfed from a dir.
It seems that your @foxmail.com email is still broken, because I haven't
seen this email on mailing list till now. I'd suggest you could fix
your email first.
> The parent's path don'y need to compare every times,just loop the sub dirs.
It's a very simple approach to implement this feature (exact match) as
the first step. Because for most paths without valid prefix, it will
just bail out on the first char and it will have no impact if we don't
use this feature.
Laterly, I'd suggest some sorted string array to solve what you
mentioned. (You could improve it in the following way.)
Take an example, we could keep such sorted string array in memory.
0 A/B
1 A/B/C
2 A/B/D
3 A/B/E/F
4 A/G
5 H
For each access, we can binary search and narrow down the range dynamicly.
e.g. if we open A dir, the search range will be from 0 to 4. and if we
open A/B dir, the search range will then be narrowed down to 0~3.
> It also would better to use relate path on image path not the absolute
> path in the exclude path.
No, I think it works on relative path. You could test it out first.
Thanks,
Gao Xiang
>
> What your's opinion ?
>
> On 2020/2/5 1:02, Gao Xiang via Linux-erofs wrote:
> > Hi Guifu,
> >
> > I cleanup the whole patch and get rid of all dedupe check in order
> > to make it as simple as possible since I think not too many exclude
> > files in general.
> >
> > Please take some time helping testing the following patch and
> > hope to get your feedback.
> >
> > Thanks,
> > Gao Xiang
> >
> >
> >
> >>From 600fc166cdaaa0e458181729245ce11affa83ac5 Mon Sep 17 00:00:00 2001
> > From: Li Guifu <blucer.lee at foxmail.com>
> > Date: Mon, 3 Feb 2020 23:38:11 +0800
> > Subject: [PATCH v3] erofs-utils: introduce exclude dirs and files
> >
> > Add excluded file feature "--exclude-path=", which can be used
> > to build EROFS image without some user specific files or dirs.
> >
> > Note that you may give multiple `--exclude-path' options.
> >
> > Signed-off-by: Li Guifu <blucer.lee at foxmail.com>
> > Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
> > ---
> > changes since v2:
> > - cleanup the whole implementation;
> > - complete usage message;
> > - complete manual page.
> >
> > include/erofs/exclude.h | 23 +++++++++++
> > lib/Makefile.am | 2 +-
> > lib/exclude.c | 89 +++++++++++++++++++++++++++++++++++++++++
> > lib/inode.c | 5 +++
> > man/mkfs.erofs.1 | 4 ++
> > mkfs/main.c | 26 +++++++++---
> > 6 files changed, 142 insertions(+), 7 deletions(-)
> > create mode 100644 include/erofs/exclude.h
> > create mode 100644 lib/exclude.c
> >
> > diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
> > new file mode 100644
> > index 0000000..580fefe
> > --- /dev/null
> > +++ b/include/erofs/exclude.h
> > @@ -0,0 +1,23 @@
> > +/* SPDX-License-Identifier: GPL-2.0+ */
> > +/*
> > + * erofs-utils/include/erofs/exclude.h
> > + *
> > + * Created by Li Guifu <blucer.lee at foxmail.com>
> > + */
> > +#ifndef __EROFS_EXCLUDE_H
> > +#define __EROFS_EXCLUDE_H
> > +
> > +struct erofs_exclude_rule {
> > + struct list_head list;
> > +
> > + char *pattern;
> > +};
> > +
> > +void erofs_exclude_set_root(const char *rootdir);
> > +void erofs_cleanup_exclude_rules(void);
> > +
> > +int erofs_parse_exclude_path(const char *args);
> > +struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
> > + const char *name);
> > +#endif
> > +
> > diff --git a/lib/Makefile.am b/lib/Makefile.am
> > index 1ff81f9..e4b51e6 100644
> > --- a/lib/Makefile.am
> > +++ b/lib/Makefile.am
> > @@ -3,7 +3,7 @@
> >
> > noinst_LTLIBRARIES = liberofs.la
> > liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
> > - compress.c compressor.c
> > + compress.c compressor.c exclude.c
> > liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
> > if ENABLE_LZ4
> > liberofs_la_CFLAGS += ${LZ4_CFLAGS}
> > diff --git a/lib/exclude.c b/lib/exclude.c
> > new file mode 100644
> > index 0000000..9b48325
> > --- /dev/null
> > +++ b/lib/exclude.c
> > @@ -0,0 +1,89 @@
> > +// SPDX-License-Identifier: GPL-2.0+
> > +/*
> > + * erofs-utils/lib/exclude.c
> > + *
> > + * Created by Li Guifu <blucer.lee at foxmail.com>
> > + */
> > +#include <string.h>
> > +#include <stdlib.h>
> > +#include "erofs/err.h"
> > +#include "erofs/list.h"
> > +#include "erofs/print.h"
> > +#include "erofs/exclude.h"
> > +
> > +static LIST_HEAD(exclude_head);
> > +static unsigned int rpathlen; /* root directory prefix length */
> > +
> > +void erofs_exclude_set_root(const char *rootdir)
> > +{
> > + rpathlen = strlen(rootdir);
> > +}
> > +
> > +static struct erofs_exclude_rule *erofs_insert_exclude(const char *s)
> > +{
> > + struct erofs_exclude_rule *e;
> > +
> > + e = malloc(sizeof(*e));
> > + if (!e)
> > + return ERR_PTR(-ENOMEM);
> > +
> > + /* exact match */
> > + e->pattern = strdup(s);
> > + if (!e->pattern) {
> > + free(e);
> > + return ERR_PTR(-ENOMEM);
> > + }
> > +
> > + list_add_tail(&e->list, &exclude_head);
> > + erofs_info("exclude path %s inserted", e->pattern);
> > + return e;
> > +}
> > +
> > +void erofs_cleanup_exclude_rules(void)
> > +{
> > + struct erofs_exclude_rule *e, *n;
> > +
> > + list_for_each_entry_safe(e, n, &exclude_head, list) {
> > + list_del(&e->list);
> > + free(e->pattern);
> > + free(e);
> > + }
> > +}
> > +
> > +int erofs_parse_exclude_path(const char *args)
> > +{
> > + struct erofs_exclude_rule *e = erofs_insert_exclude(args);
> > +
> > + if (IS_ERR(e)) {
> > + erofs_cleanup_exclude_rules();
> > + return PTR_ERR(e);
> > + }
> > + return 0;
> > +}
> > +
> > +struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
> > + const char *name)
> > +{
> > + char buf[PATH_MAX];
> > + const char *s;
> > + struct erofs_exclude_rule *e;
> > +
> > + if (!dir) {
> > + /* no prefix */
> > + s = name;
> > + } else {
> > + sprintf(buf, "%s/%s", dir, name);
> > + s = buf;
> > + }
> > +
> > + s += rpathlen;
> > + while (*s == '/')
> > + s++;
> > +
> > + list_for_each_entry(e, &exclude_head, list) {
> > + if (!strcmp(e->pattern, s))
> > + return e;
> > + }
> > + return NULL;
> > +}
> > +
> > diff --git a/lib/inode.c b/lib/inode.c
> > index bd0652b..7114023 100644
> > --- a/lib/inode.c
> > +++ b/lib/inode.c
> > @@ -20,6 +20,7 @@
> > #include "erofs/io.h"
> > #include "erofs/compress.h"
> > #include "erofs/xattr.h"
> > +#include "erofs/exclude.h"
> >
> > struct erofs_sb_info sbi;
> >
> > @@ -877,6 +878,10 @@ struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
> > !strncmp(dp->d_name, "lost+found", strlen("lost+found")))
> > continue;
> >
> > + /* skip if it's a exclude file */
> > + if (erofs_is_exclude_path(dir->i_srcpath, dp->d_name))
> > + continue;
> > +
> > d = erofs_d_alloc(dir, dp->d_name);
> > if (IS_ERR(d)) {
> > ret = PTR_ERR(d);
> > diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
> > index d6bf828..aa927a9 100644
> > --- a/man/mkfs.erofs.1
> > +++ b/man/mkfs.erofs.1
> > @@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output.
> > Set all files to the given UNIX timestamp. Reproducible builds requires setting
> > all to a specific one.
> > .TP
> > +.BI "\-\-exclude-path=" path
> > +Ignore file that matches the exact literal path.
> > +You may give multiple `--exclude-path' options.
> > +.TP
> > .B \-\-help
> > Display this help and exit.
> > .SH AUTHOR
> > diff --git a/mkfs/main.c b/mkfs/main.c
> > index 817a6c1..d913b5d 100644
> > --- a/mkfs/main.c
> > +++ b/mkfs/main.c
> > @@ -21,6 +21,7 @@
> > #include "erofs/io.h"
> > #include "erofs/compress.h"
> > #include "erofs/xattr.h"
> > +#include "erofs/exclude.h"
> >
> > #ifdef HAVE_LIBUUID
> > #include <uuid/uuid.h>
> > @@ -30,6 +31,7 @@
> >
> > static struct option long_options[] = {
> > {"help", no_argument, 0, 1},
> > + {"exclude-path", required_argument, NULL, 2},
> > {0, 0, 0, 0},
> > };
> >
> > @@ -50,12 +52,13 @@ static void usage(void)
> > {
> > fputs("usage: [options] FILE DIRECTORY\n\n"
> > "Generate erofs image from DIRECTORY to FILE, and [options] are:\n"
> > - " -zX[,Y] X=compressor (Y=compression level, optional)\n"
> > - " -d# set output message level to # (maximum 9)\n"
> > - " -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> > - " -EX[,...] X=extended options\n"
> > - " -T# set a fixed UNIX timestamp # to all files\n"
> > - " --help display this help and exit\n"
> > + " -zX[,Y] X=compressor (Y=compression level, optional)\n"
> > + " -d# set output message level to # (maximum 9)\n"
> > + " -x# set xattr tolerance to # (< 0, disable xattrs; default 2)\n"
> > + " -EX[,...] X=extended options\n"
> > + " -T# set a fixed UNIX timestamp # to all files\n"
> > + " --exclude-path=X avoid including file X (X = exact literal path)\n"
> > + " --help display this help and exit\n"
> > "\nAvailable compressors are: ", stderr);
> > print_available_compressors(stderr, ", ");
> > }
> > @@ -178,6 +181,15 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
> > }
> > break;
> >
> > + case 2:
> > + opt = erofs_parse_exclude_path(optarg);
> > + if (opt) {
> > + erofs_err("failed to parse exclude path: %s",
> > + erofs_strerror(opt));
> > + return opt;
> > + }
> > + break;
> > +
> > case 1:
> > usage();
> > exit(0);
> > @@ -372,6 +384,7 @@ int main(int argc, char **argv)
> > }
> >
> > erofs_show_config();
> > + erofs_exclude_set_root(cfg.c_src_path);
> >
> > sb_bh = erofs_buffer_init();
> > if (IS_ERR(sb_bh)) {
> > @@ -428,6 +441,7 @@ int main(int argc, char **argv)
> > exit:
> > z_erofs_compress_exit();
> > dev_close();
> > + erofs_cleanup_exclude_rules();
> > erofs_exit_configure();
> >
> > if (err) {
> >
More information about the Linux-erofs
mailing list