Re: [PATCH v2] erofs-utils: add Apache 2.0 license
Let's release liberofs under GPL-2.0+ OR Apache-2.0 license for better integration. Signed-off-by: Huang Jianan Acked-by: Li Guifu Thanks, Li Guifu --- include/erofs/blobchunk.h | 2 +- include/erofs/block_list.h | 2 +- include/erofs/cache.h | 2 +- include/erofs/compress.h | 2 +- include/erofs/compress_hints.h | 2 +- include/erofs/config.h | 2 +- include/erofs/decompress.h | 2 +- include/erofs/defs.h | 2 +- include/erofs/err.h| 2 +- include/erofs/exclude.h| 2 +- include/erofs/inode.h | 2 +- include/erofs/internal.h | 2 +- include/erofs/io.h | 2 +- include/erofs/list.h | 2 +- include/erofs/print.h | 2 +- include/erofs/trace.h | 2 +- include/erofs/xattr.h | 2 +- lib/Makefile.am| 2 +- lib/blobchunk.c| 2 +- lib/block_list.c | 2 +- lib/cache.c| 2 +- lib/compress.c | 2 +- lib/compress_hints.c | 2 +- lib/compressor.c | 2 +- lib/compressor.h | 2 +- lib/compressor_liblzma.c | 2 +- lib/compressor_lz4.c | 2 +- lib/compressor_lz4hc.c | 2 +- lib/config.c | 2 +- lib/data.c | 2 +- lib/decompress.c | 2 +- lib/exclude.c | 2 +- lib/inode.c| 2 +- lib/io.c | 2 +- lib/namei.c| 2 +- lib/super.c| 2 +- lib/xattr.c| 2 +- lib/zmap.c | 2 +- 38 files changed, 38 insertions(+), 38 deletions(-) diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h index 4e1ae79..49cb7bf 100644 --- a/include/erofs/blobchunk.h +++ b/include/erofs/blobchunk.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * erofs-utils/lib/blobchunk.h * diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h index ca8053e..78fab44 100644 --- a/include/erofs/block_list.h +++ b/include/erofs/block_list.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C), 2021, Coolpad Group Limited. * Created by Yue Hu diff --git a/include/erofs/cache.h b/include/erofs/cache.h index 7957ee5..de12399 100644 --- a/include/erofs/cache.h +++ b/include/erofs/cache.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/compress.h b/include/erofs/compress.h index fdbf5ff..21eac57 100644 --- a/include/erofs/compress.h +++ b/include/erofs/compress.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/compress_hints.h b/include/erofs/compress_hints.h index 43f80e1..659c5b6 100644 --- a/include/erofs/compress_hints.h +++ b/include/erofs/compress_hints.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C), 2008-2021, OPPO Mobile Comm Corp., Ltd. * Created by Huang Jianan diff --git a/include/erofs/config.h b/include/erofs/config.h index cb064b6..e41c079 100644 --- a/include/erofs/config.h +++ b/include/erofs/config.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/decompress.h b/include/erofs/decompress.h index e649c80..82bf7b8 100644 --- a/include/erofs/decompress.h +++ b/include/erofs/decompress.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C), 2008-2020, OPPO Mobile Comm Corp., Ltd. * Created by Huang Jianan diff --git a/include/erofs/defs.h b/include/erofs/defs.h index 4db237f..e745bc0 100644 --- a/include/erofs/defs.h +++ b/include/erofs/defs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/err.h b/include/erofs/err.h index 18f152a..08b0bdb 100644 --- a/include/erofs/err.h +++ b/include/erofs/err.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h index 599f018..3f17032 100644 --- a/include/erofs/exclude.h +++ b
[PATCH v3] AOSP: erofs-utils: add block list support
From: Yue Hu
Android update engine will treat EROFS filesystem image as one single
file. Let's add block list support to optimize OTA size.
Signed-off-by: Yue Hu
Signed-off-by: Gao Xiang
Signed-off-by: Li Guifu
---
include/erofs/block_list.h | 27 +++
include/erofs/config.h | 1 +
lib/block_list.c | 91 ++
lib/compress.c | 2 +
lib/inode.c| 6 +++
mkfs/main.c| 14 ++
6 files changed, 141 insertions(+)
create mode 100644 include/erofs/block_list.h
create mode 100644 lib/block_list.c
diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
new file mode 100644
index 000..7756d8a
--- /dev/null
+++ b/include/erofs/block_list.h
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/include/erofs/block_list.h
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu
+ */
+#ifndef __EROFS_BLOCK_LIST_H
+#define __EROFS_BLOCK_LIST_H
+
+#include "internal.h"
+
+#ifdef WITH_ANDROID
+int erofs_droid_blocklist_fopen(void);
+void erofs_droid_blocklist_fclose(void);
+void erofs_droid_blocklist_write(struct erofs_inode *inode,
+erofs_blk_t blk_start, erofs_blk_t nblocks);
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+ erofs_blk_t blkaddr);
+#else
+static inline void erofs_droid_blocklist_write(struct erofs_inode *inode,
+erofs_blk_t blk_start, erofs_blk_t nblocks) {}
+static inline
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+ erofs_blk_t blkaddr) {}
+#endif
+#endif
diff --git a/include/erofs/config.h b/include/erofs/config.h
index d140a73..67e7a0f 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -65,6 +65,7 @@ struct erofs_configure {
char *mount_point;
char *target_out_path;
char *fs_config_file;
+ char *block_list_file;
#endif
};
diff --git a/lib/block_list.c b/lib/block_list.c
new file mode 100644
index 000..90fe0fd
--- /dev/null
+++ b/lib/block_list.c
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/block_list.c
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu
+ */
+#ifdef WITH_ANDROID
+#include
+#include
+#include "erofs/block_list.h"
+
+#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
+#include "erofs/print.h"
+
+static FILE *block_list_fp = NULL;
+
+int erofs_droid_blocklist_fopen(void)
+{
+ if (block_list_fp)
+ return 0;
+
+ block_list_fp = fopen(cfg.block_list_file, "w");
+
+ if (!block_list_fp)
+ return -1;
+ return 0;
+}
+
+void erofs_droid_blocklist_fclose(void)
+{
+ if (!block_list_fp)
+ return;
+
+ fclose(block_list_fp);
+ block_list_fp = NULL;
+}
+
+static void blocklist_write(const char *path, erofs_blk_t blk_start,
+ erofs_blk_t nblocks, bool has_tail)
+{
+ const char *fspath = erofs_fspath(path);
+
+ fprintf(block_list_fp, "/%s", cfg.mount_point);
+
+ if (fspath[0] != '/')
+ fprintf(block_list_fp, "/");
+
+ if (nblocks == 1)
+ fprintf(block_list_fp, "%s %u", fspath, blk_start);
+ else if (nblocks > 1)
+ fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
+ blk_start + nblocks - 1);
+ else
+ fprintf(block_list_fp, "%s", fspath);
+
+ if (!has_tail)
+ fprintf(block_list_fp, "\n");
+}
+
+void erofs_droid_blocklist_write(struct erofs_inode *inode,
+erofs_blk_t blk_start, erofs_blk_t nblocks)
+{
+ if (!block_list_fp || !cfg.mount_point || !nblocks)
+ return;
+
+ blocklist_write(inode->i_srcpath, blk_start, nblocks,
+ !!inode->idata_size);
+}
+
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+ erofs_blk_t blkaddr)
+{
+ if (!block_list_fp || !cfg.mount_point)
+ return;
+
+ /* XXX: a bit hacky.. may need a better approach */
+ if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
+ return;
+
+ if (erofs_blknr(inode->i_size)) { // its block list has been output
before
+ if (blkaddr == NULL_ADDR)
+ fprintf(block_list_fp, "\n");
+ else
+ fprintf(block_list_fp, " %u\n", blkaddr);
+ return;
+ }
+ if (blkaddr != NULL_ADDR)
+ blocklist_write(inode->i_srcpath, blkaddr, 1, false);
+}
+#endif
diff --git a/lib/compress.c b/lib/comp
Re: [PATCH v2] AOSP: erofs-utils: add block list support
Gao Xiang
There are three condition need to be fixed.
The whole inline file should not be printed anything even *\n*
The last inline block should add *\n* to trigger a new line
The last block and not inline should be print independently
On 6/23/21 1:31 PM, Gao Xiang wrote:
From: Yue Hu
Android update engine will treat EROFS filesystem image as one single
file. Let's add block list support to optimize OTA size.
Signed-off-by: Yue Hu
Signed-off-by: Gao Xiang
---
some cleanups.
Hi all,
please kindly check it again if it works since I dont have such
environment. If it doesn't work, please help fix it..
Thanks,
Gao Xiang
include/erofs/block_list.h | 27 +++
include/erofs/config.h | 1 +
lib/block_list.c | 86 ++
lib/compress.c | 2 ++
lib/inode.c| 6
mkfs/main.c| 14
6 files changed, 136 insertions(+)
create mode 100644 include/erofs/block_list.h
create mode 100644 lib/block_list.c
diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
new file mode 100644
index ..7756d8a5784c
--- /dev/null
+++ b/include/erofs/block_list.h
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/include/erofs/block_list.h
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu
+ */
+#ifndef __EROFS_BLOCK_LIST_H
+#define __EROFS_BLOCK_LIST_H
+
+#include "internal.h"
+
+#ifdef WITH_ANDROID
+int erofs_droid_blocklist_fopen(void);
+void erofs_droid_blocklist_fclose(void);
+void erofs_droid_blocklist_write(struct erofs_inode *inode,
+erofs_blk_t blk_start, erofs_blk_t nblocks);
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+ erofs_blk_t blkaddr);
+#else
+static inline void erofs_droid_blocklist_write(struct erofs_inode *inode,
+erofs_blk_t blk_start, erofs_blk_t nblocks) {}
+static inline
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+ erofs_blk_t blkaddr) {}
+#endif
+#endif
diff --git a/include/erofs/config.h b/include/erofs/config.h
index d140a735bd49..67e7a0fed24c 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -65,6 +65,7 @@ struct erofs_configure {
char *mount_point;
char *target_out_path;
char *fs_config_file;
+ char *block_list_file;
#endif
};
diff --git a/lib/block_list.c b/lib/block_list.c
new file mode 100644
index ..ffe780e91900
--- /dev/null
+++ b/lib/block_list.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/block_list.c
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu
+ */
+#ifdef WITH_ANDROID
+#include
+#include
+#include "erofs/block_list.h"
+
+#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
+#include "erofs/print.h"
+
+static FILE *block_list_fp = NULL;
+
+int erofs_droid_blocklist_fopen(void)
+{
+ if (block_list_fp)
+ return 0;
+
+ block_list_fp = fopen(cfg.block_list_file, "w");
+
+ if (!block_list_fp)
+ return -1;
+ return 0;
+}
+
+void erofs_droid_blocklist_fclose(void)
+{
+ if (!block_list_fp)
+ return;
+
+ fclose(block_list_fp);
+ block_list_fp = NULL;
+}
+
+static void blocklist_write(const char *path, erofs_blk_t blk_start,
+ erofs_blk_t nblocks, bool has_tail)
+{
+ const char *fspath = erofs_fspath(path);
+
+ fprintf(block_list_fp, "/%s", cfg.mount_point);
+
+ if (fspath[0] != '/')
+ fprintf(block_list_fp, "/");
+
+ if (nblocks == 1)
+ fprintf(block_list_fp, "%s %u", fspath, blk_start);
+ else if (nblocks > 1)
+ fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
+ blk_start + nblocks - 1);
+ else
+ fprintf(block_list_fp, "%s", fspath);
+
+ if (!has_tail)
+ fprintf(block_list_fp, "\n");
+}
+
+void erofs_droid_blocklist_write(struct erofs_inode *inode,
+erofs_blk_t blk_start, erofs_blk_t nblocks)
+{
+ if (!block_list_fp || !cfg.mount_point)
+ return;
+
+ blocklist_write(inode->i_srcpath, blk_start, nblocks,
+ !!inode->idata_size);
+}
+
+void erofs_droid_blocklist_write_tail_end(struct erofs_inode *inode,
+ erofs_blk_t blkaddr)
+{
+ if (!block_list_fp || !cfg.mount_point)
+ return;
+
+ /* XXX: a bit hacky.. may need a better approach */
+ if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
+ return;
+
+ if (blkaddr != NULL_ADDR)
+ fprintf(block_list_fp, " %u\n", blkaddr);
+ else
+
Re: [PATCH] AOSP: erofs-utils: add block list support
Hu Yue
Thanks to your contribution.
Base test shows it could record block map list correctly.
Some codes need be refactored for further ahead.
On 6/21/21 11:02 PM, Yue Hu wrote:
From: Yue Hu
Android update engine will treat EROFS filesystem image as one single
file. Let's add block list support to optimize OTA size.
Change-Id: I21d6177dff0ee65d3c57023b102e991d40873f0d
Signed-off-by: Yue Hu
---
include/erofs/block_list.h | 19 ++
include/erofs/config.h | 1 +
lib/block_list.c | 86 ++
lib/compress.c | 8 +
lib/inode.c| 21 ++-
mkfs/main.c| 17 +
6 files changed, 151 insertions(+), 1 deletion(-)
create mode 100644 include/erofs/block_list.h
create mode 100644 lib/block_list.c
step1:
block_list.c/h are new file, please add them to lib/Makefile.am or
it will not be built.
You would also add a Android.mk/bp for android target build
diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h
new file mode 100644
index 000..cbf1050
--- /dev/null
+++ b/include/erofs/block_list.h
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/include/erofs/block_list.h
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu
+ */
+#ifndef __EROFS_BLOCK_LIST_H
+#define __EROFS_BLOCK_LIST_H
+
+#include "internal.h"
+
+int block_list_fopen(void);
+void block_list_fclose(void);
+void write_block_list(const char *path, erofs_blk_t blk_start,
+ erofs_blk_t nblocks, bool has_tail);
+void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
+ bool inline_data, erofs_blk_t blkaddr);
+#endif
diff --git a/include/erofs/config.h b/include/erofs/config.h
index d140a73..67e7a0f 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -65,6 +65,7 @@ struct erofs_configure {
char *mount_point;
char *target_out_path;
char *fs_config_file;
+ char *block_list_file;
#endif
};
diff --git a/lib/block_list.c b/lib/block_list.c
new file mode 100644
index 000..6ebe0f9
--- /dev/null
+++ b/lib/block_list.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/block_list.c
+ *
+ * Copyright (C), 2021, Coolpad Group Limited.
+ * Created by Yue Hu
+ */
+#ifdef WITH_ANDROID
+#include
+
+#include "erofs/block_list.h"
+
+#define pr_fmt(fmt) "EROFS block_list: " FUNC_LINE_FMT fmt "\n"
+#include "erofs/print.h"
+
+static FILE *block_list_fp = NULL;
+
+int block_list_fopen(void)
+{
+ if (block_list_fp)
+ return 0;
+
+ block_list_fp = fopen(cfg.block_list_file, "w");
+
+ if (block_list_fp == NULL)
+ return -1;
step2:
The return code would be replace with errno please.
So further error message could be showed up.
+
+ return 0;
+}
+
+void block_list_fclose(void)
+{
+ if (block_list_fp) {
+ fclose(block_list_fp);
+ block_list_fp = NULL;
+ }
+}
+
+void write_block_list(const char *path, erofs_blk_t blk_start,
+ erofs_blk_t nblocks, bool has_tail)
+{
+ const char *fspath = erofs_fspath(path);
+
+ if (!block_list_fp || !cfg.mount_point)
+ return;
+
+ /* only tail-end data */
+ if (!nblocks)
+ return;
+
+ fprintf(block_list_fp, "/%s", cfg.mount_point);
+
+ if (fspath[0] != '/')
+ fprintf(block_list_fp, "/");
+
+ if (nblocks == 1) {
+ fprintf(block_list_fp, "%s %u", fspath, blk_start);
+ } else {
+ fprintf(block_list_fp, "%s %u-%u", fspath, blk_start,
+ blk_start + nblocks - 1);
+ }
+
+ if (!has_tail)
+ fprintf(block_list_fp, "\n");
+}
+
+void write_block_list_tail_end(const char *path, erofs_blk_t nblocks,
+ bool inline_data, erofs_blk_t blkaddr)
+{
+ if (!block_list_fp || !cfg.mount_point)
+ return;
step3:
if (!block_list_fp || !cfg.mount_point)
These codes are double checked at write_block_list() function.
Could you do a further optimize ?
+
+ if (!nblocks && !inline_data) {
+ erofs_dbg("%s : only tail-end non-inline data", path);
+ write_block_list(path, blkaddr, 1, false);
+ return;
+ }
+
+ if (nblocks) {
+ if (!inline_data)
+ fprintf(block_list_fp, " %u", blkaddr);
+
+ fprintf(block_list_fp, "\n");
+ }
+}
step4:
write_block_list_tail_end() has much if-condition nesting,
It could return early at the begin of it.
+#endif
diff --git a/lib/compress.c b/lib/compress.c
index 2093bfd..5dec0c3 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -19,6 +19,10 @@
#include "erofs/compress.h"
#include
Re: [PATCH v1 2/2] erofs-utils: introduce erofs_subdirs to one dir for sort
GaoXiang
On 2021/5/7 15:09, Gao Xiang wrote:
> Hi Guifu,
>
> On Wed, May 05, 2021 at 10:26:15PM +0800, Li Guifu via Linux-erofs wrote:
>> The structure erofs_subdirs has a dir number and a list_head,
>> the list_head is the same with i_subdirs in the inode.
>> Using erofs_subdirs as a temp place for dentrys under the dir,
>> and then sort it before replace to i_subdirs
>>
>> Signed-off-by: Li Guifu
>> ---
>> include/erofs/internal.h | 5 +++
>> lib/inode.c | 95 +---
>> 2 files changed, 64 insertions(+), 36 deletions(-)
>>
>> diff --git a/include/erofs/internal.h b/include/erofs/internal.h
>> index 1339341..7cd42ca 100644
>> --- a/include/erofs/internal.h
>> +++ b/include/erofs/internal.h
>> @@ -172,6 +172,11 @@ struct erofs_inode {
>> #endif
>> };
>>
>> +struct erofs_subdirs {
>> +struct list_head i_subdirs;
>> +unsigned int nr_subdirs;
>> +};
>> +
>> static inline bool is_inode_layout_compression(struct erofs_inode *inode)
>> {
>> return erofs_inode_is_data_compressed(inode->datalayout);
>> diff --git a/lib/inode.c b/lib/inode.c
>> index 787e5b4..3e138a6 100644
>> --- a/lib/inode.c
>> +++ b/lib/inode.c
>> @@ -96,7 +96,7 @@ unsigned int erofs_iput(struct erofs_inode *inode)
>> return 0;
>> }
>>
>> -struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
>> +struct erofs_dentry *erofs_d_alloc(struct erofs_subdirs *subdirs,
>> const char *name)
>> {
>> struct erofs_dentry *d = malloc(sizeof(*d));
>> @@ -107,7 +107,8 @@ struct erofs_dentry *erofs_d_alloc(struct erofs_inode
>> *parent,
>> strncpy(d->name, name, EROFS_NAME_LEN - 1);
>> d->name[EROFS_NAME_LEN - 1] = '\0';
>>
>> -list_add_tail(>d_child, >i_subdirs);
>> +list_add_tail(>d_child, >i_subdirs);
>> +subdirs->nr_subdirs++;
>> return d;
>> }
>>
>> @@ -150,38 +151,12 @@ static int comp_subdir(const void *a, const void *b)
>> return strcmp(da->name, db->name);
>> }
>>
>> -int erofs_prepare_dir_file(struct erofs_inode *dir, unsigned int nr_subdirs)
>> +int erofs_prepare_dir_file(struct erofs_inode *dir)
>
> Err... nope, that is not what I suggested, I was suggesting
>
> int erofs_prepare_dir_file(struct erofs_inode *dir,
> struct erofs_subdirs *subdirs)
>
>> {
>> -struct erofs_dentry *d, *n, **sorted_d;
>> -unsigned int d_size, i_nlink, i;
>> +struct erofs_dentry *d;
>> +unsigned int d_size, i_nlink;
>> int ret;
>>
>> -/* dot is pointed to the current dir inode */
>> -d = erofs_d_alloc(dir, ".");
>> -d->inode = erofs_igrab(dir);
>> -d->type = EROFS_FT_DIR;
>> -
>> -/* dotdot is pointed to the parent dir */
>> -d = erofs_d_alloc(dir, "..");
>> -d->inode = erofs_igrab(dir->i_parent);
>> -d->type = EROFS_FT_DIR;
>
> Leave these two here, since it's a part of dir preparation.
I think this erofs_prepare_dir_file() funtion just do space prepare
that dirs have been sorted before
. and .. should been added along with readir
[PATCH v1 2/2] erofs-utils: introduce erofs_subdirs to one dir for sort
The structure erofs_subdirs has a dir number and a list_head,
the list_head is the same with i_subdirs in the inode.
Using erofs_subdirs as a temp place for dentrys under the dir,
and then sort it before replace to i_subdirs
Signed-off-by: Li Guifu
---
include/erofs/internal.h | 5 +++
lib/inode.c | 95 +---
2 files changed, 64 insertions(+), 36 deletions(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 1339341..7cd42ca 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -172,6 +172,11 @@ struct erofs_inode {
#endif
};
+struct erofs_subdirs {
+ struct list_head i_subdirs;
+ unsigned int nr_subdirs;
+};
+
static inline bool is_inode_layout_compression(struct erofs_inode *inode)
{
return erofs_inode_is_data_compressed(inode->datalayout);
diff --git a/lib/inode.c b/lib/inode.c
index 787e5b4..3e138a6 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -96,7 +96,7 @@ unsigned int erofs_iput(struct erofs_inode *inode)
return 0;
}
-struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
+struct erofs_dentry *erofs_d_alloc(struct erofs_subdirs *subdirs,
const char *name)
{
struct erofs_dentry *d = malloc(sizeof(*d));
@@ -107,7 +107,8 @@ struct erofs_dentry *erofs_d_alloc(struct erofs_inode
*parent,
strncpy(d->name, name, EROFS_NAME_LEN - 1);
d->name[EROFS_NAME_LEN - 1] = '\0';
- list_add_tail(>d_child, >i_subdirs);
+ list_add_tail(>d_child, >i_subdirs);
+ subdirs->nr_subdirs++;
return d;
}
@@ -150,38 +151,12 @@ static int comp_subdir(const void *a, const void *b)
return strcmp(da->name, db->name);
}
-int erofs_prepare_dir_file(struct erofs_inode *dir, unsigned int nr_subdirs)
+int erofs_prepare_dir_file(struct erofs_inode *dir)
{
- struct erofs_dentry *d, *n, **sorted_d;
- unsigned int d_size, i_nlink, i;
+ struct erofs_dentry *d;
+ unsigned int d_size, i_nlink;
int ret;
- /* dot is pointed to the current dir inode */
- d = erofs_d_alloc(dir, ".");
- d->inode = erofs_igrab(dir);
- d->type = EROFS_FT_DIR;
-
- /* dotdot is pointed to the parent dir */
- d = erofs_d_alloc(dir, "..");
- d->inode = erofs_igrab(dir->i_parent);
- d->type = EROFS_FT_DIR;
-
- /* sort subdirs */
- nr_subdirs += 2;
- sorted_d = malloc(nr_subdirs * sizeof(d));
- if (!sorted_d)
- return -ENOMEM;
- i = 0;
- list_for_each_entry_safe(d, n, >i_subdirs, d_child) {
- list_del(>d_child);
- sorted_d[i++] = d;
- }
- DBG_BUGON(i != nr_subdirs);
- qsort(sorted_d, nr_subdirs, sizeof(d), comp_subdir);
- for (i = 0; i < nr_subdirs; i++)
- list_add_tail(_d[i]->d_child, >i_subdirs);
- free(sorted_d);
-
/* let's calculate dir size and update i_nlink */
d_size = 0;
i_nlink = 0;
@@ -926,13 +901,58 @@ void erofs_d_invalidate(struct erofs_dentry *d)
erofs_iput(inode);
}
+void erofs_subdirs_init(struct erofs_inode *dir, struct erofs_subdirs *subdirs)
+{
+ struct erofs_dentry *d;
+
+ subdirs->nr_subdirs = 0;
+ init_list_head(>i_subdirs);
+
+ /* dot is pointed to the current dir inode */
+ d = erofs_d_alloc(subdirs, ".");
+ d->inode = erofs_igrab(dir);
+ d->type = EROFS_FT_DIR;
+
+ /* dotdot is pointed to the parent dir */
+ d = erofs_d_alloc(subdirs, "..");
+ d->inode = erofs_igrab(dir->i_parent);
+ d->type = EROFS_FT_DIR;
+}
+
+static int erofs_subdirs_sorted(struct erofs_subdirs *subdirs)
+{
+ struct erofs_dentry *d, *n, **sorted_d;
+ unsigned int i;
+ const unsigned int nr_subdirs = subdirs->nr_subdirs;
+
+ if (nr_subdirs == 0) return 0;
+
+ sorted_d = malloc(nr_subdirs * sizeof(d));
+ if (!sorted_d)
+ return -ENOMEM;
+ i = 0;
+ list_for_each_entry_safe(d, n, >i_subdirs, d_child) {
+ list_del(>d_child);
+ sorted_d[i++] = d;
+ }
+
+ DBG_BUGON(i != nr_subdirs);
+ DBG_BUGON(!list_empty(>i_subdirs));
+
+ qsort(sorted_d, nr_subdirs, sizeof(d), comp_subdir);
+ for (i = 0; i < nr_subdirs; i++)
+ list_add_tail(_d[i]->d_child, >i_subdirs);
+ free(sorted_d);
+ return 0;
+}
+
struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
{
int ret;
DIR *_dir;
struct dirent *dp;
struct erofs_dentry *d;
- unsigned int nr_subdirs;
+ struct erofs_subdirs subdirs;
ret = erofs_prepare_xattr_ibody(dir);
if (ret < 0)
@@ -972,7 +992,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct
[PATCH v1 1/2] erofs-utils: add list_replace from linux kernel for dirs sorted later
A temp list head will be replaced to inode i_subdirs.
Signed-off-by: Li Guifu
---
include/erofs/list.h | 16
1 file changed, 16 insertions(+)
diff --git a/include/erofs/list.h b/include/erofs/list.h
index 3572726..7238418 100644
--- a/include/erofs/list.h
+++ b/include/erofs/list.h
@@ -67,6 +67,22 @@ static inline int list_empty(struct list_head *head)
return head->next == head;
}
+/**
+ * list_replace - replace old entry by new one
+ * @old : the element to be replaced
+ * @new : the new element to insert
+ *
+ * If @old was empty, it will be overwritten.
+ */
+static inline void list_replace(struct list_head *old,
+ struct list_head *new)
+{
+ new->next = old->next;
+ new->next->prev = new;
+ new->prev = old->prev;
+ new->prev->next = new;
+}
+
#define list_entry(ptr, type, member) container_of(ptr, type, member)
#define list_first_entry(ptr, type, member)
\
--
2.17.1
Re: [PATCH v4 5/5] erofs-utils: manpage: add manual for erofsfuse
On 2021/4/30 12:03, Gao Xiang wrote: > This patch adds missing erofsfuse manpage. > > Signed-off-by: Gao Xiang > --- > man/erofsfuse.1 | 44 > 1 file changed, 44 insertions(+) > create mode 100644 man/erofsfuse.1 > It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH v4 4/5] erofs-utils: zero out garbage trailing data for non-0padding cases
On 2021/4/30 12:03, Gao Xiang wrote:
> When "-E legacy-compress" is used, lz4 0padding feature will be
> disabled by default in order to support old kernels (< Linux v5.3).
>
> In that case, the current mkfs leaves previous garbage data after
> valid compressed data if the length becomes shorter. This doesn't
> matter for kernels >= v5.0 since LZ4_decompress_safe_partial() is used.
>
> However, for staging erofs v4.19, it used an in-house customized
> lz4 implemention due to LZ4_decompress_safe_partial() didn't work
> as expected at that time, yet it doesn't allow trailing random
> data in practice or decompression failure could happen.
>
> I don't think it really matters since "obsoleted_mkfs" works perfectly
> for such old staging versions (v4.19). Anyway, trailing garbage data
> sounds unreasonable, so let's zero out it now.
>
> Fixes: 66653ef10a7f ("erofs-utils: introduce compression for regular files")
> Signed-off-by: Gao Xiang
> ---
> lib/compress.c | 14 +++++-
> 1 file changed, 9 insertions(+), 5 deletions(-)
>
It looks good
Reviewed-by: Li Guifu
Thanks,
Re: [PATCH v4 2/5] erofs-utils: warn out experimental big pcluster
On 2021/4/30 12:03, Gao Xiang wrote: > It's still an experimental feature for now. Also set the default > logging level to 2 in order to print warn messages. > > Signed-off-by: Gao Xiang It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH v4 3/5] erofs-utils: manpage: add missing -C option for big pcluster
On 2021/4/30 12:03, Gao Xiang wrote: > Update the manpage as well. > > Signed-off-by: Gao Xiang It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH 1/3] erofs-utils: sync up with in-kernel erofs_fs.h
On 2021/4/27 10:37, Gao Xiang wrote: > This matches the latest in-kernel version. > > Signed-off-by: Gao Xiang > --- > include/erofs_fs.h | 17 +++-- > 1 file changed, 11 insertions(+), 6 deletions(-) > It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH v2] erofs-utils: use qsort() to sort dir->i_subdirs
On 2021/4/11 22:41, Gao Xiang wrote: > Guifu, > > On Sun, Apr 11, 2021 at 10:10:09PM +0800, Li GuiFu via Linux-erofs wrote: >> Hu Weiwen >> It really do a high sort performance increase, >> I have a idea that keeping the erofs_prepare_dir_file() function >> paramter stable, Using a independent function to do dirs sort. >> > > I think Weiwen's implementation looks fine to me, if you tend to > not passing nr_subdirs as a cleaner solution, my suggestion would > be: > 1) introduce a somewhat erofs_subdirs, which includes >- a list_head for all subdir dentries generated from d_alloc; >- a nr_subdirs count; > 2) update erofs_d_alloc to >erofs_d_alloc(struct erofs_subdirs *, const char *); > 3) update erofs_prepare_dir_file to >erofs_prepare_dir_file(struct erofs_inode *, struct erofs_subdir *). > > Yet I'd like to apply the current solution first since it helps the > dir creation performance. If someone has interest to the solution > above, new cleanup is always welcomed. > > Reviewed-by: Gao Xiang > ok, It is also good Reviewed-by: Li Guifu Thanks,
Re: [PATCH v2] erofs-utils: use qsort() to sort dir->i_subdirs
Hu Weiwen
It really do a high sort performance increase,
I have a idea that keeping the erofs_prepare_dir_file() function
paramter stable, Using a independent function to do dirs sort.
On 2021/4/5 17:38, Hu Weiwen wrote:
> Original implementation use insertion sort, and its time complexity is
> O(n^2). This patch use qsort instead. When I create a directory with
> 100k entries, this reduces the user space time from around 3 mins to
> 0.5s.
>
> Create such a large directory for benchmark with:
> mkdir large; cd large; touch $(seq 10);
>
> Signed-off-by: Hu Weiwen
> ---
> lib/inode.c | 53 +
> 1 file changed, 33 insertions(+), 20 deletions(-)
>
> diff --git a/lib/inode.c b/lib/inode.c
> index d52facf..ef55e88 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -96,21 +96,6 @@ unsigned int erofs_iput(struct erofs_inode *inode)
> return 0;
> }
>
> -static int dentry_add_sorted(struct erofs_dentry *d, struct list_head *head)
> -{
> - struct list_head *pos;
> -
> - list_for_each(pos, head) {
> - struct erofs_dentry *d2 =
> - container_of(pos, struct erofs_dentry, d_child);
> -
> - if (strcmp(d->name, d2->name) < 0)
> - break;
> - }
> - list_add_tail(>d_child, pos);
> - return 0;
> -}
> -
> struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
> const char *name)
> {
> @@ -122,7 +107,7 @@ struct erofs_dentry *erofs_d_alloc(struct erofs_inode
> *parent,
> strncpy(d->name, name, EROFS_NAME_LEN - 1);
> d->name[EROFS_NAME_LEN - 1] = '\0';
>
> - dentry_add_sorted(d, >i_subdirs);
> + list_add_tail(>d_child, >i_subdirs);
> return d;
> }
>
> @@ -156,10 +141,19 @@ static int __allocate_inode_bh_data(struct erofs_inode
> *inode,
> return 0;
> }
>
> +static int comp_subdir(const void *a, const void *b)
> +{
> + const struct erofs_dentry *da, *db;
> +
> + da = *((const struct erofs_dentry **)a);
> + db = *((const struct erofs_dentry **)b);
> + return strcmp(da->name, db->name);
> +}
> +
> -int erofs_prepare_dir_file(struct erofs_inode *dir)
> +int erofs_prepare_dir_file(struct erofs_inode *dir, unsigned int nr_subdirs)
Todo 1: keep these function parameter stable
> {
> - struct erofs_dentry *d;
> - unsigned int d_size, i_nlink;
> + struct erofs_dentry *d, *n, **sorted_d;
> + unsigned int d_size, i_nlink, i;
> int ret;
>
> /* dot is pointed to the current dir inode */
> @@ -172,6 +166,22 @@ int erofs_prepare_dir_file(struct erofs_inode *dir)
> d->inode = erofs_igrab(dir->i_parent);
> d->type = EROFS_FT_DIR;
>
> + /* sort subdirs */
> + nr_subdirs += 2;
> + sorted_d = malloc(nr_subdirs * sizeof(d));
> + if (!sorted_d)
> + return -ENOMEM;
> + i = 0;
> + list_for_each_entry_safe(d, n, >i_subdirs, d_child) {
> + list_del(>d_child);
> + sorted_d[i++] = d;
> + }
> + DBG_BUGON(i != nr_subdirs);
> + qsort(sorted_d, nr_subdirs, sizeof(d), comp_subdir);
> + for (i = 0; i < nr_subdirs; i++)
> + list_add_tail(_d[i]->d_child, >i_subdirs);
> + free(sorted_d);
> +
Todo 2: make these codes refact to a independent function
> /* let's calculate dir size and update i_nlink */
> d_size = 0;
> i_nlink = 0;
> @@ -922,6 +932,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct
> erofs_inode *dir)
> DIR *_dir;
> struct dirent *dp;
> struct erofs_dentry *d;
> + unsigned int nr_subdirs;
>
> ret = erofs_prepare_xattr_ibody(dir);
> if (ret < 0)
> @@ -961,6 +972,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct
> erofs_inode *dir)
> return ERR_PTR(-errno);
> }
>
> + nr_subdirs = 0;
> while (1) {
> /*
>* set errno to 0 before calling readdir() in order to
> @@ -984,6 +996,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct
> erofs_inode *dir)
> ret = PTR_ERR(d);
> goto err_closedir;
> }
> + nr_subdirs++;
>
> /* to count i_nlink for directories */
> d->type = (dp->d_type == DT_DIR ?
> @@ -996,7 +1009,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct
> erofs_inode *dir)
> }
> closedir(_dir);
>
> - ret = erofs_prepare_dir_file(dir);
> + ret = erofs_prepare_dir_file(dir, nr_subdirs);
Todo 3 : nr_subdirs may not be needed, it can be get from one
for-loop-count fixly.
If it may cause perfermance decrease, try to add a dir count in the
erofs_inode struct
> if (ret)
> goto err;
>
> --
> 2.25.1
>
Re: [PATCH v2] erofs-utils: add cmd argument to override uid/gid
On 2021/4/1 20:01, Gao Xiang wrote:
> On Thu, Apr 01, 2021 at 07:36:10PM +0800, Hu Weiwen wrote:
>> Also added '--all-root' option to set uid and gid to root conveniently.
>>
>> This function can be useful if we want to pack some data owned by user with
>> large uid, but we want to use compact inode.
>>
>> This interface mimics that of 'mksquashfs'.
>>
>> Signed-off-by: Hu Weiwen
>
> Yey! I've applied with the following modification,
>
> diff --git a/include/erofs/config.h b/include/erofs/config.h
> index e6eaef66b91c..15390f4ca9c8 100644
> --- a/include/erofs/config.h
> +++ b/include/erofs/config.h
> @@ -54,8 +54,7 @@ struct erofs_configure {
> /* < 0, xattr disabled and INT_MAX, always use inline xattrs */
> int c_inline_xattr_tolerance;
> u64 c_unix_timestamp;
> - u32 c_uid;
> - u32 c_gid;
> + u32 c_uid, c_gid;
> #ifdef WITH_ANDROID
> char *mount_point;
> char *target_out_path;
> diff --git a/mkfs/main.c b/mkfs/main.c
> index 72b7f17e1c66..d8823b539194 100644
> --- a/mkfs/main.c
> +++ b/mkfs/main.c
> @@ -77,8 +77,8 @@ static void usage(void)
> #ifdef HAVE_LIBSELINUX
> " --file-contexts=X specify a file contexts file to setup
> selinux labels\n"
> #endif
> - " --force-uid=UIDset all file uids to UID\n"
> - " --force-gid=GIDset all file gids to GID\n"
> + " --force-uid=# set all file uids to # (# = UID)\n"
> + " --force-gid=# set all file gids to # (# = GID)\n"
> " --all-root make all files owned by root\n"
> " --help display this help and exit\n"
> #ifdef WITH_ANDROID
>
> Otherwise looks good to me,
> Reviewed-by: Gao Xiang
>
It looks good
Reviewed-by: Li Guifu
Thanks,
Re: [PATCH v2] erofs-utils: fuse: fix random readlink error
On 2021/2/10 3:38, Gao Xiang via Linux-erofs wrote:
> Hi Weiwen,
>
> On Sat, Jan 30, 2021 at 02:07:47AM +0800, Hu Weiwen wrote:
>> readlink should fill a **null-terminated** string in the buffer.
>>
>> To achieve this:
>> 1) memset(0) for unmapped extents;
>> 2) make erofsfuse_read() properly returning the actual bytes read;
>> 3) insert a null character if the path is truncated.
>>
>> Link:
>> https://lore.kernel.org/r/20210121101233.gc6...@desktop-n4cecto.huww98.cn
>
> Looked into this patch just now...
>
> The Link tag is only used for refering to the patch itself.
>
>> Signed-off-by: Hu Weiwen
>> ---
>
> ...
>
>>
>> @@ -91,9 +92,13 @@ static int erofs_read_raw_data(struct erofs_inode *inode,
>> char *buffer,
>>
>> if (!(map.m_flags & EROFS_MAP_MAPPED)) {
>> if (!map.m_llen) {
>> +/* reached EOF */
>> +memset(buffer + ptr - offset, 0,
>> + offset + size - ptr);
>> ptr = offset + size;
>> continue;
>> }
>> +memset(buffer + map.m_la - offset, 0, map.m_llen);
>
> There might be some minor issue --- `offset' could be larger than
> `map.m_la' if sparse file is supported then.
>
> I made an update version of this to fix these (some cleanup is
> included as well), it would be nice of you to take another look at
> this as well...
>
> Thanks,
> Gao Xiang
>
> From bfbd8ee056aca57a77034b8723f3f828f806747b Mon Sep 17 00:00:00 2001
> From: Hu Weiwen
> Date: Sat, 30 Jan 2021 02:07:47 +0800
> Subject: [PATCH v3] erofs-utils: fuse: fix random readlink error
>
> readlink should fill a **null-terminated** string in the buffer [1].
>
> To achieve this:
> 1) memset(0) for unmapped extents;
> 2) make erofsfuse_read() properly returning the actual bytes read;
> 3) insert a null character if the path is truncated.
>
> [1] https://lore.kernel.org/r/20210121101233.gc6...@desktop-n4cecto.huww98.cn
> Signed-off-by: Hu Weiwen
> Signed-off-by: Gao Xiang
> ---
> fuse/main.c | 8
> lib/data.c | 20
> 2 files changed, 20 insertions(+), 8 deletions(-)
>
It looks good
Reviewed-by: Li Guifu
Thanks,
Re: [PATCH v2 2/2] erofs-utils: more sanity check for buffer allocation optimization
On 2021/2/14 23:35, Gao Xiang via Linux-erofs wrote: > From: Gao Xiang > > In case that new buffer allocation optimization logic is > potentially broken. > > Signed-off-by: Gao Xiang > --- It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH v3] erofs-utils: fix battach on full buffer blocks
On 2021/2/15 0:00, Gao Xiang via Linux-erofs wrote: > From: Hu Weiwen > > When the subsequent erofs_battach() is called on an buffer block of > which (bb->buffers.off % EROFS_BLKSIZ == 0), `tail_blkaddr' won't be > updated correctly. This bug can be reproduced by: > > mkdir bug-repo > head -c 4032 /dev/urandom > bug-repo/1 > head -c 4095 /dev/urandom > bug-repo/2 > head -c 12345 /dev/urandom > bug-repo/3 # arbitrary size > mkfs.erofs -Eforce-inode-compact bug-repo.erofs.img bug-repo > Then mount this image and see that file `3' in the image is different > from `bug-repo/3'. > > This patch fix this by: > * Handle `oob' and `tail_blkaddr' for the case above properly; > * Don't inline tail-packing data for such case, since the tail-packing >data is actually in a different block from inode itself even kernel >can handle such cases properly. > > Signed-off-by: Hu Weiwen > Reviewed-by: Gao Xiang > Signed-off-by: Gao Xiang > --- It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH v7 3/3] erofs-utils: optimize buffer allocation logic
On 2021/1/23 1:11, Gao Xiang via Linux-erofs wrote: > From: Hu Weiwen > > When using EROFS to pack our dataset which consists of millions of > files, mkfs.erofs is very slow compared with mksquashfs. > > The bottleneck is `erofs_balloc' and `erofs_mapbh' function, which > iterate over all previously allocated buffer blocks, making the > complexity of the algrithm O(N^2) where N is the number of files. > > With this patch: > > * global `last_mapped_block' is mantained to avoid full scan in > `erofs_mapbh` function. > > * global `mapped_buckets' maintains a list of already mapped buffer > blocks for each type and for each possible used bytes in the last > EROFS_BLKSIZ. Then it is used to identify the most suitable blocks in > future `erofs_balloc', avoiding full scan. Note that not-mapped (and the > last mapped) blocks can be expended, so we deal with them separately. > > When I test it with ImageNet dataset (1.33M files, 147GiB), it takes > about 4 hours. Most time is spent on IO. > > Cc: Huang Jianan > Signed-off-by: Hu Weiwen > Signed-off-by: Gao Xiang > --- > include/erofs/cache.h | 1 + > lib/cache.c | 105 -- > 2 files changed, 93 insertions(+), 13 deletions(-) > It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH v7 2/3] erofs-utils: introduce erofs_bfind_for_attach()
On 2021/1/23 1:11, Gao Xiang via Linux-erofs wrote: > From: Gao Xiang > > Seperate erofs_balloc() to make the logic more clearer. > > Cc: Hu Weiwen > Signed-off-by: Gao Xiang > --- > lib/cache.c | 81 + > 1 file changed, 50 insertions(+), 31 deletions(-) > It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH v7 1/3] erofs-utils: get rid of `end' argument from erofs_mapbh()
On 2021/1/23 1:11, Gao Xiang via Linux-erofs wrote: > From: Hu Weiwen > > `end` arguement is completely broken now. Also, it could > be reintroduced later if needed. > > Signed-off-by: Hu Weiwen > Signed-off-by: Gao Xiang > --- > include/erofs/cache.h | 2 +- > lib/cache.c | 6 ++ > lib/compress.c| 2 +- > lib/inode.c | 10 +- > lib/xattr.c | 2 +- > mkfs/main.c | 2 +- > 6 files changed, 11 insertions(+), 13 deletions(-) > It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH v3] erofs-utils: fix memory leak when erofs_fill_inode() fails
On 2021/1/22 0:21, Hu Weiwen wrote: > Signed-off-by: Hu Weiwen > --- > lib/inode.c | 4 +++- > 1 file changed, 3 insertions(+), 1 deletion(-) > It looks good Reviewed-by: Li Guifu
Re: [PATCH] erofs-utils: fix BUILD_BUG_ON
On 2021/1/31 17:47, Hu Weiwen wrote: > Signed-off-by: Hu Weiwen > --- > include/erofs/defs.h | 2 +- > 1 file changed, 1 insertion(+), 1 deletion(-) > It looks good Reviewed-by: Li Guifu
Re: [PATCH v2] AOSP: erofs-utils: fix sub-directory prefix for canned fs_config
On 2020/12/28 18:51, Gao Xiang wrote:
> From: Gao Xiang
>
> "failed to find [%s] in canned fs_config" was observed by using
> "--fs-config-file" option as reported by Yue Hu [1].
>
> The root cause was that the mountpoint prefix to subdirectories is
> also needed if "--mount-point" presents. However, such prefix cannot
> be added by just using erofs_fspath().
>
> One exception is that the root directory itself needs to be handled
> specially for canned fs_config. For such case, the prefix of the root
> directory has to be dropped instead.
>
> [1] https://lkml.kernel.org/r/20201222020430.12512-1-zbest...@gmail.com
>
> Link: https://lore.kernel.org/r/20201226062736.29920-1-hsiang...@aol.com
> Fixes: 8a9e8046f170 ("AOSP: erofs-utils: add fs_config support")
> Reported-by: Yue Hu
> Signed-off-by: Gao Xiang
> ---
> changes since v2:
> - fix IS_ROOT misuse reported by Jianan, very sorry about this since
>I know little about canned fs_config.
>
> (please kindly test again...)
>
> lib/inode.c | 39 +++++++++--
> 1 file changed, 25 insertions(+), 14 deletions(-)
>
It looks good
Reviewed-by: Li Guifu
Thanks,
Re: [PATCH v2] AOSP: erofs-utils: fix sub-directory prefix for canned fs_config
Thank for your works, Yue Hu, Jianan, Gao Xiang
On 2020/12/28 18:51, Gao Xiang wrote:
> From: Gao Xiang
>
> "failed to find [%s] in canned fs_config" was observed by using
> "--fs-config-file" option as reported by Yue Hu [1].
>
> The root cause was that the mountpoint prefix to subdirectories is
> also needed if "--mount-point" presents. However, such prefix cannot
> be added by just using erofs_fspath().
>
> One exception is that the root directory itself needs to be handled
> specially for canned fs_config. For such case, the prefix of the root
> directory has to be dropped instead.
>
> [1] https://lkml.kernel.org/r/20201222020430.12512-1-zbest...@gmail.com
>
> Link: https://lore.kernel.org/r/20201226062736.29920-1-hsiang...@aol.com
> Fixes: 8a9e8046f170 ("AOSP: erofs-utils: add fs_config support")
> Reported-by: Yue Hu
> Signed-off-by: Gao Xiang
> ---
> changes since v2:
> - fix IS_ROOT misuse reported by Jianan, very sorry about this since
>I know little about canned fs_config.
>
> (please kindly test again...)
>
> lib/inode.c | 39 +--
> 1 file changed, 25 insertions(+), 14 deletions(-)
>
> diff --git a/lib/inode.c b/lib/inode.c
> index 0c4839d..e6159c9 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -696,32 +696,43 @@ int erofs_droid_inode_fsconfig(struct erofs_inode
> *inode,
> /* filesystem_config does not preserve file type bits */
> mode_t stat_file_type_mask = st->st_mode & S_IFMT;
> unsigned int uid = 0, gid = 0, mode = 0;
> - char *fspath;
> + const char *fspath;
> + char *decorated = NULL;
>
> inode->capabilities = 0;
> + if (!cfg.fs_config_file && !cfg.mount_point)
> + return 0;
> +
> + if (!cfg.mount_point ||
> + /* have to drop the mountpoint for rootdir of canned fsconfig */
> + (cfg.fs_config_file && erofs_fspath(path)[0] == '\0')) {
> + fspath = erofs_fspath(path);
> + } else {
> + if (asprintf(, "%s/%s", cfg.mount_point,
> + erofs_fspath(path)) <= 0)
> + return -ENOMEM;
> + fspath = decorated;
> + }
> +
erofs_fspath has been written for three times, and called always.
What do you think refact it ? Do it with
fspath = erofs_fspath(path);
if need decorated:
fspath = decorated
Re: [PATCH] erofs-utils: fuse: disable backtrace if unsupported
On 2020/12/17 17:06, Gao Xiang via Linux-erofs wrote:
> From: Gao Xiang
>
> Let's enable backtrace conditionally since it's a GNU extension.
>
> Fixes: 5e35b75ad499 ("erofs-utils: introduce fuse implementation")
> Signed-off-by: Gao Xiang
> ---
> configure.ac | 3 ++-
> fuse/main.c | 8 ++--
> 2 files changed, 8 insertions(+), 3 deletions(-)
>
It looks good
Reviewed-by: Li Guifu
Thanks,
Re: [PATCH] erofs-utils: fix multiple definition of `sbi'
On 2020/12/9 7:24, Gao Xiang wrote: > On Tue, Dec 08, 2020 at 06:16:57PM +0200, nl6720 wrote: >> On Tuesday, 8 December 2020 12:57:41 EET Gao Xiang wrote: >>> As nl6720 reported [1], lib/inode.o (mkfs) and lib/super.o (erofsfuse) >>> could be compiled together by some options. Fix it now. >>> >>> [1] https://lore.kernel.org/r/10789285.Na0ui7I3VY@walnut >>> Reported-by: nl6720 >>> Signed-off-by: Gao Xiang >>> --- It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH] erofs-utils: fuse: fix linking when using --with-selinux
On 2020/12/11 10:29, David Michael wrote: > The libselinux functions selabel_open and selabel_close are called > by lib/config.c, so include libselinux in CFLAGS and LIBS to fix > building erofsfuse. > > Signed-off-by: David Michael > --- > > Hi, > > Trying to build both mkfs.erofs with SELinux and erofsfuse at the same > time (with both --enable-fuse and --with-selinux) results in the > following linking errors: > > /usr/bin/ld: ../lib/.libs/liberofs.a(liberofs_la-config.o): in function > `erofs_selabel_open': > /home/dm0/rpmbuild/BUILD/erofs-utils-1.2/lib/config.c:75: undefined reference > to `selabel_open' > /usr/bin/ld: ../lib/.libs/liberofs.a(liberofs_la-config.o): in function > `erofs_exit_configure': > /home/dm0/rpmbuild/BUILD/erofs-utils-1.2/lib/config.c:42: undefined reference > to `selabel_close' > > Are these programs supposed to be configured separately? If this build > configuration is supposed to work, this change fixes linking. > > Thanks. > > David > > fuse/Makefile.am | 4 ++-- > 1 file changed, 2 insertions(+), 2 deletions(-) > It's fixed Reviewed-by: Li Guifu Tested-by: Li Guifu Thanks,
Re: [PATCH] erofs-utils: mkfs: fix uuid.h location
On 2020/12/9 8:49, Gao Xiang via Linux-erofs wrote:
> From: Gao Xiang
>
> As Karel reported [1], "The subdirectory in
> #include
>
> is unnecessary (or wrong), if you use
> PKG_CHECK_MODULES([libuuid], [uuid])
>
> than it returns the subdirectory as -I, see
>
> $ pkg-config --cflags uuid
> -I/usr/include/uuid
>
> so the correct way is
> #include ". Let's fix it now!
>
> [1] https://lore.kernel.org/r/20201208100910.dqqh5cqihewky...@ws.net.home
>
> Reported-by: Karel Zak
> Fixes: e023d47593ff ("erofs-utils: support 128-bit filesystem UUID")
> Signed-off-by: Gao Xiang
> ---
> mkfs/main.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
It looks good
Reviewed-by: Li Guifu
Thanks,
Re: [PATCH v2] erofs-utils: update i_nlink stat for directories
On 2020/12/5 17:16, Gao Xiang wrote: > From: Gao Xiang > > Previously, nlink of directories was treated as 1 for simplicity. > > Since st_nlink for dirs is actually not well defined, nlink=1 seems > to pacify `find' (even without -noleaf option) and other utilities. > AFAICT, isofs, romfs and cramfs always set it to 1, Overlayfs sets > it to 1 conditionally, btrfs[1], ceph[2] and FUSE client historically > set it to 1. > > The convention under unix is that it's # of subdirs including "." > and "..". This patch tries to follow such convention if possible to > optimize `find' performance since it's not quite hard for local fs. > > [1] https://lore.kernel.org/r/20100124003336.GP23006@think > [2] https://lore.kernel.org/r/20180521092729.17470-1-lhenriq...@suse.com > Signed-off-by: Gao Xiang > --- > v1: https://lore.kernel.org/r/20201205055732.14276-1-hsiang...@aol.com > changes since v1: > - update a DBG_BUGON statement suggestted by Guifu. > > lib/inode.c | 33 +---- > 1 file changed, 29 insertions(+), 4 deletions(-) > It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH] erofs-utils: update i_nlink stat for directories
On 2020/12/5 16:43, Gao Xiang wrote: > On Sat, Dec 05, 2020 at 04:38:37PM +0800, Gao Xiang wrote: >> On Sat, Dec 05, 2020 at 04:32:44PM +0800, Li GuiFu via Linux-erofs wrote: >> >> ... >> >>> >>>> @@ -957,6 +974,10 @@ struct erofs_inode *erofs_mkfs_build_tree(struct >>>> erofs_inode *dir) >>>>ret = PTR_ERR(d); >>>>goto err_closedir; >>>>} >>>> + >>>> + /* to count i_nlink for directories */ >>>> + d->type = (dp->d_type == DT_DIR ? >>>> + EROFS_FT_DIR : EROFS_FT_UNKNOWN); >>>>} >>>> >>> It's confused that d->type was set to EROFS_FT_UNKNOWN when not a dir >>> It's not clearness whether the program goes wrong or get the wrong data >>> Actually it's a correct procedure >> >> It's just set temporarily, since only dirs are useful when counting subdirs, >> so >> only needs to differ dirs and non-dirs here. (Previously d->type is unused >> at this time.) > > btw, I once tried to set up d->type via dp->d_type here, but it increases a > lot of code and seems unnecessary (since deriving from i_mode is enough). > So again, here we only cares about dir and non-dirs (we don't care much about > the specific kind of non-dirs here). > > Thanks, > Gao Xiang > >> >> ... >> >>>> - d->type = erofs_type_by_mode[d->inode->i_mode >> S_SHIFT]; >>>> + ftype = erofs_mode_to_ftype(d->inode->i_mode); >>>> + DBG_BUGON(d->type != EROFS_FT_UNKNOWN && d->type != ftype); >>>> + d->type = ftype; >> >> The real on-disk d->type will be set here rather than the above. Yes, what it makes confused is here, EROFS_FT_UNKNOWN is just temporary. So how about change to ASSERT at EROFS_FT_DIR DBG_BUGON(d->type == EROFS_FT_DIR && ftype != EROFS_FT_DIR);
Re: [PATCH v4 3/3] erofs-utils: fuse: add compressed file support
On 2020/11/27 20:24, Gao Xiang via Linux-erofs wrote: > From: Huang Jianan > > This patch adds a simple approach (~ 700 LOC) to EROFS fixed-sized > output decompression without inplace I/O or decompression inplace > so it's easy to be ported everywhere with less constraint. > > However, the on-disk compressed index parser (aka. zmap) is largely > kept in line with the kernel side, therefore new on-disk features > under development can be verified effectively first here. > > Signed-off-by: Huang Jianan > Signed-off-by: Guo Weichao > Signed-off-by: Gao Xiang > --- > changes since v3: > - fix build error without lz4 library. > > include/erofs/decompress.h | 35 > include/erofs/defs.h | 5 + > include/erofs/internal.h | 8 + > lib/Makefile.am| 2 +- > lib/data.c | 72 ++- > lib/decompress.c | 87 > lib/namei.c| 4 + > lib/zmap.c | 415 + > 8 files changed, 626 insertions(+), 2 deletions(-) > create mode 100644 include/erofs/decompress.h > create mode 100644 lib/decompress.c > create mode 100644 lib/zmap.c > It looks good Reviewed-by: Li Guifu Tested-by: Li Guifu Thanks,
Re: [PATCH v3 2/3] erofs-utils: fuse: support symlink & special inode
On 2020/11/27 19:46, Gao Xiang via Linux-erofs wrote: > From: Huang Jianan > > This patch adds symlink and special inode (e.g. block dev, char, > socket, pipe inode) support. > > Signed-off-by: Huang Jianan > Signed-off-by: Guo Weichao > Signed-off-by: Gao Xiang > --- > fuse/main.c | 10 ++ > lib/namei.c | 22 ++ > 2 files changed, 28 insertions(+), 4 deletions(-) > It looks good Reviewed-by: Li Guifu Tested-by: Li Guifu Thanks,
Re: [PATCH] erofs-utils: update i_nlink stat for directories
On 2020/12/5 13:57, Gao Xiang via Linux-erofs wrote:
> From: Gao Xiang
>
> Previously, nlink of directories is treated as 1 for simplicity.
>
> Since st_nlink for dirs is actualy not well defined, nlink=1 seems
> to pacify `find' (even without -noleaf option) and other utilities.
> AFAICT, isofs, romfs and cramfs always set it to 1, Overlayfs sets
> it to 1 conditionally, btrfs[1], ceph[2] and FUSE client historically
> set it to 1.
>
> The convention under unix is that it's # of subdirs including "."
> and "..". This patch tries to follow such convention if possible to
> optimize `find' performance since it's not quite hard for local fs.
>
> [1] https://lore.kernel.org/r/20100124003336.GP23006@think
> [2] https://lore.kernel.org/r/20180521092729.17470-1-lhenriq...@suse.com
> Signed-off-by: Gao Xiang
> ---
> lib/inode.c | 33 +
> 1 file changed, 29 insertions(+), 4 deletions(-)
>
> diff --git a/lib/inode.c b/lib/inode.c
> index 618eb284550f..357ac480154a 100644
> --- a/lib/inode.c
> +++ b/lib/inode.c
> @@ -25,7 +25,7 @@
> struct erofs_sb_info sbi;
>
> @@ -957,6 +974,10 @@ struct erofs_inode *erofs_mkfs_build_tree(struct
> erofs_inode *dir)
> ret = PTR_ERR(d);
> goto err_closedir;
> }
> +
> + /* to count i_nlink for directories */
> + d->type = (dp->d_type == DT_DIR ?
> + EROFS_FT_DIR : EROFS_FT_UNKNOWN);
> }
>
It's confused that d->type was set to EROFS_FT_UNKNOWN when not a dir
It's not clearness whether the program goes wrong or get the wrong data
Actually it's a correct procedure
> if (errno) {
> @@ -978,6 +999,7 @@ struct erofs_inode *erofs_mkfs_build_tree(struct
> erofs_inode *dir)
>
> list_for_each_entry(d, >i_subdirs, d_child) {
> char buf[PATH_MAX];
> + unsigned char ftype;
>
> if (is_dot_dotdot(d->name)) {
> erofs_d_invalidate(d);
> @@ -1000,7 +1022,10 @@ fail:
> goto err;
> }
>
> - d->type = erofs_type_by_mode[d->inode->i_mode >> S_SHIFT];
> + ftype = erofs_mode_to_ftype(d->inode->i_mode);
> + DBG_BUGON(d->type != EROFS_FT_UNKNOWN && d->type != ftype);
> + d->type = ftype;
> +
> erofs_d_invalidate(d);
> erofs_info("add file %s/%s (nid %llu, type %d)",
> dir->i_srcpath, d->name, (unsigned long long)d->nid,
>
Re: [PATCH v2 2/2] erofs-utils: fix cross-device submounts
On 2020/12/5 1:56, Gao Xiang via Linux-erofs wrote:
> From: Gao Xiang
>
> Use device ID and inode number to identify hardlinks
> rather than inode number only.
>
> Fixes: a17497f0844a ("erofs-utils: introduce inode operations")
> Signed-off-by: Gao Xiang
> ---
> changes since v1:
> - fix improper inline comment update;
>
> include/erofs/internal.h | 7 ++-
> lib/inode.c | 14 --
> 2 files changed, 14 insertions(+), 7 deletions(-)
>
It looks good
Reviewed-by: Li Guifu
Thanks,
Re: [PATCH 1/2] erofs-utils: don't create hardlinked directories
On 2020/12/5 1:20, Gao Xiang via Linux-erofs wrote:
> From: Gao Xiang
>
> From: Gao Xiang
>
> Fix an issue which behaves the same as the following
> mkisofs BZ due to bindmount:
> https://bugzilla.redhat.com/show_bug.cgi?id=1749860
>
> Fixes: a17497f0844a ("erofs-utils: introduce inode operations")
> Signed-off-by: Gao Xiang
> ---
> regression testcases will be uploaded to:
> https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git/log/?h=experimental-tests
> for now (erofs-utils v1.3 will include testcases then.)
>
> lib/inode.c | 13 ++---
> 1 file changed, 10 insertions(+), 3 deletions(-)
>
It looks good
Reviewed-by: Li Guifu
Thanks,
Re: [PATCH] erofs-utils: update .gitignore
On 2020/12/2 17:53, Gao Xiang via Linux-erofs wrote: > Add more extensions to .gitignore. > > Signed-off-by: Gao Xiang > --- > .gitignore | 9 +++-- > 1 file changed, 7 insertions(+), 2 deletions(-) > It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH] erofs-utils: xattr: fix OOB access due to alignment
On 2020/11/27 22:33, Gao Xiang wrote:
> From: Gao Xiang
>
> erofs_buf_write_bhops can only be safely used for block-aligned
> buffers, otherwise, it could write random out-of-bound data due
> to buffer alignment. Such random data is meaningless but it does
> harm to reproducable builds.
>
> Fixes: 116ac0a254fc ("erofs-utils: introduce shared xattr support")
> Reported-by: Huang Jianan
> Signed-off-by: Gao Xiang
> ---
> lib/xattr.c | 18 +-
> 1 file changed, 17 insertions(+), 1 deletion(-)
>
It looks good
Reviewed-by: Li Guifu
Thanks,
[PATCH v2] erofs-utils: stop build tree if file fails to open
stop and exit immediately if it fails to open a file, e.g mkfs.erofs
doesn't run under the root user (e.g. run without sudo.)
Signed-off-by: Li Guifu
---
lib/inode.c | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/lib/inode.c b/lib/inode.c
index fee5c96..eb2e0f2 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -908,7 +908,9 @@ struct erofs_inode *erofs_mkfs_build_tree(struct
erofs_inode *dir)
if (ret)
return ERR_PTR(ret);
} else {
- erofs_write_file(dir);
+ ret = erofs_write_file(dir);
+ if (ret)
+ return ERR_PTR(ret);
}
erofs_prepare_inode_buffer(dir);
@@ -982,10 +984,11 @@ struct erofs_inode *erofs_mkfs_build_tree(struct
erofs_inode *dir)
d->inode = erofs_mkfs_build_tree_from_path(dir, buf);
if (IS_ERR(d->inode)) {
+ ret = PTR_ERR(d->inode);
fail:
d->inode = NULL;
d->type = EROFS_FT_UNKNOWN;
- continue;
+ goto err_closedir;
}
d->type = erofs_type_by_mode[d->inode->i_mode >> S_SHIFT];
--
2.17.1
Re: [PATCH 1/3] erofs-utils: introduce fuse implementation
Gao Xiang
It run good, some codes format need to be adjusted
Please re-send to make more readable
On 2020/11/21 1:41, Gao Xiang via Linux-erofs wrote:
> From: Li Guifu
>
> Let's add a simple erofsfuse approach, and benefits are:
>
> - images can be supported on various platforms;
> - new unpack tool can be developed based on this;
> - new on-disk features can be iterated, verified effectively.
>
> This commit only addresses reading uncompressed regular files.
> Other file (e.g. compressed file) support is out of scope here.
>
> Note that erofsfuse is an unstable feature for now (also notice
> LZ4_decompress_safe_partial() was broken in lz4-1.9.2, which
> just fixed in lz4-1.9.3), let's disable it by default for a while.
>
> Signed-off-by: Li Guifu
> Signed-off-by: Huang Jianan
> Signed-off-by: Guo Weichao
> Signed-off-by: Gao Xiang
> ---
> +
> +static void signal_handle_sigsegv(int signal)
> +{
> + void *array[10];
> + size_t nptrs;
> + char **strings;
> + size_t i;
> +
> + erofs_dump("\n");
> + erofs_dump("Segmentation Fault. Starting backtrace:\n");
> + nptrs = backtrace(array, 10);
> + strings = backtrace_symbols(array, nptrs);
> + if (strings) {
> + for (i = 0; i < nptrs; i++)
> + erofs_dbg("%s", strings[i]);
erofs_dbg change to erofs_dump to make all log printed, "\n" is needed
> + free(strings);
> + }
> + erofs_dump("\n");
> + abort();
> +}
> +
> +
>
> +static inline unsigned int erofs_bitrange(unsigned int value, unsigned int
> bit,
> + unsigned int bits)
> +{
> +
empty line is not needed
> + return (value >> bit) & ((1 << bits) - 1);
> +}
> +
> +
empty line is not needed
> +static inline unsigned int erofs_inode_version(unsigned int value)
> +{
> + return erofs_bitrange(value, EROFS_I_VERSION_BIT,
> + EROFS_I_VERSION_BITS);
> +}
> +
> +static inline unsigned int erofs_inode_datalayout(unsigned int value)
> +{
> + return erofs_bitrange(value, EROFS_I_DATALAYOUT_BIT,
> + EROFS_I_DATALAYOUT_BITS);
> +}
> +
[PATCH] erofs-utils: stop mkfs when access permission denied
It would not has the permission to access one file when mkfs.erofs
was not run in the root mode, eg run without sudo, So stop and
exit immediately
Signed-off-by: Li Guifu
---
lib/inode.c | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/lib/inode.c b/lib/inode.c
index fee5c96..4641561 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -364,6 +364,7 @@ int erofs_write_file(struct erofs_inode *inode)
}
/* fallback to all data uncompressed */
+ errno = 0;
fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
if (fd < 0)
return -errno;
@@ -908,7 +909,9 @@ struct erofs_inode *erofs_mkfs_build_tree(struct
erofs_inode *dir)
if (ret)
return ERR_PTR(ret);
} else {
- erofs_write_file(dir);
+ ret = erofs_write_file(dir);
+ if (ret)
+ return ERR_PTR(ret);
}
erofs_prepare_inode_buffer(dir);
@@ -982,10 +985,11 @@ struct erofs_inode *erofs_mkfs_build_tree(struct
erofs_inode *dir)
d->inode = erofs_mkfs_build_tree_from_path(dir, buf);
if (IS_ERR(d->inode)) {
+ ret = PTR_ERR(d->inode);
fail:
d->inode = NULL;
d->type = EROFS_FT_UNKNOWN;
- continue;
+ goto err_closedir;
}
d->type = erofs_type_by_mode[d->inode->i_mode >> S_SHIFT];
--
2.17.1
Re: [PATCH v2 1/2] erofs-utils: drop known issue in README
On 2020/11/21 10:26, Gao Xiang via Linux-erofs wrote: > From: Gao Xiang > > Since lz4-1.9.3 has been released, > https://github.com/lz4/lz4/releases/tag/v1.9.3 > > Move this lz4hc issue (lz4 <= 1.9.2) to "Comments" instead. > > Signed-off-by: Gao Xiang > --- > v2: fix "lz4 <= 1.8.2" to "lz4 <= 1.9.2" typo. It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH 1/2] erofs-utils: drop known issue in README
On 2020/11/16 20:55, Gao Xiang via Linux-erofs wrote: > From: Gao Xiang > > Since lz4-1.9.3 has been released, > https://github.com/lz4/lz4/releases/tag/v1.9.3 > > Move this lz4hc issue (lz4 <= 1.8.2) to "Comments" instead. > > Signed-off-by: Gao Xiang It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH] erofs-utils: fix the project prefix to "erofs-utils"
On 2020/10/24 18:05, Gao Xiang via Linux-erofs wrote: > Some of them were "erofs_utils" in source headers by mistake. > > Signed-off-by: Gao Xiang > --- It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH 4/4] mkfs: add option to use a pre-defined UUID
On 2020/10/30 20:30, Gao Xiang wrote: > Usage: mkfs.erofs -U > > The filesystem UUID can now be optionally specified during filesystem > creation. The default behavior is still to generate a random UUID. > > This is useful for reproducible builds. > > Signed-off-by: Gao Xiang It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH 3/4] mkfs: introduce erofs_mkfs_default_options()
On 2020/10/30 20:30, Gao Xiang wrote: > Gather all default settings, and generate UUID before > parse_options_cfg(), therefore the UUID can be overridden > later by command line for reproducible images. > > Signed-off-by: Gao Xiang > --- It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH 2/4] erofs-utils: support $SOURCE_DATE_EPOCH
On 2020/10/30 20:30, Gao Xiang wrote: > Currently, we use -T to set a given UNIX timestamp for all > files, yet reproducible builds [1] requires what is called > "timestamp clamping", IOW, a timestamp must be used no later > than the value of this variable. > > Let's support $SOURCE_DATE_EPOCH as well. > > [1] https://reproducible-builds.org/specs/source-date-epoch/ > Suggested-by: nl6720 > Signed-off-by: Gao Xiang It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH 1/4] erofs-utils: fix build error without lz4 library
On 2020/10/30 20:30, Gao Xiang wrote:
> This fixes a recent build error if lz4 library doesn't install,
>
> /bin/sh ../libtool --tag=CC --mode=link gcc -Wall -Werror -I../include -g
> -O2 -o mkfs.erofs mkfs_erofs-main.o -luuid ../lib/liberofs.la -llz4
> libtool: link: gcc -Wall -Werror -I../include -g -O2 -o mkfs.erofs
> mkfs_erofs-main.o -luuid ../lib/.libs/liberofs.a -llz4
> /usr/bin/ld: cannot find -llz4
>
> Fixes: c497d89e5eac ("erofs-utils: enhance static linking for lz4 1.8.x")
> Signed-off-by: Gao Xiang
It looks good
Reviewed-by: Li Guifu
Thanks,
Re: [PATCH] erofs-utils: README: update known-issue status of lz4hc
On 2020/10/12 15:34, Gao Xiang wrote: > Known issue of LZ4_compress_HC_destSize() mentioned in README > was targeted by lz4 upstream days ago. > > Update README so all users can be noticed. > > Signed-off-by: Gao Xiang It looks good thanks Reviewed-by: Li Guifu
Re: [PATCH] AOSP: erofs-utils: update usage due to fs_config
On 2020/10/12 8:38, Gao Xiang wrote:
> After fs_config support is added, usage() should
> be updated together as well.
>
> Fixes: 8a9e8046f170 ("AOSP: erofs-utils: add fs_config support")
> Cc: Li Guifu
> Signed-off-by: Gao Xiang
Reviewed-by: Li Guifu
Re: [PATCH v2] AOSP: erofs-utils: add fs_config support
在 2020/10/10 0:33, Li GuiFu 写道: > > > 在 2020/9/29 13:13, Gao Xiang 写道: >> So that mkfs can directly generate images with fs_config. >> All code for AOSP is wraped up with WITH_ANDROID macro. >> >> Signed-off-by: Gao Xiang >> --- >> changes since v1: >> - fix compile issues on Android / Linux build; >> - tested with Android system booting; >> >> include/erofs/config.h | 12 ++ >> include/erofs/internal.h | 3 +++ >> lib/inode.c | 49 + >> lib/xattr.c | 52 >> mkfs/main.c | 29 +- >> 5 files changed, 144 insertions(+), 1 deletion(-) >> Please update the usage about mount-point
[PATCH v11] erofs-utils: introduce segment compression
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu
---
Changes from v10
- chang variable uncomprofs to clusterofs which only used
when write uncompress block
include/erofs/config.h | 1 +
lib/compress.c | 30 --
lib/config.c | 1 +
man/mkfs.erofs.1 | 4
mkfs/main.c| 16 +++-
5 files changed, 45 insertions(+), 7 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..e5f1bfb 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -35,6 +35,7 @@ struct erofs_configure {
char *c_img_path;
char *c_src_path;
char *c_compr_alg_master;
+ u64 c_compr_seg_size;
int c_compr_level_master;
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..a2a278c 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr;/* pointing to the next blkaddr */
u16 clusterofs;
+ u64 segavail;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -116,7 +117,7 @@ static void vle_write_indexes(struct
z_erofs_vle_compress_ctx *ctx,
}
static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
- unsigned int *len,
+ unsigned int *len, unsigned int *clusterofs,
char *dst)
{
int ret;
@@ -125,14 +126,19 @@ static int write_uncompressed_block(struct
z_erofs_vle_compress_ctx *ctx,
/* reset clusterofs to 0 if permitted */
if (!erofs_sb_has_lz4_0padding() &&
ctx->head >= ctx->clusterofs) {
+ *clusterofs = ctx->clusterofs;
ctx->head -= ctx->clusterofs;
*len += ctx->clusterofs;
ctx->clusterofs = 0;
+ count = min(EROFS_BLKSIZ, *len);
+ } else {
+ *clusterofs = 0;
+ count = min_t(u64, ctx->segavail, *len);
+ if (count > EROFS_BLKSIZ)
+ count = EROFS_BLKSIZ;
}
/* write uncompressed data */
- count = min(EROFS_BLKSIZ, *len);
-
memcpy(dst, ctx->queue + ctx->head, count);
memset(dst + count, 0, EROFS_BLKSIZ - count);
@@ -157,14 +163,21 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ unsigned int clusterofs;
+
+ if (ctx->segavail <= EROFS_BLKSIZ) {
+ if (len < ctx->segavail && !final)
+ break;
+
+ goto nocompression;
+ }
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
break;
}
-
- count = len;
+ count = min_t(u64, len, ctx->segavail);
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
, dst, EROFS_BLKSIZ);
@@ -175,11 +188,12 @@ static int vle_compress_one(struct erofs_inode *inode,
erofs_strerror(ret));
}
nocompression:
- ret = write_uncompressed_block(ctx, , dst);
+ ret = write_uncompressed_block(ctx, , ,
dst);
if (ret < 0)
return ret;
count = ret;
raw = true;
+ ctx->segavail -= count - clusterofs;
} else {
/* write compressed data */
erofs_dbg("Writing %u compressed data to block %u",
@@ -194,6 +208,7 @@ nocompression:
if (ret)
return ret;
raw = false;
+ ctx->segavail -= count;
}
ctx->head += count;
@@ -202,6 +217,8 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ if (!ctx->segavail)
+ ctx->segavail = cfg.c_compr_seg_size;
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
@@ -422,6 +439,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.hea
[PATCH v10] erofs-utils: introduce segment compression
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu
---
include/erofs/config.h | 1 +
lib/compress.c | 29 +++--
lib/config.c | 1 +
man/mkfs.erofs.1 | 4
mkfs/main.c| 16 +++-
5 files changed, 44 insertions(+), 7 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..e5f1bfb 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -35,6 +35,7 @@ struct erofs_configure {
char *c_img_path;
char *c_src_path;
char *c_compr_alg_master;
+ u64 c_compr_seg_size;
int c_compr_level_master;
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..2ea5809 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr;/* pointing to the next blkaddr */
u16 clusterofs;
+ u64 segavail;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -116,7 +117,7 @@ static void vle_write_indexes(struct
z_erofs_vle_compress_ctx *ctx,
}
static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
- unsigned int *len,
+ unsigned int *len, unsigned int *ucomproft,
char *dst)
{
int ret;
@@ -125,14 +126,19 @@ static int write_uncompressed_block(struct
z_erofs_vle_compress_ctx *ctx,
/* reset clusterofs to 0 if permitted */
if (!erofs_sb_has_lz4_0padding() &&
ctx->head >= ctx->clusterofs) {
+ *ucomproft = ctx->clusterofs;
ctx->head -= ctx->clusterofs;
*len += ctx->clusterofs;
ctx->clusterofs = 0;
+ count = min(EROFS_BLKSIZ, *len);
+ } else {
+ *ucomproft = 0;
+ count = min_t(u64, ctx->segavail, *len);
+ if (count > EROFS_BLKSIZ)
+ count = EROFS_BLKSIZ;
}
/* write uncompressed data */
- count = min(EROFS_BLKSIZ, *len);
-
memcpy(dst, ctx->queue + ctx->head, count);
memset(dst + count, 0, EROFS_BLKSIZ - count);
@@ -157,14 +163,21 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ unsigned int ucomproft = 0;
+
+ if (ctx->segavail <= EROFS_BLKSIZ) {
+ if (len < ctx->segavail && !final)
+ break;
+
+ goto nocompression;
+ }
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
break;
}
-
- count = len;
+ count = min_t(u64, len, ctx->segavail);
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
, dst, EROFS_BLKSIZ);
@@ -175,7 +188,7 @@ static int vle_compress_one(struct erofs_inode *inode,
erofs_strerror(ret));
}
nocompression:
- ret = write_uncompressed_block(ctx, , dst);
+ ret = write_uncompressed_block(ctx, , ,
dst);
if (ret < 0)
return ret;
count = ret;
@@ -202,6 +215,9 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ ctx->segavail -= count - ucomproft;
+ if (!ctx->segavail)
+ ctx->segavail = cfg.c_compr_seg_size;
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
@@ -422,6 +438,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.segavail = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..721ff61 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_seg_size = -1;
}
void erofs_show_config(void)
diff --git a/
[PATCH v8] erofs-utils: introduce segment compression
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu
---
include/erofs/config.h | 1 +
lib/compress.c | 47 ++
lib/config.c | 1 +
man/mkfs.erofs.1 | 4
mkfs/main.c| 18 +++-
5 files changed, 52 insertions(+), 19 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..e5f1bfb 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -35,6 +35,7 @@ struct erofs_configure {
char *c_img_path;
char *c_src_path;
char *c_compr_alg_master;
+ u64 c_compr_seg_size;
int c_compr_level_master;
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..6c0708d 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr;/* pointing to the next blkaddr */
u16 clusterofs;
+ u64 segavail;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -116,23 +117,11 @@ static void vle_write_indexes(struct
z_erofs_vle_compress_ctx *ctx,
}
static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
- unsigned int *len,
- char *dst)
+ unsigned int count, char *dst)
{
int ret;
- unsigned int count;
-
- /* reset clusterofs to 0 if permitted */
- if (!erofs_sb_has_lz4_0padding() &&
- ctx->head >= ctx->clusterofs) {
- ctx->head -= ctx->clusterofs;
- *len += ctx->clusterofs;
- ctx->clusterofs = 0;
- }
-
- /* write uncompressed data */
- count = min(EROFS_BLKSIZ, *len);
+ DBG_BUGON(count > EROFS_BLKSIZ);
memcpy(dst, ctx->queue + ctx->head, count);
memset(dst + count, 0, EROFS_BLKSIZ - count);
@@ -157,14 +146,22 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ unsigned int limit = EROFS_BLKSIZ;
+
+ if (ctx->segavail <= EROFS_BLKSIZ) {
+ if (len < ctx->segavail && !final)
+ break;
+
+ limit = ctx->segavail;
+ goto nocompression;
+ }
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
break;
}
-
- count = len;
+ count = min_t(u64, len, ctx->segavail);
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
, dst, EROFS_BLKSIZ);
@@ -175,10 +172,18 @@ static int vle_compress_one(struct erofs_inode *inode,
erofs_strerror(ret));
}
nocompression:
- ret = write_uncompressed_block(ctx, , dst);
+ /* reset clusterofs to 0 if permitted */
+ if (!erofs_sb_has_lz4_0padding() &&
+ ctx->head >= ctx->clusterofs) {
+ ctx->head -= ctx->clusterofs;
+ len += ctx->clusterofs;
+ limit += ctx->clusterofs;
+ ctx->clusterofs = 0;
+ }
+ count = min(limit, len);
+ ret = write_uncompressed_block(ctx, count, dst);
if (ret < 0)
return ret;
- count = ret;
raw = true;
} else {
/* write compressed data */
@@ -203,6 +208,11 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ if (count >= ctx->segavail)
+ ctx->segavail = cfg.c_compr_seg_size;
+ else
+ ctx->segavail -= count;
+
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
round_down(ctx->head, EROFS_BLKSIZ);
@@ -422,6 +432,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.sega
[PATCH v6] erofs-utils: introduce segment limits compression
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu
---
Changes since v3 suggest by Gao Xiang:
- add a limits varialbe to give the limits size in the write_uncompress_block
- Set comments more readable
include/erofs/config.h | 1 +
lib/compress.c | 29 +
lib/config.c | 1 +
man/mkfs.erofs.1 | 4
mkfs/main.c| 18 +-
5 files changed, 44 insertions(+), 9 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..e5f1bfb 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -35,6 +35,7 @@ struct erofs_configure {
char *c_img_path;
char *c_src_path;
char *c_compr_alg_master;
+ u64 c_compr_seg_size;
int c_compr_level_master;
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..8a79895 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr;/* pointing to the next blkaddr */
u16 clusterofs;
+ u64 segavail;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -116,23 +117,21 @@ static void vle_write_indexes(struct
z_erofs_vle_compress_ctx *ctx,
}
static int write_uncompressed_block(struct z_erofs_vle_compress_ctx *ctx,
- unsigned int *len,
+ unsigned int *len, unsigned int limits,
char *dst)
{
int ret;
- unsigned int count;
+ unsigned int count = min(limits, *len); /* write uncompressed data */
/* reset clusterofs to 0 if permitted */
if (!erofs_sb_has_lz4_0padding() &&
ctx->head >= ctx->clusterofs) {
ctx->head -= ctx->clusterofs;
*len += ctx->clusterofs;
+ count += ctx->clusterofs;
ctx->clusterofs = 0;
}
- /* write uncompressed data */
- count = min(EROFS_BLKSIZ, *len);
-
memcpy(dst, ctx->queue + ctx->head, count);
memset(dst + count, 0, EROFS_BLKSIZ - count);
@@ -157,14 +156,22 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ unsigned int limits = EROFS_BLKSIZ;
+
+ if (ctx->segavail <= EROFS_BLKSIZ) {
+ if (len < ctx->segavail && !final)
+ break;
+
+ limits = ctx->segavail;
+ goto nocompression;
+ }
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
break;
}
-
- count = len;
+ count = min_t(u64, len, ctx->segavail);
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
, dst, EROFS_BLKSIZ);
@@ -175,7 +182,7 @@ static int vle_compress_one(struct erofs_inode *inode,
erofs_strerror(ret));
}
nocompression:
- ret = write_uncompressed_block(ctx, , dst);
+ ret = write_uncompressed_block(ctx, , limits, dst);
if (ret < 0)
return ret;
count = ret;
@@ -203,6 +210,11 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ if (count >= ctx->segavail)
+ ctx->segavail = cfg.c_compr_seg_size;
+ else
+ ctx->segavail -= count;
+
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
round_down(ctx->head, EROFS_BLKSIZ);
@@ -422,6 +434,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.segavail = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..fbb2914 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.
[PATCH v5] erofs-utils: introduce segment compression
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu
---
include/erofs/config.h | 1 +
lib/compress.c | 16 ++--
lib/config.c | 1 +
man/mkfs.erofs.1 | 4
mkfs/main.c| 16 +++-
5 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..995664d 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -36,6 +36,7 @@ struct erofs_configure {
char *c_src_path;
char *c_compr_alg_master;
int c_compr_level_master;
+ u64 c_compr_seg_size;
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..383ee00 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr;/* pointing to the next blkaddr */
u16 clusterofs;
+ u64 segavail;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -158,13 +159,19 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ count = min_t(u64, len, ctx->segavail);
+ if (ctx->segavail <= EROFS_BLKSIZ) {
+ if (len < ctx->segavail && !final)
+ break;
+ goto nocompression;
+ }
+
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
break;
}
- count = len;
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
, dst, EROFS_BLKSIZ);
@@ -174,8 +181,9 @@ static int vle_compress_one(struct erofs_inode *inode,
inode->i_srcpath,
erofs_strerror(ret));
}
+ count = len;
nocompression:
- ret = write_uncompressed_block(ctx, , dst);
+ ret = write_uncompressed_block(ctx, , dst);
if (ret < 0)
return ret;
count = ret;
@@ -202,6 +210,9 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ ctx->segavail -= count;
+ if (!ctx->segavail)
+ ctx->segavail = cfg.c_compr_seg_size;
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
@@ -422,6 +433,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.segavail = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..de982e1 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_seg_size = UINT64_MAX;
}
void erofs_show_config(void)
diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index 891c5a8..b12cb22 100644
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -52,6 +52,10 @@ Forcely generate extended inodes (64-byte inodes) to output.
Set all files to the given UNIX timestamp. Reproducible builds requires setting
all to a specific one.
.TP
+.BI "\-S " #
+Set the max input stream size at one compression. The default is unsigned
64bit MAX.
+It must be algin to EROFS block size(4096).
+.TP
.BI "\-\-exclude-path=" path
Ignore file that matches the exact literal path.
You may give multiple `--exclude-path' options.
diff --git a/mkfs/main.c b/mkfs/main.c
index 94bf1e6..96cc053 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -61,6 +61,7 @@ static void usage(void)
" -x# set xattr tolerance to # (< 0, disable
xattrs; default 2)\n"
" -EX[,...] X=extended options\n"
" -T# set a fixed UNIX timestamp # to all files\n"
+ " -S# set the max input stream size # at one
compression\n"
" --exclude-path=X avoid including file X (X = exact literal
path)\n"
Re: [PATCH] erofs-utils: pass down inode for erofs_prepare_xattr_ibody
On 2020/6/17 15:27, Gao Xiang via Linux-erofs wrote: > From: Gao Xiang > > Instead of several independent arguments for convenience. > No logic changes. > > Signed-off-by: Gao Xiang > --- > > > It looks good > > Reviewed-by: Li Guifu >
[PATCH v4] erofs-utils: introduce segment compression
Support segment compression which seperates files in several logic
units (segments) and each segment is compressed independently.
Advantages:
- more friendly for data differencing;
- it can also be used for parallel compression in the same file later.
Signed-off-by: Li Guifu
---
Changes since v3 suggest by Gao Xiang:
- add 'S#' parameter to custome compression segment size
- move limit logic to size decrease
include/erofs/config.h | 1 +
lib/compress.c | 8 ++--
lib/config.c | 1 +
mkfs/main.c| 16 +++-
4 files changed, 23 insertions(+), 3 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..9125c1e 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -36,6 +36,7 @@ struct erofs_configure {
char *c_src_path;
char *c_compr_alg_master;
int c_compr_level_master;
+ unsigned int c_compr_seg_size; /* max segment compress size */
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..eb024aa 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,7 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr;/* pointing to the next blkaddr */
u16 clusterofs;
+ unsigned int comprlimits;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -163,8 +164,7 @@ static int vle_compress_one(struct erofs_inode *inode,
goto nocompression;
break;
}
-
- count = len;
+ count = min(len, ctx->comprlimits);
ret = erofs_compress_destsize(h, compressionlevel,
ctx->queue + ctx->head,
, dst, EROFS_BLKSIZ);
@@ -202,6 +202,9 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ ctx->comprlimits -= count;
+ if (!ctx->comprlimits)
+ ctx->comprlimits = cfg.c_compr_seg_size;
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
@@ -422,6 +425,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.comprlimits = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..1c39403 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_seg_size = 1024U * EROFS_BLKSIZ;
}
void erofs_show_config(void)
diff --git a/mkfs/main.c b/mkfs/main.c
index 94bf1e6..036d818 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -61,6 +61,7 @@ static void usage(void)
" -x# set xattr tolerance to # (< 0, disable
xattrs; default 2)\n"
" -EX[,...] X=extended options\n"
" -T# set a fixed UNIX timestamp # to all files\n"
+ " -S# set the max input stream size # to one
compress\n"
" --exclude-path=X avoid including file X (X = exact literal
path)\n"
" --exclude-regex=X avoid including files that match X (X =
regular expression)\n"
#ifdef HAVE_LIBSELINUX
@@ -138,7 +139,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
char *endptr;
int opt, i;
- while((opt = getopt_long(argc, argv, "d:x:z:E:T:",
+ while((opt = getopt_long(argc, argv, "d:x:z:E:T:S:",
long_options, NULL)) != -1) {
switch (opt) {
case 'z':
@@ -188,6 +189,19 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
return -EINVAL;
}
break;
+ case 'S':
+ cfg.c_compr_seg_size = strtol(optarg, , 0);
+ if (*endptr != '\0') {
+ erofs_err("invalid compress segment size %s",
+ optarg);
+ return -EINVAL;
+ }
+ if (cfg.c_compr_seg_size % EROFS_BLKSIZ != 0) {
+ erofs_err("segment size:%u should be align to
%u",
+ cfg.c_compr_seg_size, EROFS_BLKSIZ);
+ return -EINVAL;
+
[PATCH v3] erofs-utils: introduce segment size to limit the max input stream
Limit the max input stream size by adding segment compression
(e.g. 4M segment size), it will benefits:
- more friendly to block diff (and more details about this);
- it can also be used for parallel compression in the same file.
Signed-off-by: Li Guifu
---
include/erofs/config.h | 1 +
lib/compress.c | 11 +++
lib/config.c | 1 +
3 files changed, 13 insertions(+)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..9125c1e 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -36,6 +36,7 @@ struct erofs_configure {
char *c_src_path;
char *c_compr_alg_master;
int c_compr_level_master;
+ unsigned int c_compr_seg_size; /* max segment compress size */
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..8fdbfb2 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,8 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr;/* pointing to the next blkaddr */
u16 clusterofs;
+ unsigned int comprlimits;
+ unsigned int comr_seg_size;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -158,6 +160,12 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ if (ctx->comprlimits >= ctx->comr_seg_size ||
+ ctx->comprlimits + EROFS_BLKSIZ >= ctx->comr_seg_size) {
+ ctx->comprlimits = 0;
+ goto nocompression;
+ }
+
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
@@ -202,6 +210,7 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ ctx->comprlimits += count;
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
@@ -422,6 +431,8 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.comprlimits = 0;
+ ctx.comr_seg_size = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..1c39403 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_seg_size = 1024U * EROFS_BLKSIZ;
}
void erofs_show_config(void)
--
2.17.1
[PATCH v2] erofs-utils: add a compress limit to source input stream
It cause a differential amplification when create binary diff
image for upgrade. Give a limits to cut compress, so the amplification
will be limit in the given size.
Signed-off-by: Li Guifu
---
changes since v1:
- fix variable "readcount" use the min data with comprlimits
include/erofs/internal.h | 1 +
lib/compress.c | 22 +-
2 files changed, 18 insertions(+), 5 deletions(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 41da189..367c0b0 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -41,6 +41,7 @@ typedef unsigned short umode_t;
#define EROFS_ISLOTBITS5
#define EROFS_SLOTSIZE (1U << EROFS_ISLOTBITS)
+#define EROFS_COMPR_LIMITS (1024U * EROFS_BLKSIZ)
typedef u64 erofs_off_t;
typedef u64 erofs_nid_t;
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..fe1cb09 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -150,7 +150,7 @@ static int vle_compress_one(struct erofs_inode *inode,
{
struct erofs_compress *const h =
unsigned int len = ctx->tail - ctx->head;
- unsigned int count;
+ unsigned int count = 0;
int ret;
static char dstbuf[EROFS_BLKSIZ * 2];
char *const dst = dstbuf + EROFS_BLKSIZ;
@@ -159,7 +159,7 @@ static int vle_compress_one(struct erofs_inode *inode,
bool raw;
if (len <= EROFS_BLKSIZ) {
- if (final)
+ if (!count || final)
goto nocompression;
break;
}
@@ -392,7 +392,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
{
struct erofs_buffer_head *bh;
struct z_erofs_vle_compress_ctx ctx;
- erofs_off_t remaining;
+ erofs_off_t remaining, comprlimits;
erofs_blk_t blkaddr, compressed_blocks;
unsigned int legacymetasize;
int ret, fd;
@@ -422,10 +422,14 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ comprlimits = EROFS_COMPR_LIMITS;
while (remaining) {
- const u64 readcount = min_t(u64, remaining,
- sizeof(ctx.queue) - ctx.tail);
+ const u64 readcount = min_t(u64,
+min_t(u64, remaining,
+ sizeof(ctx.queue) - ctx.tail),
+ comprlimits);
+
ret = read(fd, ctx.queue + ctx.tail, readcount);
if (ret != readcount) {
@@ -434,11 +438,19 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
}
remaining -= readcount;
ctx.tail += readcount;
+ comprlimits -= readcount;
+compr_continue:
/* do one compress round */
ret = vle_compress_one(inode, , false);
if (ret)
goto err_bdrop;
+ if (!comprlimits) {
+ if (ctx.head != ctx.tail)
+ goto compr_continue;
+ ctx.clusterofs = 0;
+ comprlimits = EROFS_COMPR_LIMITS;
+ }
}
/* do the final round */
--
2.17.1
[PATCH] erofs-utils: add a compress limit to source input stream
It cause a differential amplification when create binary diff
image for upgrade. Give a limits to cut compress, so the amplification
will be limit in the given size.
Signed-off-by: Li Guifu
---
include/erofs/internal.h | 1 +
lib/compress.c | 15 ---
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 41da189..367c0b0 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -41,6 +41,7 @@ typedef unsigned short umode_t;
#define EROFS_ISLOTBITS5
#define EROFS_SLOTSIZE (1U << EROFS_ISLOTBITS)
+#define EROFS_COMPR_LIMITS (1024U * EROFS_BLKSIZ)
typedef u64 erofs_off_t;
typedef u64 erofs_nid_t;
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..9fe1428 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -150,7 +150,7 @@ static int vle_compress_one(struct erofs_inode *inode,
{
struct erofs_compress *const h =
unsigned int len = ctx->tail - ctx->head;
- unsigned int count;
+ unsigned int count = 0;
int ret;
static char dstbuf[EROFS_BLKSIZ * 2];
char *const dst = dstbuf + EROFS_BLKSIZ;
@@ -159,7 +159,7 @@ static int vle_compress_one(struct erofs_inode *inode,
bool raw;
if (len <= EROFS_BLKSIZ) {
- if (final)
+ if (!count || final)
goto nocompression;
break;
}
@@ -392,7 +392,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
{
struct erofs_buffer_head *bh;
struct z_erofs_vle_compress_ctx ctx;
- erofs_off_t remaining;
+ erofs_off_t remaining, comprlimits;
erofs_blk_t blkaddr, compressed_blocks;
unsigned int legacymetasize;
int ret, fd;
@@ -422,6 +422,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ comprlimits = EROFS_COMPR_LIMITS;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
@@ -434,11 +435,19 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
}
remaining -= readcount;
ctx.tail += readcount;
+ comprlimits -= readcount;
+compr_continue:
/* do one compress round */
ret = vle_compress_one(inode, , false);
if (ret)
goto err_bdrop;
+ if (!comprlimits) {
+ if (ctx.head != ctx.tail)
+ goto compr_continue;
+ ctx.clusterofs = 0;
+ comprlimits = EROFS_COMPR_LIMITS;
+ }
}
/* do the final round */
--
2.17.1
Re: [PATCH v4] erofs-utils: support selinux file contexts
On 2020/6/8 21:08, hsiang...@aol.com wrote: > From: Gao Xiang > > Add --file-contexts flag that allows passing a selinux > file_context file to setup file selabels. > > Reviewed-and-tested-by: Li Guifu > Signed-off-by: Gao Xiang > --- > v4: freecon() should be used instead of free(). (although > they're equivalent, but that is what manpage prefers...) > It looks good Reviewed-by: Li Guifu Tested-by: Li Guifu
Re: [PATCH] erofs-utils: enhance static linking for lz4 1.8.x
On 2020/5/31 11:45, Gao Xiang via Linux-erofs wrote: > From: Gao Xiang > > Since LZ4_compress_HC_destSize is static linking only on lz4 < 1.9.0, > but usually both lz4 static and dynamic library are available. > > Previously, -all-static is used in erofs-utils compilation for such > lz4 versions, but it has conficts with libselinux linking. Use another > workable way [1] I've found instead. > > [1] > https://stackoverflow.com/questions/8045707/how-to-link-to-the-libabc-a-instead-of-libabc-so > Signed-off-by: Gao Xiang > --- > It looks good > Reviewed-by: Li Guifu > Tested-by: Li Guifu
Re: [PATCH v2] erofs-utils: support selinux file contexts
On 2020/6/6 16:17, Gao Xiang wrote: > Add --file-contexts flag that allows passing a selinux > file_context file to setup file selabels. > > Signed-off-by: Gao Xiang > --- > It looks good > Reviewed-by: Li Guifu > Tested-by: Li Guifu
Re: [PATCH v2] erofs-utils: support selinux file contexts
It cacuses one build error in my Ubuntu 18.04.1 LTS and it seems selinux static lib link cause it libtool: link: gcc -Wall -Werror -I../include -g -O2 -static -o mkfs.erofs mkfs_erofs-main.o ../lib/.libs/liberofs.a -L/home/liguifu/codes/lz4 -luuid -lselinux -llz4 /usr/lib/gcc/x86_64-linux-gnu/8/../../../x86_64-linux-gnu/libselinux.a(seusers.o): In function `getseuserbyname': (.text+0x5e9): warning: Using 'getgrouplist' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking (.text+0x570): warning: Using 'getgrnam_r' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking (.text+0x9e): warning: Using 'getpwnam_r' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking /usr/lib/gcc/x86_64-linux-gnu/8/../../../x86_64-linux-gnu/libselinux.a(regex.o): In function `regex_writef': (.text+0x73): undefined reference to `pcre_fullinfo' (.text+0xe7): undefined reference to `pcre_fullinfo' /usr/lib/gcc/x86_64-linux-gnu/8/../../../x86_64-linux-gnu/libselinux.a(regex.o): In function `regex_data_free': (.text+0x1da): undefined reference to `pcre_free' (.text+0x1e8): undefined reference to `pcre_free_study' /usr/lib/gcc/x86_64-linux-gnu/8/../../../x86_64-linux-gnu/libselinux.a(regex.o): In function `regex_prepare_data': (.text+0x248): undefined reference to `pcre_compile' (.text+0x269): undefined reference to `pcre_study' /usr/lib/gcc/x86_64-linux-gnu/8/../../../x86_64-linux-gnu/libselinux.a(regex.o): In function `regex_load_mmap': (.text+0x36a): undefined reference to `pcre_fullinfo' (.text+0x3df): undefined reference to `pcre_fullinfo' /usr/lib/gcc/x86_64-linux-gnu/8/../../../x86_64-linux-gnu/libselinux.a(regex.o): In function `regex_match': (.text+0x48b): undefined reference to `pcre_exec' /usr/lib/gcc/x86_64-linux-gnu/8/../../../x86_64-linux-gnu/libselinux.a(regex.o): In function `regex_cmp': (.text+0x4ed): undefined reference to `pcre_fullinfo' (.text+0x506): undefined reference to `pcre_fullinfo' /usr/lib/gcc/x86_64-linux-gnu/8/../../../x86_64-linux-gnu/libselinux.a(regex.o): In function `regex_version': (.text+0x11): undefined reference to `pcre_version' /usr/lib/gcc/x86_64-linux-gnu/8/../../../x86_64-linux-gnu/libselinux.a(load_policy.o): In function `selinux_mkload_policy': (.text+0x134): undefined reference to `sepol_policy_kern_vers_max' (.text+0x13d): undefined reference to `sepol_policy_kern_vers_min' (.text+0x2f3): undefined reference to `sepol_policy_kern_vers_max' (.text+0x2fc): undefined reference to `sepol_policy_kern_vers_min' (.text+0x331): undefined reference to `sepol_policy_kern_vers_max' (.text+0x33a): undefined reference to `sepol_policy_kern_vers_min' (.text+0x3b2): undefined reference to `sepol_policy_file_create' (.text+0x3c4): undefined reference to `sepol_policydb_create' (.text+0x3e0): undefined reference to `sepol_policy_file_set_mem' (.text+0x3ef): undefined reference to `sepol_policydb_read' (.text+0x405): undefined reference to `sepol_policydb_set_vers' (.text+0x41f): undefined reference to `sepol_policydb_to_image' (.text+0x451): undefined reference to `sepol_policy_file_free' (.text+0x45b): undefined reference to `sepol_policydb_free' (.text+0x4a0): undefined reference to `sepol_policy_kern_vers_max' (.text+0x4a9): undefined reference to `sepol_policy_kern_vers_min' (.text+0x4bf): undefined reference to `sepol_policy_file_free' (.text+0x4c9): undefined reference to `sepol_policydb_free' (.text+0x5bf): undefined reference to `sepol_policy_file_free' (.text+0x5d3): undefined reference to `sepol_policy_file_free' (.text+0x5dd): undefined reference to `sepol_policydb_free' (.text+0x6d9): undefined reference to `sepol_genbools_array' (.text+0x73a): undefined reference to `sepol_genusers' (.text+0x76e): undefined reference to `sepol_genbools' collect2: error: ld returned 1 exit status Makefile:398: recipe for target 'mkfs.erofs' failed On 2020/6/6 16:17, Gao Xiang wrote: > Add --file-contexts flag that allows passing a selinux > file_context file to setup file selabels. > > Signed-off-by: Gao Xiang > --- > changes since v1: > fix selinux error handing and wrap up selabel > open pointed out by Guifu; > > configure.ac | 27 + > include/erofs/config.h | 21 +++ > include/erofs/xattr.h | 3 +- > lib/config.c | 42 + > lib/exclude.c | 12 +--- > lib/inode.c| 3 +- > lib/xattr.c| 130 - > man/mkfs.erofs.1 | 3 + > mkfs/Makefile.am | 4 +- > mkfs/main.c| 15 - > 10 files changed, 217 insertions(+), 43 deletions(-) >
Re: [PATCH] erofs-utils: avoid using old compatibility type uint
On 2020/3/24 16:19, Gao Xiang wrote: > This should fix the following buildroot autobuild issues > with some configration on ARM platform [1]: > > compress.c: In function 'vle_compress_one': > compress.c:209:10: error: unknown type name 'uint' > const uint qh_aligned = round_down(ctx->head, EROFS_BLKSIZ); > ^~~~ > compress.c:210:10: error: unknown type name 'uint' > const uint qh_after = ctx->head - qh_aligned; > ^~~~ > compress.c: In function 'z_erofs_convert_to_compacted_format': > compress.c:313:8: error: unknown type name 'uint' > const uint headerpos = Z_EROFS_VLE_EXTENT_ALIGN(inode->inode_isize + > ^~~~ > compress.c:316:8: error: unknown type name 'uint' > const uint totalidx = (legacymetasize - > ^~~~ > > [1] > http://autobuild.buildroot.net/results/842a3c6416416d7badf4db9f38e3b231093a786a > Signed-off-by: Gao Xiang It looks good Reviewed-by: Li Guifu Thanks
Re: [PATCH] erofs-utils: avoid _LARGEFILE64_SOURCE and _GNU_SOURCE redefinition
On 2020/3/14 18:52, Gao Xiang wrote: > From: Gao Xiang > > This patch can be used to resolve the following build errors: > > compress.c:10: error: "_LARGEFILE64_SOURCE" redefined [-Werror] > #define _LARGEFILE64_SOURCE > > : note: this is the location of the previous definition > > io.c:9: error: "_LARGEFILE64_SOURCE" redefined [-Werror] > #define _LARGEFILE64_SOURCE > > : note: this is the location of the previous definition > > Signed-off-by: Gao Xiang It looks good Reviewed-by: Li Guifu Thanks
Re: [PATCH] erofs-utils: avoid PAGE_SIZE redefinition
On 2020/3/25 16:29, Gao Xiang wrote: > Buildroot autobuild reported a PAGE_SIZE redefinition with some > configrations on i586 toolchain [1] (I didn't notice such report > from erofs-utils travis CI or distribution builds before.) > > In file included from config.c:11: > ../include/erofs/internal.h:27: error: "PAGE_SIZE" redefined [-Werror] > #define PAGE_SIZE (1U << PAGE_SHIFT) > > In file included from ../include/erofs/defs.h:17, > from ../include/erofs/config.h:12, > from ../include/erofs/print.h:12, > from config.c:10: > .../sysroot/usr/include/limits.h:89: note: this is the location of the > previous definition > #define PAGE_SIZE PAGESIZE > > cc1: all warnings being treated as errors > > Fix it now. > > [1] > http://autobuild.buildroot.net/results/340b98caa45bafd43f109002be9da59ba7f6d971 > Signed-off-by: Gao Xiang It looks good Reviewed-by: Li Guifu Thanks
Re: [PATCH v7] erofs-utils: introduce exclude dirs and files
On 2020/2/19 10:20, Gao Xiang wrote: > Hi Guifu, > > On Tue, Feb 18, 2020 at 10:30:47PM +0800, Li Guifu wrote: >> From: Li GuiFu >> >> Add excluded file feature "--exclude-path=" and '--exclude-regex=', >> which can be used to build EROFS image without some user specific >> files or dirs. Note that you may give multiple '--exclude-path' >> or '--exclude-regex' options. >> >> Signed-off-by: Gao Xiang >> Signed-off-by: Li Guifu > > Applied to experimental branch with the following minor updates. > > If you have more suggestions, please kindly point out. Or I will > push it out to dev branch later. > > Thanks, > Gao Xiang > It looks good
[PATCH v7] erofs-utils: introduce exclude dirs and files
From: Li GuiFu
Add excluded file feature "--exclude-path=" and '--exclude-regex=',
which can be used to build EROFS image without some user specific
files or dirs. Note that you may give multiple '--exclude-path'
or '--exclude-regex' options.
Signed-off-by: Gao Xiang
Signed-off-by: Li Guifu
---
change since v7:
- fux as comment by Gao Xiang
- fix missing regex free
- update comment of option
include/erofs/exclude.h | 26
lib/Makefile.am | 2 +-
lib/exclude.c | 141
lib/inode.c | 5 ++
man/mkfs.erofs.1| 8 +++
mkfs/main.c | 36 --
6 files changed, 211 insertions(+), 7 deletions(-)
create mode 100644 include/erofs/exclude.h
create mode 100644 lib/exclude.c
diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
new file mode 100644
index 000..0a82dbe
--- /dev/null
+++ b/include/erofs/exclude.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs-utils/include/erofs/exclude.h
+ *
+ * Created by Li Guifu
+ */
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+#include
+#include
+
+struct erofs_exclude_rule {
+ struct list_head list;
+
+ char *pattern; /* save original pattern for exact or regex
match */
+ regex_t reg;
+};
+
+void erofs_exclude_set_root(const char *rootdir);
+void erofs_cleanup_exclude_rules(void);
+
+int erofs_parse_exclude_path(const char *args, bool is_regex);
+struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
+const char *name);
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 1ff81f9..e4b51e6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -3,7 +3,7 @@
noinst_LTLIBRARIES = liberofs.la
liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
- compress.c compressor.c
+ compress.c compressor.c exclude.c
liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
if ENABLE_LZ4
liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/exclude.c b/lib/exclude.c
new file mode 100644
index 000..636f048
--- /dev/null
+++ b/lib/exclude.c
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/exclude.c
+ *
+ * Created by Li Guifu
+ */
+#include
+#include
+#include "erofs/err.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+#define EXCLUDE_RULE_EXACT_SIZEoffsetof(struct erofs_exclude_rule, reg)
+#define EXCLUDE_RULE_REGEX_SIZEsizeof(struct erofs_exclude_rule)
+
+static LIST_HEAD(exclude_head);
+static LIST_HEAD(regex_exclude_head);
+
+static unsigned int rpathlen; /* root directory prefix length */
+
+void erofs_exclude_set_root(const char *rootdir)
+{
+ rpathlen = strlen(rootdir);
+}
+
+static void dump_regerror(int errcode, const char *s, const regex_t *preg)
+{
+ char str[1024]; /* overflow safe */
+
+ regerror(errcode, preg, str, sizeof(str));
+ erofs_err("invalid regex %s,because %s\n", s, str);
+}
+
+static struct erofs_exclude_rule *erofs_insert_exclude(const char *s,
+ bool is_regex)
+{
+ int ret = -ENOMEM;
+ struct erofs_exclude_rule *r;
+ struct list_head *h;
+ unsigned int size;
+
+ size = is_regex ? EXCLUDE_RULE_REGEX_SIZE : EXCLUDE_RULE_EXACT_SIZE;
+ r = malloc(size);
+ if (!r)
+ return ERR_PTR(-ENOMEM);
+
+ r->pattern = strdup(s);
+ if (!r->pattern)
+ goto err_rule;
+
+ if (is_regex) {
+ ret = regcomp(>reg, s, REG_EXTENDED|REG_NOSUB);
+ if(ret) {
+ dump_regerror(ret, s, >reg);
+ goto err_rule;
+ }
+ h = _exclude_head;
+ } else {
+ h = _head;
+ }
+
+ list_add_tail(>list, h);
+ erofs_info("Insert exclude:[%s]\n", s);
+ return r;
+
+err_rule:
+ free(r);
+ return ERR_PTR(ret);
+}
+
+void erofs_cleanup_exclude_rules(void)
+{
+ struct erofs_exclude_rule *r, *n;
+ struct list_head *h;
+
+ h = _head;
+ list_for_each_entry_safe(r, n, h, list) {
+ list_del(>list);
+ free(r->pattern);
+ free(r);
+ }
+
+ h = _exclude_head;
+ list_for_each_entry_safe(r, n, h, list) {
+ list_del(>list);
+ free(r->pattern);
+ regfree(>reg);
+ free(r);
+ }
+
+}
+
+int erofs_parse_exclude_path(const char *args, bool is_regex)
+{
+ struct erofs_exclude_rule *r = erofs_insert_exclude(args, is_regex);
+
+ if (IS_ERR(r)) {
+ erofs_cleanup_exclude_rules();
+ return PTR_ERR(r)
Re: [PATCH v6] erofs-utils: introduce exclude dirs and files
Good idea On 2020/2/18 10:32, Gao Xiang wrote: > On Mon, Feb 17, 2020 at 09:16:53PM +0800, Li Guifu wrote: >> From: Li GuiFu >> >> Add excluded file feature "--exclude-path=" and '--exclude-regex=', >> which can be used to build EROFS image without some user specific >> files or dirs. Note that you may give multiple '--exclude-path' >> or '--exclude-regex' options. >> >> Signed-off-by: Gao Xiang >> Signed-off-by: Li Guifu
[PATCH v6] erofs-utils: introduce exclude dirs and files
From: Li GuiFu
Add excluded file feature "--exclude-path=" and '--exclude-regex=',
which can be used to build EROFS image without some user specific
files or dirs. Note that you may give multiple '--exclude-path'
or '--exclude-regex' options.
Signed-off-by: Gao Xiang
Signed-off-by: Li Guifu
---
change since v6, fix as comments suggested by Gao Xiang
- update a new email address
- refact regex and pattern match
include/erofs/exclude.h | 26
lib/Makefile.am | 2 +-
lib/exclude.c | 136
lib/inode.c | 5 ++
man/mkfs.erofs.1| 8 +++
mkfs/main.c | 36 +--
6 files changed, 206 insertions(+), 7 deletions(-)
create mode 100644 include/erofs/exclude.h
create mode 100644 lib/exclude.c
diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
new file mode 100644
index 000..0a82dbe
--- /dev/null
+++ b/include/erofs/exclude.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs-utils/include/erofs/exclude.h
+ *
+ * Created by Li Guifu
+ */
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+#include
+#include
+
+struct erofs_exclude_rule {
+ struct list_head list;
+
+ char *pattern; /* save original pattern for exact or regex
match */
+ regex_t reg;
+};
+
+void erofs_exclude_set_root(const char *rootdir);
+void erofs_cleanup_exclude_rules(void);
+
+int erofs_parse_exclude_path(const char *args, bool is_regex);
+struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
+const char *name);
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 1ff81f9..e4b51e6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -3,7 +3,7 @@
noinst_LTLIBRARIES = liberofs.la
liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
- compress.c compressor.c
+ compress.c compressor.c exclude.c
liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
if ENABLE_LZ4
liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/exclude.c b/lib/exclude.c
new file mode 100644
index 000..6dae67e
--- /dev/null
+++ b/lib/exclude.c
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/exclude.c
+ *
+ * Created by Li Guifu
+ */
+#include
+#include
+#include "erofs/err.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+#define EXCLUDE_RULE_EXACT_SIZEoffsetof(struct erofs_exclude_rule, reg)
+#define EXCLUDE_RULE_REGEX_SIZEsizeof(struct erofs_exclude_rule)
+
+static LIST_HEAD(exclude_head);
+static LIST_HEAD(regex_exclude_head);
+
+static unsigned int rpathlen; /* root directory prefix length */
+
+void erofs_exclude_set_root(const char *rootdir)
+{
+ rpathlen = strlen(rootdir);
+}
+
+static void dump_regerror(int errcode, const char *s, const regex_t *preg)
+{
+ char str[1024]; /* overflow safe */
+
+ regerror(errcode, preg, str, 1024);
+ erofs_err("invalid regex %s,because %s\n", s, str);
+}
+
+static struct erofs_exclude_rule *erofs_insert_exclude(const char *s,
+ bool is_regex)
+{
+ int ret = -ENOMEM;
+ struct erofs_exclude_rule *r;
+ struct list_head *h;
+ unsigned int size;
+
+ size = is_regex ? EXCLUDE_RULE_REGEX_SIZE : EXCLUDE_RULE_EXACT_SIZE;
+ r = malloc(size);
+ if (!r)
+ return ERR_PTR(-ENOMEM);
+
+ r->pattern = strdup(s);
+ if (!r->pattern)
+ goto err_rule;
+
+ if (is_regex) {
+ ret = regcomp(>reg, s, REG_EXTENDED|REG_NOSUB);
+ if(ret) {
+ dump_regerror(ret, s, >reg);
+ goto err_rule;
+ }
+ h = _exclude_head;
+ } else {
+ h = _head;
+ }
+
+ list_add_tail(>list, h);
+ erofs_info("Insert exclude:[%s]\n", s);
+ return r;
+
+err_rule:
+ free(r);
+ return ERR_PTR(ret);
+}
+
+static inline void erofs_free_exclude_rules(struct list_head *h)
+{
+ struct erofs_exclude_rule *r, *n;
+
+ list_for_each_entry_safe(r, n, h, list) {
+ list_del(>list);
+ free(r->pattern);
+ free(r);
+ }
+}
+
+void erofs_cleanup_exclude_rules(void)
+{
+ erofs_free_exclude_rules(_head);
+ erofs_free_exclude_rules(_exclude_head);
+}
+
+int erofs_parse_exclude_path(const char *args, bool is_regex)
+{
+ struct erofs_exclude_rule *r = erofs_insert_exclude(args, is_regex);
+
+ if (IS_ERR(r)) {
+ erofs_cleanup_exclude_rules();
+ return PTR_ERR(r);
+ }
+ return 0;
+}
+
+struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
+
[PATCH v5] erofs-utils: introduce exclude dirs and files
From: Li GuiFu
Add excluded file feature "--exclude-path=" and '--exlude-regex=',
which can be used to build EROFS image without some user specific
files or dirs.
Note that you may give multiple '--exclude-path' or '--exclude-regex'
options.
Signed-off-by: Li Guifu
Signed-off-by: Gao Xiang
---
change since v5:
- add regex free
- clean log print
include/erofs/exclude.h | 28 +
lib/Makefile.am | 2 +-
lib/exclude.c | 133
lib/inode.c | 5 ++
man/mkfs.erofs.1| 8 +++
mkfs/main.c | 36 +--
6 files changed, 205 insertions(+), 7 deletions(-)
create mode 100644 include/erofs/exclude.h
create mode 100644 lib/exclude.c
diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
new file mode 100644
index 000..e158a14
--- /dev/null
+++ b/include/erofs/exclude.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs-utils/include/erofs/exclude.h
+ *
+ * Created by Li Guifu
+ */
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+#include
+#include
+
+struct erofs_exclude_rule {
+ struct list_head list;
+
+ union {
+ char *pattern;
+ regex_t *reg;
+ };
+};
+
+void erofs_exclude_set_root(const char *rootdir);
+void erofs_cleanup_exclude_rules(void);
+
+int erofs_parse_exclude_path(const char *args, bool is_regex);
+struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
+const char *name);
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 1ff81f9..e4b51e6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -3,7 +3,7 @@
noinst_LTLIBRARIES = liberofs.la
liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
- compress.c compressor.c
+ compress.c compressor.c exclude.c
liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
if ENABLE_LZ4
liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/exclude.c b/lib/exclude.c
new file mode 100644
index 000..268a6f7
--- /dev/null
+++ b/lib/exclude.c
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/exclude.c
+ *
+ * Created by Li Guifu
+ */
+#include
+#include
+#include "erofs/err.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+static LIST_HEAD(exclude_head);
+static LIST_HEAD(regex_exclude_head);
+
+static unsigned int rpathlen; /* root directory prefix length */
+
+void erofs_exclude_set_root(const char *rootdir)
+{
+ rpathlen = strlen(rootdir);
+}
+
+static struct erofs_exclude_rule
+*erofs_insert_exclude(const char *s, bool is_regex)
+{
+ int ret = -ENOMEM;
+ struct erofs_exclude_rule *r;
+ struct list_head *h;
+
+ r = malloc(sizeof(*r));
+ if (!r)
+ return ERR_PTR(-ENOMEM);
+
+ /* exact match */
+ if (is_regex) {
+ r->reg = malloc(sizeof(regex_t));
+ if (!r->reg)
+ goto err_rule;
+ ret = regcomp(r->reg, s, REG_EXTENDED|REG_NOSUB);
+ if(ret) {
+ char str[1024]; /* overflow safe */
+
+ regerror(ret, r->reg, str, 1024);
+ erofs_err("invalid regex %s,because %s\n", s, str);
+ goto err_rule;
+ }
+ h = _exclude_head;
+ } else {
+ r->pattern = strdup(s);
+
+ if (!r->pattern)
+ goto err_rule;
+ h = _head;
+ }
+
+ list_add_tail(>list, h);
+ erofs_info("Insert exclude:[%s]\n", s);
+ return r;
+err_rule:
+ free(r);
+ return ERR_PTR(ret);
+}
+
+void erofs_cleanup_exclude_rules(void)
+{
+ struct erofs_exclude_rule *r, *n;
+
+ list_for_each_entry_safe(r, n, _head, list) {
+ list_del(>list);
+ free(r->pattern);
+ free(r);
+ }
+
+ list_for_each_entry_safe(r, n, _exclude_head, list) {
+ list_del(>list);
+ regfree(r->reg);
+ free(r);
+ }
+}
+
+int erofs_parse_exclude_path(const char *args, bool is_regex)
+{
+ struct erofs_exclude_rule *r = erofs_insert_exclude(args, is_regex);
+
+ if (IS_ERR(r)) {
+ erofs_cleanup_exclude_rules();
+ return PTR_ERR(r);
+ }
+ return 0;
+}
+
+struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
+const char *name)
+{
+ char buf[PATH_MAX];
+ const char *s;
+ struct erofs_exclude_rule *r;
+
+ if (!dir) {
+ /* no prefix */
+ s = name;
+ } else {
+ sprintf(buf, "%s/%s", di
[PATCH v4] erofs-utils: introduce exclude dirs and files
From: Li GuiFu
Add excluded file feature "--exclude-path=" and '--exlude-regex=',
which can be used to build EROFS image without some user specific
files or dirs.
Note that you may give multiple '--exclude-path' or '--exclude-regex'
options.
Signed-off-by: Li Guifu
Signed-off-by: Gao Xiang
---
include/erofs/exclude.h | 28 +
lib/Makefile.am | 2 +-
lib/exclude.c | 127
lib/inode.c | 5 ++
man/mkfs.erofs.1| 8 +++
mkfs/main.c | 36 ++--
6 files changed, 199 insertions(+), 7 deletions(-)
create mode 100644 include/erofs/exclude.h
create mode 100644 lib/exclude.c
diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
new file mode 100644
index 000..e158a14
--- /dev/null
+++ b/include/erofs/exclude.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs-utils/include/erofs/exclude.h
+ *
+ * Created by Li Guifu
+ */
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+#include
+#include
+
+struct erofs_exclude_rule {
+ struct list_head list;
+
+ union {
+ char *pattern;
+ regex_t *reg;
+ };
+};
+
+void erofs_exclude_set_root(const char *rootdir);
+void erofs_cleanup_exclude_rules(void);
+
+int erofs_parse_exclude_path(const char *args, bool is_regex);
+struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
+const char *name);
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 1ff81f9..e4b51e6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -3,7 +3,7 @@
noinst_LTLIBRARIES = liberofs.la
liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
- compress.c compressor.c
+ compress.c compressor.c exclude.c
liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
if ENABLE_LZ4
liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/exclude.c b/lib/exclude.c
new file mode 100644
index 000..a293eec
--- /dev/null
+++ b/lib/exclude.c
@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/exclude.c
+ *
+ * Created by Li Guifu
+ */
+#include
+#include
+#include "erofs/err.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+static LIST_HEAD(exclude_head);
+static LIST_HEAD(regex_exclude_head);
+
+static unsigned int rpathlen; /* root directory prefix length */
+
+void erofs_exclude_set_root(const char *rootdir)
+{
+ rpathlen = strlen(rootdir);
+}
+
+static struct erofs_exclude_rule
+*erofs_insert_exclude(const char *s, bool is_regex)
+{
+ int ret = -ENOMEM;
+ struct erofs_exclude_rule *r;
+ struct list_head *h;
+
+ r = malloc(sizeof(*r));
+ if (!r)
+ return ERR_PTR(-ENOMEM);
+
+ /* exact match */
+ if (is_regex) {
+ r->reg = malloc(sizeof(regex_t));
+ if (!r->reg)
+ goto err_rule;
+ ret = regcomp(r->reg, s, REG_EXTENDED|REG_NOSUB);
+ if(ret) {
+ char str[1024]; /* overflow safe */
+
+ regerror(ret, r->reg, str, 1024);
+ erofs_err("invalid regex %s,because %s\n", s, str);
+ goto err_rule;
+ }
+ h = _exclude_head;
+ } else {
+ r->pattern = strdup(s);
+
+ if (!r->pattern)
+ goto err_rule;
+ h = _head;
+ }
+
+ list_add_tail(>list, h);
+ erofs_dump("Insert exclude path:[%s]\n", s);
+ return r;
+err_rule:
+ free(r);
+ return ERR_PTR(ret);
+}
+
+void erofs_cleanup_exclude_rules(void)
+{
+ struct erofs_exclude_rule *r, *n;
+
+ list_for_each_entry_safe(r, n, _head, list) {
+ list_del(>list);
+ free(r->pattern);
+ free(r);
+ }
+}
+
+int erofs_parse_exclude_path(const char *args, bool is_regex)
+{
+ struct erofs_exclude_rule *r = erofs_insert_exclude(args, is_regex);
+
+ if (IS_ERR(r)) {
+ erofs_cleanup_exclude_rules();
+ return PTR_ERR(r);
+ }
+ return 0;
+}
+
+struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
+const char *name)
+{
+ char buf[PATH_MAX];
+ const char *s;
+ struct erofs_exclude_rule *r;
+
+ if (!dir) {
+ /* no prefix */
+ s = name;
+ } else {
+ sprintf(buf, "%s/%s", dir, name);
+ s = buf;
+ }
+
+ s += rpathlen;
+ while (*s == '/')
+ s++;
+
+ list_for_each_entry(r, _head, list) {
+ if (!strcmp(r->pattern, s))
+ return r;
Re: Re: [PATCH v2] erofs-utils: introduce exclude dirs and files
Hi
My e-mail of foxmail was broken in the mainline, use aliyun instead.
On 2020/2/5 1:02, Gao Xiang via Linux-erofs wrote:
> Hi Guifu,
>
> I cleanup the whole patch and get rid of all dedupe check in order
> to make it as simple as possible since I think not too many exclude
> files in general.
>
> Please take some time helping testing the following patch and
> hope to get your feedback.
>
> Thanks,
> Gao Xiang
>
>
>
>>From 600fc166cdaaa0e458181729245ce11affa83ac5 Mon Sep 17 00:00:00 2001
> From: Li Guifu
> Date: Mon, 3 Feb 2020 23:38:11 +0800
> Subject: [PATCH v3] erofs-utils: introduce exclude dirs and files
>
> Add excluded file feature "--exclude-path=", which can be used
> to build EROFS image without some user specific files or dirs.
>
> Note that you may give multiple `--exclude-path' options.
>
> Signed-off-by: Li Guifu
> Signed-off-by: Gao Xiang
> ---
> changes since v2:
> - cleanup the whole implementation;
> - complete usage message;
> - complete manual page.
>
> include/erofs/exclude.h | 23 +++
> lib/Makefile.am | 2 +-
> lib/exclude.c | 89 +
> lib/inode.c | 5 +++
> man/mkfs.erofs.1| 4 ++
> mkfs/main.c | 26 +---
> 6 files changed, 142 insertions(+), 7 deletions(-)
> create mode 100644 include/erofs/exclude.h
> create mode 100644 lib/exclude.c
>
> diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
> new file mode 100644
> index 000..580fefe
> --- /dev/null
> +++ b/include/erofs/exclude.h
> @@ -0,0 +1,23 @@
> +/* SPDX-License-Identifier: GPL-2.0+ */
> +/*
> + * erofs-utils/include/erofs/exclude.h
> + *
> + * Created by Li Guifu
> + */
> +#ifndef __EROFS_EXCLUDE_H
> +#define __EROFS_EXCLUDE_H
> +
> +struct erofs_exclude_rule {
> + struct list_head list;
> +
> + char *pattern;
> +};
> +
> +void erofs_exclude_set_root(const char *rootdir);
> +void erofs_cleanup_exclude_rules(void);
> +
> +int erofs_parse_exclude_path(const char *args);
> +struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
> + const char *name);
> +#endif
> +
> diff --git a/lib/Makefile.am b/lib/Makefile.am
> index 1ff81f9..e4b51e6 100644
> --- a/lib/Makefile.am
> +++ b/lib/Makefile.am
> @@ -3,7 +3,7 @@
>
> noinst_LTLIBRARIES = liberofs.la
> liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
> - compress.c compressor.c
> + compress.c compressor.c exclude.c
> liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
> if ENABLE_LZ4
> liberofs_la_CFLAGS += ${LZ4_CFLAGS}
> diff --git a/lib/exclude.c b/lib/exclude.c
> new file mode 100644
> index 000..9b48325
> --- /dev/null
> +++ b/lib/exclude.c
> @@ -0,0 +1,89 @@
> +// SPDX-License-Identifier: GPL-2.0+
> +/*
> + * erofs-utils/lib/exclude.c
> + *
> + * Created by Li Guifu
> + */
> +#include
> +#include
> +#include "erofs/err.h"
> +#include "erofs/list.h"
> +#include "erofs/print.h"
> +#include "erofs/exclude.h"
> +
> +static LIST_HEAD(exclude_head);
> +static unsigned int rpathlen;/* root directory prefix length
> */
> +
> +void erofs_exclude_set_root(const char *rootdir)
> +{
> + rpathlen = strlen(rootdir);
> +}
> +
> +static struct erofs_exclude_rule *erofs_insert_exclude(const char *s)
> +{
> + struct erofs_exclude_rule *e;
> +
> + e = malloc(sizeof(*e));
> + if (!e)
> + return ERR_PTR(-ENOMEM);
> +
> + /* exact match */
> + e->pattern = strdup(s);
> + if (!e->pattern) {
> + free(e);
> + return ERR_PTR(-ENOMEM);
> + }
> +
> + list_add_tail(>list, _head);
> + erofs_info("exclude path %s inserted", e->pattern);
> + return e;
> +}
> +
> +void erofs_cleanup_exclude_rules(void)
> +{
> + struct erofs_exclude_rule *e, *n;
> +
> + list_for_each_entry_safe(e, n, _head, list) {
> + list_del(>list);
> + free(e->pattern);
> + free(e);
> + }
> +}
> +
> +int erofs_parse_exclude_path(const char *args)
> +{
> + struct erofs_exclude_rule *e = erofs_insert_exclude(args);
> +
> + if (IS_ERR(e)) {
> + erofs_cleanup_exclude_rules();
> + return PTR_ERR(e);
> + }
> + return 0;
> +}
> +
> +struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir,
> + const c
[PATCH v2] erofs-utils: introduce exclude dirs and files
From: Li Guifu
Add excluded file feature "--exclude-path=", which can be used
to build EROFS image without some user specific files or dirs.
Such multiple files can be seperated by ','.
Signed-off-by: Li Guifu
---
include/erofs/exclude.h | 28
lib/Makefile.am | 2 +-
lib/exclude.c | 149
lib/inode.c | 10 +++
mkfs/main.c | 11 +++
5 files changed, 199 insertions(+), 1 deletion(-)
create mode 100644 include/erofs/exclude.h
create mode 100644 lib/exclude.c
diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
new file mode 100644
index 000..3581046
--- /dev/null
+++ b/include/erofs/exclude.h
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * exclude.h
+ * Created by Li Guifu
+ */
+
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+
+struct exclude_rule {
+ struct list_head list;
+ struct list_head hlist;
+ char *pattern;
+};
+
+void erofs_init_exclude_rules(void);
+void erofs_free_exclude_rules(void);
+int erofs_parse_exclude_path(const char *args);
+struct exclude_rule *erofs_pattern_matched(const char *s);
+struct exclude_rule *erofs_entry_matched(struct exclude_rule *e,
+const char *s, unsigned int len);
+
+static inline int erofs_is_pattern_end(struct exclude_rule *e)
+{
+ return list_empty(>hlist);
+}
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 1ff81f9..e4b51e6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -3,7 +3,7 @@
noinst_LTLIBRARIES = liberofs.la
liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
- compress.c compressor.c
+ compress.c compressor.c exclude.c
liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
if ENABLE_LZ4
liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/exclude.c b/lib/exclude.c
new file mode 100644
index 000..1c65fb9
--- /dev/null
+++ b/lib/exclude.c
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * exclude.c
+ * Created by Li Guifu
+ */
+
+#include
+#include
+#include
+#include "erofs/err.h"
+#include "erofs/defs.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+static struct exclude_rule exclude_rules;
+
+static struct exclude_rule *new_rule(const char *s, unsigned int len)
+{
+ struct exclude_rule *e = malloc(sizeof(struct exclude_rule));
+
+ if (!e)
+ return ERR_PTR(-ENOMEM);
+
+ e->pattern = strndup(s, len);
+ if (!e->pattern)
+ goto err_strdup;
+
+ init_list_head(>hlist);
+ init_list_head(>list);
+
+ return e;
+
+err_strdup:
+ free(e);
+ return ERR_PTR(-ENOMEM);
+}
+
+struct exclude_rule *erofs_entry_matched(struct exclude_rule *e,
+const char *s, unsigned int len)
+{
+ struct exclude_rule *pos;
+
+ while (*s == '/') {
+ s++;
+ len--;
+ }
+ list_for_each_entry(pos, >hlist, list) {
+ unsigned int l = strlen(pos->pattern);
+
+ if (l == len && !strcmp(pos->pattern, s))
+ return pos;
+ }
+
+ return ERR_PTR(-ENOMEM);
+}
+
+static int add_exclude_rules(struct exclude_rule *e, const char *s)
+{
+ const char *ptr;
+ struct exclude_rule *erule;
+
+ while (*s == '/')
+ s++;
+ ptr = s;
+
+forward:
+ while(*ptr != '/' && *ptr != '\0')
+ ptr++;
+
+ erule = erofs_entry_matched(e, s, ptr - s);
+ if (IS_ERR(erule)) {
+ struct exclude_rule *r = new_rule(s, ptr - s);
+
+ if (IS_ERR(r))
+ return PTR_ERR(r);
+ list_add_tail(>list, >hlist);
+ erule = r;
+ }
+ e = erule;
+
+ if (*ptr++ != '\0') {
+ s = ptr;
+ goto forward;
+ }
+
+ return 0;
+}
+
+static void free_exclude_rules(struct list_head *h)
+{
+ struct exclude_rule *e, *n;
+
+ list_for_each_entry_safe(e, n, h, list) {
+ list_del(>list);
+ free_exclude_rules(>hlist);
+ free(e->pattern);
+ free(e);
+ }
+}
+void erofs_init_exclude_rules(void)
+{
+ init_list_head(_rules.list);
+ init_list_head(_rules.hlist);
+ exclude_rules.pattern = "/";
+}
+
+void erofs_free_exclude_rules(void)
+{
+ free_exclude_rules(_rules.hlist);
+}
+
+int erofs_parse_exclude_path(const char *args)
+{
+ const char *str, *ptr = args;
+ const char sep = ',';
+
+forward:
+ while(*ptr != sep && *ptr != '\0')
+ ptr++;
+
+ str = strndup(args, ptr - args);
+ if (!str)
+ goto err_free_paths;
+
+
Re: [PATCH v1] erofs-utils: introduce exclude dirs and files
Gao Xiang
It seems a good iead about functions' names and commit message, I will
fix them asap
A value is needed to long option exclude-path in the getopt_long, Do you
have a another idea ?
On 2020/2/3 17:08, Gao Xiang wrote:
> Hi Guifu,
>
> On Thu, Jan 23, 2020 at 03:04:02PM +0800, Li Guifu wrote:
>> From: Li Guifu
> I could imagine that gmail is hard to work on mainland China,
> but what's wrong with @foxmail.com this time? It could be better
> to use some email address from a stable provider.
>
>> Add support exlcude directory when build image with long option
>> arg --exclude-path=dir1/dir2,dir11/dir22 or short option arg
>> -e dir1/dir2,dir11/dir22 that seperated by ','
> How about the following commit message?
>
> Add excluded file feature "--exclude-path=", which can be used
> to build EROFS image without some user specific files or dirs.
> Such multiple files can be seperated by ','.
>
> and I tend to avoid the short option '-e' in order to keep it
> for later use (e.g. file system encryption).
>
>> Signed-off-by: Li Guifu
>> ---
>> include/erofs/exclude.h | 28
>> lib/Makefile.am | 2 +-
>> lib/exclude.c | 149
>> lib/inode.c | 10 +++
>> mkfs/main.c | 13 +++-
>> 5 files changed, 200 insertions(+), 2 deletions(-)
>> create mode 100644 include/erofs/exclude.h
>> create mode 100644 lib/exclude.c
>>
>> diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
>> new file mode 100644
>> index 000..3f9fb4d
>> --- /dev/null
>> +++ b/include/erofs/exclude.h
>> @@ -0,0 +1,28 @@
>> +// SPDX-License-Identifier: GPL-2.0+
>> +/*
>> + * erofs-utils\include\erofs\exclude.h
> erofs-utils/include/erofs/exclude.h
>
> No backslash here
>
>> + * Created by Li Guifu
>> + */
>> +
>> +#ifndef __EROFS_EXCLUDE_H
>> +#define __EROFS_EXCLUDE_H
>> +
>> +struct rule_entry {
> rule is too general here. exclude_rule_entry
>
>> +struct list_head list;
>> +struct list_head hlist;
>> +char *name;
>> +};
>> +
>> +void erofs_init_rules(void);
>> +void erofs_free_rules(void);
>
> Same here erofs_init_exclude_rules(), erofs_free_exclude_rules();
>
>> +int erofs_parse_exclude_path(const char *args);
>> +struct rule_entry *erofs_pattern_matched(const char *s);
>> +struct rule_entry *erofs_entry_matched(struct rule_entry *e,
>> + const char *s, unsigned int len);
>> +
>> +static inline int erofs_is_pattern_end(struct rule_entry *e)
>> +{
>> +return list_empty(>hlist);
>> +}
>> +#endif
>> +
>> diff --git a/lib/Makefile.am b/lib/Makefile.am
>> index 1ff81f9..e4b51e6 100644
>> --- a/lib/Makefile.am
>> +++ b/lib/Makefile.am
>> @@ -3,7 +3,7 @@
>>
>> noinst_LTLIBRARIES = liberofs.la
>> liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
>> - compress.c compressor.c
>> + compress.c compressor.c exclude.c
>> liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
>> if ENABLE_LZ4
>> liberofs_la_CFLAGS += ${LZ4_CFLAGS}
>> diff --git a/lib/exclude.c b/lib/exclude.c
>> new file mode 100644
>> index 000..25a9d32
>> --- /dev/null
>> +++ b/lib/exclude.c
>> @@ -0,0 +1,149 @@
>> +// SPDX-License-Identifier: GPL-2.0+
>> +/*
>> + * erofs-utils\lib\exclude.c
> Same here.
>
>> + * Created by Li Guifu
>> + */
>> +
>> +#include
>> +#include
>> +#include
>> +
>> +#include "erofs/defs.h"
>> +#include "erofs/list.h"
>> +#include "erofs/print.h"
>> +#include "erofs/exclude.h"
>> +
>> +static struct rule_entry ext_rule;
> exclude_rule ?
>
>> +
>> +static struct rule_entry *new_entry(const char *s, unsigned int len)
> The function name is too general as well. new_exclude_rule()
>
>> +{
>> +struct rule_entry *e = malloc(sizeof(struct rule_entry));
>> +
>> +if (!e)
>> +return NULL;
> How about using ERR_PTR(-ENOMEM) instead...
>
>> +
>> +e->name = strndup(s, len);
>> +if (!e->name)
>> +goto err_strdup;
>> +
>> +init_list_head(>hlist);
>> +init_list_head(>list);
>> +
>> +return e;
>> +
>> +err_strdup:
>> +free(e);
>> +return NULL;
> Same here.
>
>> +}
>> +
>> +struct rule_entry *erofs_
[PATCH v1] erofs-utils: introduce exclude dirs and files
From: Li Guifu
Add support exlcude directory when build image with long option
arg --exclude-path=dir1/dir2,dir11/dir22 or short option arg
-e dir1/dir2,dir11/dir22 that seperated by ','
Signed-off-by: Li Guifu
---
include/erofs/exclude.h | 28
lib/Makefile.am | 2 +-
lib/exclude.c | 149
lib/inode.c | 10 +++
mkfs/main.c | 13 +++-
5 files changed, 200 insertions(+), 2 deletions(-)
create mode 100644 include/erofs/exclude.h
create mode 100644 lib/exclude.c
diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
new file mode 100644
index 000..3f9fb4d
--- /dev/null
+++ b/include/erofs/exclude.h
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils\include\erofs\exclude.h
+ * Created by Li Guifu
+ */
+
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+
+struct rule_entry {
+ struct list_head list;
+ struct list_head hlist;
+ char *name;
+};
+
+void erofs_init_rules(void);
+void erofs_free_rules(void);
+int erofs_parse_exclude_path(const char *args);
+struct rule_entry *erofs_pattern_matched(const char *s);
+struct rule_entry *erofs_entry_matched(struct rule_entry *e,
+ const char *s, unsigned int len);
+
+static inline int erofs_is_pattern_end(struct rule_entry *e)
+{
+ return list_empty(>hlist);
+}
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 1ff81f9..e4b51e6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -3,7 +3,7 @@
noinst_LTLIBRARIES = liberofs.la
liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
- compress.c compressor.c
+ compress.c compressor.c exclude.c
liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
if ENABLE_LZ4
liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/exclude.c b/lib/exclude.c
new file mode 100644
index 000..25a9d32
--- /dev/null
+++ b/lib/exclude.c
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils\lib\exclude.c
+ * Created by Li Guifu
+ */
+
+#include
+#include
+#include
+
+#include "erofs/defs.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+static struct rule_entry ext_rule;
+
+static struct rule_entry *new_entry(const char *s, unsigned int len)
+{
+ struct rule_entry *e = malloc(sizeof(struct rule_entry));
+
+ if (!e)
+ return NULL;
+
+ e->name = strndup(s, len);
+ if (!e->name)
+ goto err_strdup;
+
+ init_list_head(>hlist);
+ init_list_head(>list);
+
+ return e;
+
+err_strdup:
+ free(e);
+ return NULL;
+}
+
+struct rule_entry *erofs_entry_matched(struct rule_entry *e,
+ const char *s, unsigned int len)
+{
+ struct rule_entry *pos;
+
+ while (*s == '/') {
+ s++;
+ len--;
+ }
+ list_for_each_entry(pos, >hlist, list) {
+ unsigned int l = strlen(pos->name);
+
+ if (l == len && !strcmp(pos->name, s))
+ return pos;
+ }
+
+ return NULL;
+}
+
+static int add_rules(struct rule_entry *e, const char *s)
+{
+ const char *ptr;
+ struct rule_entry *le;
+
+ while (*s == '/')
+ s++;
+ ptr = s;
+
+forward:
+ while(*ptr != '/' && *ptr != '\0')
+ ptr++;
+
+ le = erofs_entry_matched(e, s, ptr - s);
+ if (!le) {
+ struct rule_entry *me = new_entry(s, ptr - s);
+
+ if (!me)
+ return -ENOMEM;
+ list_add_tail(>list, >hlist);
+ le = me;
+ }
+ e = le;
+
+ if (*ptr++ != '\0') {
+ s = ptr;
+ goto forward;
+ }
+
+ return 0;
+}
+
+static void free_rules(struct list_head *h)
+{
+ struct rule_entry *e, *n;
+
+ list_for_each_entry_safe(e, n, h, list) {
+ list_del(>list);
+ free_rules(>hlist);
+ free(e->name);
+ free(e);
+ }
+}
+void erofs_init_rules(void)
+{
+ init_list_head(_rule.list);
+ init_list_head(_rule.hlist);
+ ext_rule.name = "/";
+}
+
+void erofs_free_rules(void)
+{
+ free_rules(_rule.hlist);
+}
+
+int erofs_parse_exclude_path(const char *args)
+{
+ const char *str, *ptr = args;
+ const char sep = ',';
+
+forward:
+ while(*ptr != sep && *ptr != '\0')
+ ptr++;
+
+ str = strndup(args, ptr - args);
+ if (!str)
+ goto err_free_paths;
+
+ if (add_rules(_rule, str))
+ goto err_free_paths;
+
+ if (*ptr++ != '\0') {
+ args = ptr;
+ goto forward;
+ }
+
+ return 0;
+
+err_free_paths:
[PATCH v1] erofs-utils: introduce exclude dirs and files
From: Li Guifu
Add support exlcude directory when build image with long option
arg --exclude-path=dir1/dir2,dir11/dir22 or short option arg
-e dir1/dir2,dir11/dir22 that seperated by ','
Signed-off-by: Li Guifu
---
include/erofs/exclude.h | 28
lib/Makefile.am | 2 +-
lib/exclude.c | 149
lib/inode.c | 10 +++
mkfs/main.c | 13 +++-
5 files changed, 200 insertions(+), 2 deletions(-)
create mode 100644 include/erofs/exclude.h
create mode 100644 lib/exclude.c
diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h
new file mode 100644
index 000..3f9fb4d
--- /dev/null
+++ b/include/erofs/exclude.h
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils\include\erofs\exclude.h
+ * Created by Li Guifu
+ */
+
+#ifndef __EROFS_EXCLUDE_H
+#define __EROFS_EXCLUDE_H
+
+struct rule_entry {
+ struct list_head list;
+ struct list_head hlist;
+ char *name;
+};
+
+void erofs_init_rules(void);
+void erofs_free_rules(void);
+int erofs_parse_exclude_path(const char *args);
+struct rule_entry *erofs_pattern_matched(const char *s);
+struct rule_entry *erofs_entry_matched(struct rule_entry *e,
+ const char *s, unsigned int len);
+
+static inline int erofs_is_pattern_end(struct rule_entry *e)
+{
+ return list_empty(>hlist);
+}
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 1ff81f9..e4b51e6 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -3,7 +3,7 @@
noinst_LTLIBRARIES = liberofs.la
liberofs_la_SOURCES = config.c io.c cache.c inode.c xattr.c \
- compress.c compressor.c
+ compress.c compressor.c exclude.c
liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
if ENABLE_LZ4
liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/exclude.c b/lib/exclude.c
new file mode 100644
index 000..25a9d32
--- /dev/null
+++ b/lib/exclude.c
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils\lib\exclude.c
+ * Created by Li Guifu
+ */
+
+#include
+#include
+#include
+
+#include "erofs/defs.h"
+#include "erofs/list.h"
+#include "erofs/print.h"
+#include "erofs/exclude.h"
+
+static struct rule_entry ext_rule;
+
+static struct rule_entry *new_entry(const char *s, unsigned int len)
+{
+ struct rule_entry *e = malloc(sizeof(struct rule_entry));
+
+ if (!e)
+ return NULL;
+
+ e->name = strndup(s, len);
+ if (!e->name)
+ goto err_strdup;
+
+ init_list_head(>hlist);
+ init_list_head(>list);
+
+ return e;
+
+err_strdup:
+ free(e);
+ return NULL;
+}
+
+struct rule_entry *erofs_entry_matched(struct rule_entry *e,
+ const char *s, unsigned int len)
+{
+ struct rule_entry *pos;
+
+ while (*s == '/') {
+ s++;
+ len--;
+ }
+ list_for_each_entry(pos, >hlist, list) {
+ unsigned int l = strlen(pos->name);
+
+ if (l == len && !strcmp(pos->name, s))
+ return pos;
+ }
+
+ return NULL;
+}
+
+static int add_rules(struct rule_entry *e, const char *s)
+{
+ const char *ptr;
+ struct rule_entry *le;
+
+ while (*s == '/')
+ s++;
+ ptr = s;
+
+forward:
+ while(*ptr != '/' && *ptr != '\0')
+ ptr++;
+
+ le = erofs_entry_matched(e, s, ptr - s);
+ if (!le) {
+ struct rule_entry *me = new_entry(s, ptr - s);
+
+ if (!me)
+ return -ENOMEM;
+ list_add_tail(>list, >hlist);
+ le = me;
+ }
+ e = le;
+
+ if (*ptr++ != '\0') {
+ s = ptr;
+ goto forward;
+ }
+
+ return 0;
+}
+
+static void free_rules(struct list_head *h)
+{
+ struct rule_entry *e, *n;
+
+ list_for_each_entry_safe(e, n, h, list) {
+ list_del(>list);
+ free_rules(>hlist);
+ free(e->name);
+ free(e);
+ }
+}
+void erofs_init_rules(void)
+{
+ init_list_head(_rule.list);
+ init_list_head(_rule.hlist);
+ ext_rule.name = "/";
+}
+
+void erofs_free_rules(void)
+{
+ free_rules(_rule.hlist);
+}
+
+int erofs_parse_exclude_path(const char *args)
+{
+ const char *str, *ptr = args;
+ const char sep = ',';
+
+forward:
+ while(*ptr != sep && *ptr != '\0')
+ ptr++;
+
+ str = strndup(args, ptr - args);
+ if (!str)
+ goto err_free_paths;
+
+ if (add_rules(_rule, str))
+ goto err_free_paths;
+
+ if (*ptr++ != '\0') {
+ args = ptr;
+ goto forward;
+ }
+
+ return 0;
+
+err_free_paths:
Re: [PATCH] erofs-utils: wrap up sb feature operations
> Add some helpers for shorter lines. No logic change. > > Signed-off-by: Gao Xiang > It looks good Reviewed-by: Li Guifu Because the pre email Li Guifu is not work at chinese mainland, so replace it with Li Guifu Thanks,
[PATCH v1] erofs-utils:code clean of write file
From: Li Guifu
Make a code clean at function erofs_write_file() which
has multi jump.
Signed-off-by: Li Guifu
---
lib/inode.c | 63 ++---
1 file changed, 31 insertions(+), 32 deletions(-)
diff --git a/lib/inode.c b/lib/inode.c
index 0e19b11..052315a 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -302,22 +302,10 @@ static bool erofs_file_is_compressible(struct erofs_inode
*inode)
return true;
}
-int erofs_write_file(struct erofs_inode *inode)
+int erofs_write_file_by_fd(int fd, struct erofs_inode *inode)
{
+ int ret;
unsigned int nblocks, i;
- int ret, fd;
-
- if (!inode->i_size) {
- inode->datalayout = EROFS_INODE_FLAT_PLAIN;
- return 0;
- }
-
- if (cfg.c_compr_alg_master && erofs_file_is_compressible(inode)) {
- ret = erofs_write_compressed_file(inode);
-
- if (!ret || ret != -ENOSPC)
- return ret;
- }
/* fallback to all data uncompressed */
inode->datalayout = EROFS_INODE_FLAT_INLINE;
@@ -327,47 +315,58 @@ int erofs_write_file(struct erofs_inode *inode)
if (ret)
return ret;
- fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
- if (fd < 0)
- return -errno;
-
for (i = 0; i < nblocks; ++i) {
char buf[EROFS_BLKSIZ];
ret = read(fd, buf, EROFS_BLKSIZ);
- if (ret != EROFS_BLKSIZ) {
- if (ret < 0)
- goto fail;
- close(fd);
- return -EAGAIN;
- }
+ if (ret != EROFS_BLKSIZ)
+ return -errno;
ret = blk_write(buf, inode->u.i_blkaddr + i, 1);
if (ret)
- goto fail;
+ return ret;
}
/* read the tail-end data */
inode->idata_size = inode->i_size % EROFS_BLKSIZ;
if (inode->idata_size) {
inode->idata = malloc(inode->idata_size);
- if (!inode->idata) {
- close(fd);
+ if (!inode->idata)
return -ENOMEM;
- }
ret = read(fd, inode->idata, inode->idata_size);
if (ret < inode->idata_size) {
free(inode->idata);
inode->idata = NULL;
- close(fd);
return -EIO;
}
}
- close(fd);
+
return 0;
-fail:
- ret = -errno;
+}
+
+int erofs_write_file(struct erofs_inode *inode)
+{
+ int ret, fd;
+
+ if (!inode->i_size) {
+ inode->datalayout = EROFS_INODE_FLAT_PLAIN;
+ return 0;
+ }
+
+ if (cfg.c_compr_alg_master && erofs_file_is_compressible(inode)) {
+ ret = erofs_write_compressed_file(inode);
+
+ if (!ret || ret != -ENOSPC)
+ return ret;
+ }
+
+ fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
+ if (fd < 0)
+ return -errno;
+
+ ret = erofs_write_file_by_fd(fd, inode);
+
close(fd);
return ret;
}
--
2.17.1
Re: [PATCH v2] erofs-utils: set up all compiler/linker variables independently
On 2019/11/14 21:45, Gao Xiang wrote: > Otherwise, the following checking will be effected > and it can cause unexpected behavior on configuring. > > Founded by the upcoming XZ algorithm patches. > > Signed-off-by: Gao Xiang > --- It looks good Reviewed-by: Li Guifu Tested-by: Li Guifu Thanks,
Re: [PATCH 2/2] erofs-utils: set up all compiler/linker variables independently
Gao Xiang Can you resend this patch ? It is conflict with uuid. uuid patch : https://lore.kernel.org/linux-erofs/d4d8127a-c452-f7d4-b3b1-50098cf07...@gmail.com/T/#u On 2019/11/12 19:26, Gao Xiang wrote: > Otherwise, the following checking will be effected > and it can cause unexpected behavior on configuring. > > Founded by the upcoming XZ algorithm patches. > > Signed-off-by: Gao Xiang > ---
Re: [PATCH v4] erofs-utils: support 128-bit filesystem UUID
On 2019/11/13 14:01, Gao Xiang wrote: > Complete filesystem UUID feature on userspace side. > > Cc: Chao Yu > Cc: Li Guifu > Signed-off-by: Gao Xiang > --- It looks good Reviewed-by: Li Guifu Tested-by: Li Guifu Thanks,
Re: [PATCH v9] erofs-utils: support calculating checksum of erofs blocks
On 2019/10/30 14:28, Gao Xiang wrote: > From: Pratik Shinde > > Added code for calculating crc of erofs blocks (4K size). > For now it calculates checksum of first block. but it can > be modified to calculate crc for any no. of blocks. > > Note that the first 1024 bytes are not checksumed to allow > for the installation of x86 boot sectors and other oddities. > > Fill 'feature_compat' field of erofs_super_block so that it > can be used on kernel side. also fixing one typo. > > Signed-off-by: Pratik Shinde > Reviewed-by: Chao Yu > Signed-off-by: Gao Xiang > --- > It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH v2] erofs-utils: add manual for mkfs.erofs
On 2019/11/4 15:28, Gao Xiang wrote: > This patch adds mkfs.erofs manpage, which is a requirement of > a debian binary package (See Debian Policy Manual section 12.1 [1].) > > [1] https://www.debian.org/doc/debian-policy/ch-docs.html#manual-pages > Signed-off-by: Gao Xiang > --- It looks good Reviewed-by: Li Guifu Tested-by: Li Guifu Thanks,
Re: [PATCH] erofs-utils: document more known matters to README
On 2019/10/23 17:20, Gao Xiang wrote: We are about to release erofs-utils 1.0, add more words to README on known fixed issues about lz4 upstream library. Cc: Li Guifu Cc: Chao Yu Cc: Yann Collet Signed-off-by: Gao Xiang It looks good Reviewed-by: Li Guifu Thanks,
Re: [PATCH] erofs-utils: simplify prints in usage()
On 2019/10/23 14:37, Gao Xiang wrote: Use a single puts instead, trivial update. Suggested-by: Li Guifu Signed-off-by: Gao Xiang --- It looks good Reviewed-by: Li Guifu Thanks,
[PATCH v2] erofs-utils: list available compressors for help command
From: Gao Xiang
Users can get knowledge of supported compression
algorithms then.
Signed-off-by: Gao Xiang
Signed-off-by: Li Guifu
---
include/erofs/compress.h | 2 ++
lib/compressor.c | 23 ++-
lib/compressor.h | 1 +
lib/compressor_lz4.c | 1 +
lib/compressor_lz4hc.c | 1 +
mkfs/main.c | 15 +++
6 files changed, 34 insertions(+), 9 deletions(-)
diff --git a/include/erofs/compress.h b/include/erofs/compress.h
index e0abb8f..fa91873 100644
--- a/include/erofs/compress.h
+++ b/include/erofs/compress.h
@@ -21,5 +21,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode);
int z_erofs_compress_init(void);
int z_erofs_compress_exit(void);
+const char *z_erofs_list_available_compressors(unsigned int i);
+
#endif
diff --git a/lib/compressor.c b/lib/compressor.c
index 8cc2f43..6502437 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -12,6 +12,15 @@
#define EROFS_CONFIG_COMPR_DEF_BOUNDARY(128)
+static struct erofs_compressor *compressors[] = {
+#if LZ4_ENABLED
+#if LZ4HC_ENABLED
+ _compressor_lz4hc,
+#endif
+ _compressor_lz4,
+#endif
+};
+
int erofs_compress_destsize(struct erofs_compress *c,
int compression_level,
void *src,
@@ -36,18 +45,14 @@ int erofs_compress_destsize(struct erofs_compress *c,
return ret;
}
+const char *z_erofs_list_available_compressors(unsigned int i)
+{
+ return (i >= ARRAY_SIZE(compressors)) ? NULL : compressors[i]->name;
+}
+
int erofs_compressor_init(struct erofs_compress *c,
char *alg_name)
{
- static struct erofs_compressor *compressors[] = {
-#if LZ4_ENABLED
-#if LZ4HC_ENABLED
- _compressor_lz4hc,
-#endif
- _compressor_lz4,
-#endif
- };
-
int ret, i;
/* should be written in "minimum compression ratio * 100" */
diff --git a/lib/compressor.h b/lib/compressor.h
index 6429b2a..4b76c4c 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -14,6 +14,7 @@
struct erofs_compress;
struct erofs_compressor {
+ const char *name;
int default_level;
int best_level;
diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c
index 0d33223..a9cbb80 100644
--- a/lib/compressor_lz4.c
+++ b/lib/compressor_lz4.c
@@ -39,6 +39,7 @@ static int compressor_lz4_init(struct erofs_compress *c,
}
struct erofs_compressor erofs_compressor_lz4 = {
+ .name = "lz4",
.default_level = 0,
.best_level = 0,
.init = compressor_lz4_init,
diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c
index 14e0175..a160c1a 100644
--- a/lib/compressor_lz4hc.c
+++ b/lib/compressor_lz4hc.c
@@ -52,6 +52,7 @@ static int compressor_lz4hc_init(struct erofs_compress *c,
}
struct erofs_compressor erofs_compressor_lz4hc = {
+ .name = "lz4hc",
.default_level = LZ4HC_CLEVEL_DEFAULT,
.best_level = LZ4HC_CLEVEL_MAX,
.init = compressor_lz4hc_init,
diff --git a/mkfs/main.c b/mkfs/main.c
index 31cf1c2..1161b3f 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -29,6 +29,19 @@ static struct option long_options[] = {
{0, 0, 0, 0},
};
+static void print_available_compressors(FILE *f, const char *delim)
+{
+ unsigned int i = 0;
+ const char *s;
+
+ while ((s = z_erofs_list_available_compressors(i)) != NULL) {
+ if (i++)
+ fputs(delim, f);
+ fputs(s, f);
+ }
+ fputc('\n', f);
+}
+
static void usage(void)
{
fprintf(stderr, "usage: [options] FILE DIRECTORY\n\n");
@@ -39,6 +52,8 @@ static void usage(void)
fprintf(stderr, " -EX[,...] X=extended options\n");
fprintf(stderr, " -T# set a fixed UNIX timestamp # to all
files\n");
fprintf(stderr, " --helpdisplay this help and exit\n");
+ fprintf(stderr, "\nAvailable compressors are: ");
+ print_available_compressors(stderr, ", ");
}
static int parse_extended_opts(const char *opts)
--
2.17.1
Re: [PATCH] erofs-utils: list available compressors for help command
Hi Gao Xiang
What do you think put the compressor's name into structure of
erofs_compressor ?
So compressor's name will be matched to its implement not represented
at two place.
On 2019/10/22 12:20, Gao Xiang wrote:
Users can get knowledge of supported compression
algorithms then.
Signed-off-by: Gao Xiang
---
include/erofs/compress.h | 2 ++
lib/compressor.c | 17 +
mkfs/main.c | 15 +++
3 files changed, 34 insertions(+)
diff --git a/include/erofs/compress.h b/include/erofs/compress.h
index e0abb8f1c422..fa918732b532 100644
--- a/include/erofs/compress.h
+++ b/include/erofs/compress.h
@@ -21,5 +21,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode);
int z_erofs_compress_init(void);
int z_erofs_compress_exit(void);
+const char *z_erofs_list_available_compressors(unsigned int i);
+
#endif
diff --git a/lib/compressor.c b/lib/compressor.c
index 8cc2f438479b..c593c769d46f 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -36,6 +36,23 @@ int erofs_compress_destsize(struct erofs_compress *c,
return ret;
}
+const char *z_erofs_list_available_compressors(unsigned int i)
+{
+ static const char *compressors[] = {
+#if LZ4_ENABLED
+#if LZ4HC_ENABLED
+ "lz4hc",
+#endif
+ "lz4",
+#endif
+ NULL,
+ };
+
+ if (i >= ARRAY_SIZE(compressors))
+ return NULL;
+ return compressors[i];
+}
+
int erofs_compressor_init(struct erofs_compress *c,
char *alg_name)
{
diff --git a/mkfs/main.c b/mkfs/main.c
index 31cf1c2408d1..1161b3f0f7cc 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -29,6 +29,19 @@ static struct option long_options[] = {
{0, 0, 0, 0},
};
+static void print_available_compressors(FILE *f, const char *delim)
+{
+ unsigned int i = 0;
+ const char *s;
+
+ while ((s = z_erofs_list_available_compressors(i)) != NULL) {
+ if (i++)
+ fputs(delim, f);
+ fputs(s, f);
+ }
+ fputc('\n', f);
+}
+
static void usage(void)
{
fprintf(stderr, "usage: [options] FILE DIRECTORY\n\n");
@@ -39,6 +52,8 @@ static void usage(void)
fprintf(stderr, " -EX[,...] X=extended options\n");
fprintf(stderr, " -T# set a fixed UNIX timestamp # to all
files\n");
fprintf(stderr, " --helpdisplay this help and exit\n");
+ fprintf(stderr, "\nAvailable compressors are: ");
+ print_available_compressors(stderr, ", ");
}
static int parse_extended_opts(const char *opts)
Re: [PATCH v2] erofs-utils: introduce long parameter option
On 2019/10/22 12:10, Gao Xiang wrote: From: Li Guifu Only long option "--help" is valid now. Signed-off-by: Li Guifu Signed-off-by: Gao Xiang --- It looks good Tested-by: Li Guifu Thanks,
Re: [PATCH v3] erofs-utils: use cmpsgn(x, y) for standardized large value comparsion
Previously, roundup(bb->buffers.off % EROFS_BLKSIZ, alignsize)
+ incr + extrasize is an unsigned 64bit value and sgn(x) didn't
work properly. Fix it.
Fixes: b0ca535297b6 ("erofs-utils: support 64-bit internal buffer cache")
Signed-off-by: Gao Xiang
---
It looks good
Reviewed-by: Li Guifu
Tested-by: Li Guifu
Thanks,
Re: [PATCH] erofs-utils: use cmpsgn(x, y) for standardized large value comparsion
Hi Gao Xiang,
It runs ok, but could you resend it to adjust line algin ?
>> -#define sgn(x) ({ \
>> +#define cmpsgn(x, y) ({ \
>>typeof(x) _x = (x); \
>> -(_x > 0) - (_x < 0); })
>> + typeof(y) _y = (y); \
>> +(_x > _y) - (_x < _y); }) > here
On Mon, Oct 14, 2019 at 07:30:48PM +0800, Gao Xiang wrote:
Previously, roundup(bb->buffers.off % EROFS_BLKSIZ, alignsize)
+ incr + extrasize is a unsigned 64bit value and sgn(x) didn't
work properly. Fix it.
Update commit message:
erofs-utils: use cmpsgn(x, y) for standardized large value comparsion
Previously, roundup(bb->buffers.off % EROFS_BLKSIZ, alignsize)
+ incr + extrasize is an unsigned 64bit value and sgn(x) didn't
work properly. Fix it.
Fixes: b0ca535297b6 ("erofs-utils: support 64-bit internal buffer cache")
Guifu, do you have time reviewing and testing those patches?
I'd like to release erofs-utils 1.0 for upstreaming AOSP these days...
Thanks,
Gao Xiang
Signed-off-by: Gao Xiang
---
include/erofs/defs.h | 5 +++--
lib/cache.c | 6 +++---
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/include/erofs/defs.h b/include/erofs/defs.h
index 15db4e3..db51350 100644
--- a/include/erofs/defs.h
+++ b/include/erofs/defs.h
@@ -136,9 +136,10 @@ typedef int64_t s64;
type __max2 = (y); \
__max1 > __max2 ? __max1: __max2; })
-#define sgn(x) ({ \
+#define cmpsgn(x, y) ({\
typeof(x) _x = (x); \
-(_x > 0) - (_x < 0); })
+ typeof(y) _y = (y); \
+(_x > _y) - (_x < _y); })
#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
diff --git a/lib/cache.c b/lib/cache.c
index 41d2d5d..e61b201 100644
--- a/lib/cache.c
+++ b/lib/cache.c
@@ -80,9 +80,9 @@ static int __erofs_battach(struct erofs_buffer_block *bb,
bool dryrun)
{
const erofs_off_t alignedoffset = roundup(bb->buffers.off, alignsize);
- const int oob = sgn(roundup(bb->buffers.off % EROFS_BLKSIZ,
- alignsize) + incr + extrasize -
- EROFS_BLKSIZ);
+ const int oob = cmpsgn(roundup(bb->buffers.off % EROFS_BLKSIZ,
+ alignsize) + incr + extrasize,
+ EROFS_BLKSIZ);
bool tailupdate = false;
erofs_blk_t blkaddr;
--
2.17.1
Re: [PATCH-v3] erofs-utils:code for calculating crc checksum of erofs blocks.
Hi Pratik, Gao Xiang
Can sbi.fearture merge into sbi.feature_incompat ?
If not, it will has two feature parameter
It has the blocks field in the super block, chksum_blocks duplicates it.
and chksum may be used to check whole erofs image file.
Added code for calculating crc of erofs blocks (4K size).for now it calculates
checksum of first block. but can modified to calculate crc for any no. of blocks
1) Added 'chksum_blocks' field to erofs_super_block
2) removed unnecessary prints
3) moved EROFS_FEATURE_SB_CHKSUM include/erofs_fs.h
Signed-off-by: Pratik Shinde
---
include/erofs/internal.h | 1 +
include/erofs/io.h | 8 ++
include/erofs_fs.h | 3 ++-
lib/io.c | 27 +++
mkfs/main.c | 70
5 files changed, 108 insertions(+), 1 deletion(-)
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 5384946..53335bc 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -55,6 +55,7 @@ struct erofs_sb_info {
u32 feature_incompat;
u64 build_time;
u32 build_time_nsec;
+ u32 feature;
};
/* global sbi */
diff --git a/include/erofs/io.h b/include/erofs/io.h
index 9775047..e0ca8d9 100644
--- a/include/erofs/io.h
+++ b/include/erofs/io.h
@@ -19,6 +19,7 @@
int dev_open(const char *devname);
void dev_close(void);
int dev_write(const void *buf, u64 offset, size_t len);
+int dev_read(void *buf, u64 offset, size_t len);
int dev_fillzero(u64 offset, size_t len, bool padding);
int dev_fsync(void);
int dev_resize(erofs_blk_t nblocks);
@@ -31,5 +32,12 @@ static inline int blk_write(const void *buf, erofs_blk_t
blkaddr,
blknr_to_addr(nblocks));
}
+static inline int blk_read(void *buf, erofs_blk_t start,
+ u32 nblocks)
+{
+ return dev_read(buf, blknr_to_addr(start),
+blknr_to_addr(nblocks));
+}
+
#endif
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index f29aa25..8bd29d6 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -19,6 +19,7 @@
*/
#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x0001
#define EROFS_ALL_FEATURE_INCOMPAT
EROFS_FEATURE_INCOMPAT_LZ4_0PADDING
+#define EROFS_FEATURE_SB_CHKSUM0x0001
/* 128-byte erofs on-disk super block */
struct erofs_super_block {
@@ -39,7 +40,7 @@ struct erofs_super_block {
__u8 uuid[16]; /* 128-bit uuid for volume */
__u8 volume_name[16]; /* volume name */
__le32 feature_incompat;
-
+ __le32 chksum_blocks; /* number of blocks used for checksum */
__u8 reserved2[44];
};
diff --git a/lib/io.c b/lib/io.c
index 7f5f94d..52f9424 100644
--- a/lib/io.c
+++ b/lib/io.c
@@ -207,3 +207,30 @@ int dev_resize(unsigned int blocks)
return dev_fillzero(st.st_size, length, true);
}
+int dev_read(void *buf, u64 offset, size_t len)
+{
+ int ret;
+
+ if (cfg.c_dry_run)
+ return 0;
+
+ if (!buf) {
+ erofs_err("buf is NULL");
+ return -EINVAL;
+ }
+ if (offset >= erofs_devsz || len > erofs_devsz ||
+ offset > erofs_devsz - len) {
+ erofs_err("read posion[%" PRIu64 ", %zd] is too large beyond"
+ "the end of device(%" PRIu64 ").",
+ offset, len, erofs_devsz);
+ return -EINVAL;
+ }
+
+ ret = pread64(erofs_devfd, buf, len, (off64_t)offset);
+ if (ret != (int)len) {
+ erofs_err("Failed to read data from device - %s:[%" PRIu64 ",
%zd].",
+ erofs_devname, offset, len);
+ return -errno;
+ }
+ return 0;
+}
diff --git a/mkfs/main.c b/mkfs/main.c
index 91a018f..0633e83 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -22,6 +22,9 @@
#define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
+/* number of blocks for calculating checksum */
+#define EROFS_CKSUM_BLOCKS 1
+
static void usage(void)
{
fprintf(stderr, "usage: [options] FILE DIRECTORY\n\n");
@@ -85,6 +88,10 @@ static int parse_extended_opts(const char *opts)
return -EINVAL;
cfg.c_force_inodeversion = FORCE_INODE_EXTENDED;
}
+
+ if (MATCH_EXTENTED_OPT("nocrc", token, keylen)) {
+ sbi.feature &= ~EROFS_FEATURE_SB_CHKSUM;
+ }
}
return 0;
}
@@ -180,6 +187,8 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head
*bh,
.meta_blkaddr = sbi.meta_blkaddr,
.xattr_blkaddr = 0,
.feature_incompat = cpu_to_le32(sbi.feature_incompat),
+ .checksum = 0,
+ .chksum_blocks = cpu_to_le32(EROFS_CKSUM_BLOCKS)
};
const unsigned int sb_blksize =
Re: [PATCH] erofs-utils: fix old kernel compatibility for non-lz4 compression
If primary algorithm is not lz4 (e.g. compression off), clear EROFS_FEATURE_INCOMPAT_LZ4_0PADDING for old kernel (upstream kernel <= 5.2.y) compatibility. Signed-off-by: Gao Xiang --- Reviewed-by: Li Guifu Thanks,
[PATCH v4][ 2/2] erofs-utils: fix error handler notes when parameter miss
If a parameter isnot input, mkfs's error handler cannot give
a correct notes, fix it.
Signed-off-by: Li Guifu
---
mkfs/main.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/mkfs/main.c b/mkfs/main.c
index 77a4b78..adfc79c 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -143,7 +143,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
if (!cfg.c_img_path)
return -ENOMEM;
- if (optind > argc) {
+ if (optind >= argc) {
erofs_err("Source directory is missing");
return -EINVAL;
}
--
2.17.1
[PATCH v4][ 1/2] erofs-utils: introduce fixed UNIX timestamp
Introduce option "-T" for UNIX timestamp.
Signed-off-by: Li Guifu
---
include/erofs/config.h | 1 +
lib/config.c | 1 +
mkfs/main.c| 11 +--
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index fde936c..8df05a1 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -30,6 +30,7 @@ struct erofs_configure {
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
+ long long c_unix_timestamp;
};
extern struct erofs_configure cfg;
diff --git a/lib/config.c b/lib/config.c
index dc10754..cee835c 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -24,6 +24,7 @@ void erofs_init_configure(void)
sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
+ cfg.c_unix_timestamp = -1;
}
void erofs_show_config(void)
diff --git a/mkfs/main.c b/mkfs/main.c
index 978c5b4..77a4b78 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -30,6 +30,7 @@ static void usage(void)
fprintf(stderr, " -zX[,Y] X=compressor (Y=compression level,
optional)\n");
fprintf(stderr, " -d# set output message level to # (maximum
9)\n");
fprintf(stderr, " -EX[,...] X=extended options\n");
+ fprintf(stderr, " -T# set a fixed UNIX timestamp # to all
files\n");
}
static int parse_extended_opts(const char *opts)
@@ -93,7 +94,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
{
int opt, i;
- while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+ while ((opt = getopt(argc, argv, "d:z:E:T:")) != -1) {
switch (opt) {
case 'z':
if (!optarg) {
@@ -126,6 +127,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
if (opt)
return opt;
break;
+ case 'T':
+ cfg.c_unix_timestamp = atoll(optarg);
+ break;
default: /* '?' */
return -EINVAL;
@@ -224,7 +228,10 @@ int main(int argc, char **argv)
return 1;
}
- if (!gettimeofday(, NULL)) {
+ if (cfg.c_unix_timestamp != -1) {
+ sbi.build_time = cfg.c_unix_timestamp;
+ sbi.build_time_nsec = 0;
+ } else if (!gettimeofday(, NULL)) {
sbi.build_time = t.tv_sec;
sbi.build_time_nsec = t.tv_usec;
}
--
2.17.1
[PATCH v1] erofs-utils: introduce fixed timestamp
Introduce option "-T" for timestamp.
Signed-off-by: Li Guifu
---
include/erofs/config.h | 3 +++
lib/config.c | 1 +
mkfs/main.c| 14 --
3 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index fde936c..61f5c71 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -11,6 +11,8 @@
#include "defs.h"
+#define EROFS_FIXED_TIMESTAMP 1569859200 // 2019/10/1 00:00:00
+
enum {
FORCE_INODE_COMPACT = 1,
FORCE_INODE_EXTENDED,
@@ -30,6 +32,7 @@ struct erofs_configure {
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
+ long long c_timestamp;
};
extern struct erofs_configure cfg;
diff --git a/lib/config.c b/lib/config.c
index dc10754..6141497 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -24,6 +24,7 @@ void erofs_init_configure(void)
sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
+ cfg.c_timestamp = -1;
}
void erofs_show_config(void)
diff --git a/mkfs/main.c b/mkfs/main.c
index 978c5b4..85b8f34 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -30,6 +30,7 @@ static void usage(void)
fprintf(stderr, " -zX[,Y] X=compressor (Y=compression level,
optional)\n");
fprintf(stderr, " -d# set output message level to # (maximum
9)\n");
fprintf(stderr, " -EX[,...] X=extended options\n");
+ fprintf(stderr, " -T# set a fixed timestamp to files and dirs\n");
}
static int parse_extended_opts(const char *opts)
@@ -93,7 +94,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
{
int opt, i;
- while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+ while ((opt = getopt(argc, argv, "d:z:E:T::")) != -1) {
switch (opt) {
case 'z':
if (!optarg) {
@@ -126,6 +127,12 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
if (opt)
return opt;
break;
+ case 'T':
+ if (optarg)
+ cfg.c_timestamp = strtoll(optarg, NULL, 10);
+ else
+ cfg.c_timestamp = EROFS_FIXED_TIMESTAMP;
+ break;
default: /* '?' */
return -EINVAL;
@@ -224,7 +231,10 @@ int main(int argc, char **argv)
return 1;
}
- if (!gettimeofday(, NULL)) {
+ if (cfg.c_timestamp != -1) {
+ sbi.build_time = cfg.c_timestamp;
+ sbi.build_time_nsec = 0;
+ } else if (!gettimeofday(, NULL)) {
sbi.build_time = t.tv_sec;
sbi.build_time_nsec = t.tv_usec;
}
--
2.17.1
[PATCH preview][] erofs-utils:introduce fixed timestamp using "-T"
Introduce option "-T" for timestamp.
Signed-off-by: Li Guifu
---
include/erofs/config.h | 3 +++
lib/config.c | 1 +
mkfs/main.c| 14 --
3 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index fde936c..61f5c71 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -11,6 +11,8 @@
#include "defs.h"
+#define EROFS_FIXED_TIMESTAMP 1569859200 // 2019/10/1 00:00:00
+
enum {
FORCE_INODE_COMPACT = 1,
FORCE_INODE_EXTENDED,
@@ -30,6 +32,7 @@ struct erofs_configure {
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
+ long long c_timestamp;
};
extern struct erofs_configure cfg;
diff --git a/lib/config.c b/lib/config.c
index dc10754..6141497 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -24,6 +24,7 @@ void erofs_init_configure(void)
sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
+ cfg.c_timestamp = -1;
}
void erofs_show_config(void)
diff --git a/mkfs/main.c b/mkfs/main.c
index 978c5b4..85b8f34 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -30,6 +30,7 @@ static void usage(void)
fprintf(stderr, " -zX[,Y] X=compressor (Y=compression level,
optional)\n");
fprintf(stderr, " -d# set output message level to # (maximum
9)\n");
fprintf(stderr, " -EX[,...] X=extended options\n");
+ fprintf(stderr, " -T# set a fixed timestamp to files and dirs\n");
}
static int parse_extended_opts(const char *opts)
@@ -93,7 +94,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
{
int opt, i;
- while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+ while ((opt = getopt(argc, argv, "d:z:E:T::")) != -1) {
switch (opt) {
case 'z':
if (!optarg) {
@@ -126,6 +127,12 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
if (opt)
return opt;
break;
+ case 'T':
+ if (optarg)
+ cfg.c_timestamp = strtoll(optarg, NULL, 10);
+ else
+ cfg.c_timestamp = EROFS_FIXED_TIMESTAMP;
+ break;
default: /* '?' */
return -EINVAL;
@@ -224,7 +231,10 @@ int main(int argc, char **argv)
return 1;
}
- if (!gettimeofday(, NULL)) {
+ if (cfg.c_timestamp != -1) {
+ sbi.build_time = cfg.c_timestamp;
+ sbi.build_time_nsec = 0;
+ } else if (!gettimeofday(, NULL)) {
sbi.build_time = t.tv_sec;
sbi.build_time_nsec = t.tv_usec;
}
--
2.17.1
[PATCH preview][] erofs-utils:intriduce fixed timestamp using "-T"
Introduce option "-T" for timestamp.
Signed-off-by: Li Guifu
---
include/erofs/config.h | 3 +++
lib/config.c | 1 +
mkfs/main.c| 14 --
3 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index fde936c..61f5c71 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -11,6 +11,8 @@
#include "defs.h"
+#define EROFS_FIXED_TIMESTAMP 1569859200 // 2019/10/1 00:00:00
+
enum {
FORCE_INODE_COMPACT = 1,
FORCE_INODE_EXTENDED,
@@ -30,6 +32,7 @@ struct erofs_configure {
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
+ long long c_timestamp;
};
extern struct erofs_configure cfg;
diff --git a/lib/config.c b/lib/config.c
index dc10754..6141497 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -24,6 +24,7 @@ void erofs_init_configure(void)
sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING;
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
+ cfg.c_timestamp = -1;
}
void erofs_show_config(void)
diff --git a/mkfs/main.c b/mkfs/main.c
index 978c5b4..85b8f34 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -30,6 +30,7 @@ static void usage(void)
fprintf(stderr, " -zX[,Y] X=compressor (Y=compression level,
optional)\n");
fprintf(stderr, " -d# set output message level to # (maximum
9)\n");
fprintf(stderr, " -EX[,...] X=extended options\n");
+ fprintf(stderr, " -T# set a fixed timestamp to files and dirs\n");
}
static int parse_extended_opts(const char *opts)
@@ -93,7 +94,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
{
int opt, i;
- while ((opt = getopt(argc, argv, "d:z:E:")) != -1) {
+ while ((opt = getopt(argc, argv, "d:z:E:T::")) != -1) {
switch (opt) {
case 'z':
if (!optarg) {
@@ -126,6 +127,12 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
if (opt)
return opt;
break;
+ case 'T':
+ if (optarg)
+ cfg.c_timestamp = strtoll(optarg, NULL, 10);
+ else
+ cfg.c_timestamp = EROFS_FIXED_TIMESTAMP;
+ break;
default: /* '?' */
return -EINVAL;
@@ -224,7 +231,10 @@ int main(int argc, char **argv)
return 1;
}
- if (!gettimeofday(, NULL)) {
+ if (cfg.c_timestamp != -1) {
+ sbi.build_time = cfg.c_timestamp;
+ sbi.build_time_nsec = 0;
+ } else if (!gettimeofday(, NULL)) {
sbi.build_time = t.tv_sec;
sbi.build_time_nsec = t.tv_usec;
}
--
2.17.1
