Limit the max input stream size by adding segment compression
(e.g. 4M segment size), it will benefits:
- more friendly to block diff (and more details about this);
- it can also be used for parallel compression in the same file.
Signed-off-by: Li Guifu <bluce....@aliyun.com>
---
include/erofs/config.h | 1 +
lib/compress.c | 11 +++++++++++
lib/config.c | 1 +
3 files changed, 13 insertions(+)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 2f09749..9125c1e 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -36,6 +36,7 @@ struct erofs_configure {
char *c_src_path;
char *c_compr_alg_master;
int c_compr_level_master;
+ unsigned int c_compr_seg_size; /* max segment compress size */
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
diff --git a/lib/compress.c b/lib/compress.c
index 6cc68ed..8fdbfb2 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -32,6 +32,8 @@ struct z_erofs_vle_compress_ctx {
erofs_blk_t blkaddr; /* pointing to the next blkaddr */
u16 clusterofs;
+ unsigned int comprlimits;
+ unsigned int comr_seg_size;
};
#define Z_EROFS_LEGACY_MAP_HEADER_SIZE \
@@ -158,6 +160,12 @@ static int vle_compress_one(struct erofs_inode *inode,
while (len) {
bool raw;
+ if (ctx->comprlimits >= ctx->comr_seg_size ||
+ ctx->comprlimits + EROFS_BLKSIZ >= ctx->comr_seg_size) {
+ ctx->comprlimits = 0;
+ goto nocompression;
+ }
+
if (len <= EROFS_BLKSIZ) {
if (final)
goto nocompression;
@@ -202,6 +210,7 @@ nocompression:
++ctx->blkaddr;
len -= count;
+ ctx->comprlimits += count;
if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
const unsigned int qh_aligned =
@@ -422,6 +431,8 @@ int erofs_write_compressed_file(struct erofs_inode *inode)
ctx.head = ctx.tail = 0;
ctx.clusterofs = 0;
remaining = inode->i_size;
+ ctx.comprlimits = 0;
+ ctx.comr_seg_size = cfg.c_compr_seg_size;
while (remaining) {
const u64 readcount = min_t(u64, remaining,
diff --git a/lib/config.c b/lib/config.c
index da0c260..1c39403 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -23,6 +23,7 @@ void erofs_init_configure(void)
cfg.c_force_inodeversion = 0;
cfg.c_inline_xattr_tolerance = 2;
cfg.c_unix_timestamp = -1;
+ cfg.c_compr_seg_size = 1024U * EROFS_BLKSIZ;
}
void erofs_show_config(void)
--
2.17.1
