EROFS supports incremental-style builds without rewriting the entire filesystem metadata, since it does not have classic centralized inode tables or SquashFS-style directory tables.
This commit adds incremental build support for local files: $ mkfs.erofs foo.erofs layer0/ $ mkfs.erofs --incremental=data foo.erofs layer1/ ... OverlayFS whiteouts are supported for replacing specific directory entries. Additionally, these extra whiteouts can be dropped from the final image using `--ovlfs-strip` Signed-off-by: Gao Xiang <[email protected]> --- lib/inode.c | 54 +++++++++++++++++++++++++++--------------- lib/liberofs_rebuild.h | 4 ++-- lib/rebuild.c | 26 +++++++++++++------- mkfs/main.c | 1 - 4 files changed, 54 insertions(+), 31 deletions(-) diff --git a/lib/inode.c b/lib/inode.c index 810ffc2..74c9645 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -1294,6 +1294,7 @@ struct erofs_inode *erofs_new_inode(struct erofs_sb_info *sbi) static struct erofs_inode *erofs_iget_from_local(struct erofs_importer *im, const char *path) { + const struct erofs_importer_params *params = im->params; struct erofs_sb_info *sbi = im->sbi; struct erofs_inode *inode; struct stat st; @@ -1308,7 +1309,7 @@ static struct erofs_inode *erofs_iget_from_local(struct erofs_importer *im, * hard-link, just return it. Also don't lookup for directories * since hard-link directory isn't allowed. */ - if (!S_ISDIR(st.st_mode) && !im->params->hard_dereference) { + if (!S_ISDIR(st.st_mode) && !params->hard_dereference) { inode = erofs_iget(st.st_dev, st.st_ino); if (inode) return inode; @@ -1669,6 +1670,8 @@ static int erofs_mkfs_import_localdir(struct erofs_importer *im, struct erofs_in ret = PTR_ERR(inode); goto err_closedir; } + if (!dir->whiteouts && erofs_inode_is_whiteout(inode)) + dir->whiteouts = true; d->inode = inode; d->type = erofs_mode_to_ftype(inode->i_mode); __nlink += S_ISDIR(inode->i_mode); @@ -1712,16 +1715,15 @@ static void erofs_dentry_kill(struct erofs_dentry *d) free(d); } -static int erofs_mkfs_handle_directory(struct erofs_importer *im, - struct erofs_inode *dir, - bool rebuild, - bool incremental) +static int erofs_prepare_dir_inode(struct erofs_importer *im, + struct erofs_inode *dir, + bool rebuild, + bool incremental) { struct erofs_sb_info *sbi = im->sbi; struct erofs_dentry *d, *n; unsigned int i_nlink; u64 nr_subdirs; - bool delwht = im->params->ovlfs_strip && dir->whiteouts; int ret; nr_subdirs = 0; @@ -1733,11 +1735,6 @@ static int erofs_mkfs_handle_directory(struct erofs_importer *im, erofs_dentry_kill(d); continue; } - if (delwht && erofs_dentry_is_wht(sbi, d)) { - erofs_dbg("remove whiteout %s", d->inode->i_srcpath); - erofs_dentry_kill(d); - continue; - } i_nlink += (d->type == EROFS_FT_DIR); ++nr_subdirs; } @@ -1749,6 +1746,22 @@ static int erofs_mkfs_handle_directory(struct erofs_importer *im, return ret; } + if (incremental && dir->dev == sbi->dev && !dir->opaque) { + ret = erofs_rebuild_load_basedir(dir, &nr_subdirs, &i_nlink); + if (ret) + return ret; + } + if (im->params->ovlfs_strip && dir->whiteouts) { + list_for_each_entry_safe(d, n, &dir->i_subdirs, d_child) { + if (erofs_dentry_is_wht(sbi, d)) { + erofs_dbg("remove whiteout %s", + d->inode->i_srcpath); + erofs_dentry_kill(d); + --nr_subdirs; + continue; + } + } + } DBG_BUGON(nr_subdirs + 2 < i_nlink); ret = erofs_prepare_dir_file(im, dir, nr_subdirs); if (ret) @@ -1769,8 +1782,7 @@ static int erofs_mkfs_handle_directory(struct erofs_importer *im, else dir->i_nlink = 1; } - - return erofs_mkfs_go(im, EROFS_MKFS_JOB_DIR, &dir, sizeof(dir)); + return 0; } static int erofs_mkfs_begin_nondirectory(struct erofs_importer *im, @@ -1830,10 +1842,9 @@ static int erofs_mkfs_handle_inode(struct erofs_importer *im, inode->inode_isize = sizeof(struct erofs_inode_compact); } - if (incremental && S_ISDIR(inode->i_mode) && - inode->dev == inode->sbi->dev && !inode->opaque) { - ret = erofs_rebuild_load_basedir(inode); - if (ret) + if (S_ISDIR(inode->i_mode)) { + ret = erofs_prepare_dir_inode(im, inode, rebuild, incremental); + if (ret < 0) return ret; } @@ -1856,8 +1867,8 @@ static int erofs_mkfs_handle_inode(struct erofs_importer *im, if (!S_ISDIR(inode->i_mode)) { ret = erofs_mkfs_begin_nondirectory(im, inode); } else { - ret = erofs_mkfs_handle_directory(im, inode, - rebuild, incremental); + ret = erofs_mkfs_go(im, EROFS_MKFS_JOB_DIR, &inode, + sizeof(inode)); } erofs_info("file %s dumped (mode %05o)", *relpath ? relpath : "/", inode->i_mode); @@ -2071,6 +2082,11 @@ fail: int erofs_importer_load_tree(struct erofs_importer *im, bool rebuild, bool incremental) { + if (__erofs_unlikely(incremental && erofs_sb_has_metabox(im->sbi))) { + erofs_err("Metadata-compressed filesystems don't support incremental builds for now"); + return -EOPNOTSUPP; + } + return erofs_mkfs_build_tree(&((struct erofs_mkfs_buildtree_ctx) { .im = im, .rebuild = rebuild, diff --git a/lib/liberofs_rebuild.h b/lib/liberofs_rebuild.h index 1eb79cf..69802fb 100644 --- a/lib/liberofs_rebuild.h +++ b/lib/liberofs_rebuild.h @@ -16,6 +16,6 @@ struct erofs_dentry *erofs_rebuild_get_dentry(struct erofs_inode *pwd, int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi, enum erofs_rebuild_datamode mode); -int erofs_rebuild_load_basedir(struct erofs_inode *dir); - +int erofs_rebuild_load_basedir(struct erofs_inode *dir, u64 *nr_subdirs, + unsigned int *i_nlink); #endif diff --git a/lib/rebuild.c b/lib/rebuild.c index 83e30fd..c5b44d5 100644 --- a/lib/rebuild.c +++ b/lib/rebuild.c @@ -274,15 +274,18 @@ static int erofs_rebuild_update_inode(struct erofs_sb_info *dst_sb, return err; } -/* - * @mergedir: parent directory in the merged tree - * @ctx.dir: parent directory when itering erofs_iterate_dir() - * @datamode: indicate how to import inode data - */ struct erofs_rebuild_dir_context { + /* @ctx.dir: parent directory when itering erofs_iterate_dir() */ struct erofs_dir_context ctx; - struct erofs_inode *mergedir; - enum erofs_rebuild_datamode datamode; + struct erofs_inode *mergedir; /* parent directory in the merged tree */ + union { + /* indicate how to import inode data */ + enum erofs_rebuild_datamode datamode; + struct { + u64 *nr_subdirs; + unsigned int *i_nlink; + }; + }; }; static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx) @@ -458,8 +461,8 @@ int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi, static int erofs_rebuild_basedir_dirent_iter(struct erofs_dir_context *ctx) { struct erofs_rebuild_dir_context *rctx = (void *)ctx; - struct erofs_inode *dir = ctx->dir; struct erofs_inode *mergedir = rctx->mergedir; + struct erofs_inode *dir = ctx->dir; struct erofs_dentry *d; char *dname; bool dumb; @@ -484,6 +487,8 @@ static int erofs_rebuild_basedir_dirent_iter(struct erofs_dir_context *ctx) d->validnid = true; if (!mergedir->whiteouts && erofs_dentry_is_wht(dir->sbi, d)) mergedir->whiteouts = true; + *rctx->i_nlink += (ctx->de_ftype == EROFS_FT_DIR); + ++*rctx->nr_subdirs; } else if (__erofs_unlikely(d->validnid)) { /* The base image appears to be corrupted */ DBG_BUGON(1); @@ -508,7 +513,8 @@ out: return ret; } -int erofs_rebuild_load_basedir(struct erofs_inode *dir) +int erofs_rebuild_load_basedir(struct erofs_inode *dir, u64 *nr_subdirs, + unsigned int *i_nlink) { struct erofs_inode fakeinode = { .sbi = dir->sbi, @@ -540,6 +546,8 @@ int erofs_rebuild_load_basedir(struct erofs_inode *dir) .ctx.dir = &fakeinode, .ctx.cb = erofs_rebuild_basedir_dirent_iter, .mergedir = dir, + .nr_subdirs = nr_subdirs, + .i_nlink = i_nlink, }; return erofs_iterate_dir(&ctx.ctx, false); } diff --git a/mkfs/main.c b/mkfs/main.c index 7a538bd..f3cf24e 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -1835,7 +1835,6 @@ int main(int argc, char **argv) err = PTR_ERR(root); goto exit; } - incremental_mode = false; } else { root = erofs_rebuild_make_root(&g_sbi); if (IS_ERR(root)) { -- 2.43.5
