This patch introduces experimental support for merging multiple source images in mkfs. Each source image becomes a directory directly under root and keeps its UUID stored as a device table tag. The raw block data from each source is copied using erofs_copy_file_range. We preserve the file metadata and layout (FLAT_PLAIN and FLAT_INLINE). Symlink paths are handled by reading and copy link targets.
This does not yet support chunk-based files at this time or compressed images. Signed-off-by: Lucas Karpinski <[email protected]> --- lib/cache.c | 6 +++ lib/liberofs_cache.h | 1 + lib/liberofs_rebuild.h | 4 ++ lib/rebuild.c | 112 +++++++++++++++++++++++++++++++++++++++++++++++-- mkfs/main.c | 44 ++++++++++++++----- 5 files changed, 154 insertions(+), 13 deletions(-) diff --git a/lib/cache.c b/lib/cache.c index 4c7c386..49742bc 100644 --- a/lib/cache.c +++ b/lib/cache.c @@ -544,6 +544,12 @@ erofs_blk_t erofs_total_metablocks(struct erofs_bufmgr *bmgr) return bmgr->metablkcnt; } +void erofs_bset_tail(struct erofs_bufmgr *bmgr, erofs_blk_t blkaddr) +{ + if (blkaddr > bmgr->tail_blkaddr) + bmgr->tail_blkaddr = blkaddr; +} + void erofs_buffer_exit(struct erofs_bufmgr *bmgr) { DBG_BUGON(__erofs_bflush(bmgr, NULL, true)); diff --git a/lib/liberofs_cache.h b/lib/liberofs_cache.h index baac609..55e8f25 100644 --- a/lib/liberofs_cache.h +++ b/lib/liberofs_cache.h @@ -138,6 +138,7 @@ int erofs_bflush(struct erofs_bufmgr *bmgr, struct erofs_buffer_block *bb); void erofs_bdrop(struct erofs_buffer_head *bh, bool tryrevoke); +void erofs_bset_tail(struct erofs_bufmgr *bmgr, erofs_blk_t blkaddr); void erofs_buffer_exit(struct erofs_bufmgr *bmgr); #ifdef __cplusplus diff --git a/lib/liberofs_rebuild.h b/lib/liberofs_rebuild.h index d8c4c8a..fba7f39 100644 --- a/lib/liberofs_rebuild.h +++ b/lib/liberofs_rebuild.h @@ -17,6 +17,10 @@ int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi, enum erofs_rebuild_datamode mode, erofs_blk_t uniaddr_offset); +int erofs_rebuild_copy_src(struct erofs_sb_info *sbi, + struct erofs_sb_info *src, + struct erofs_device_info *dev); + int erofs_rebuild_load_basedir(struct erofs_inode *dir, u64 *nr_subdirs, unsigned int *i_nlink); #endif diff --git a/lib/rebuild.c b/lib/rebuild.c index 7e62bc9..451307a 100644 --- a/lib/rebuild.c +++ b/lib/rebuild.c @@ -14,8 +14,10 @@ #include "erofs/xattr.h" #include "erofs/blobchunk.h" #include "erofs/internal.h" +#include "erofs/io.h" #include "liberofs_rebuild.h" #include "liberofs_uuid.h" +#include "liberofs_cache.h" #ifdef HAVE_LINUX_AUFS_TYPE_H #include <linux/aufs_type.h> @@ -221,9 +223,60 @@ err: return ret; } +static int erofs_rebuild_write_full_data(struct erofs_inode *inode, + erofs_blk_t uniaddr_offset) +{ + struct erofs_sb_info *src_sbi = inode->sbi; + int err = 0; + + if (inode->datalayout == EROFS_INODE_FLAT_PLAIN) { + if (inode->u.i_blkaddr != EROFS_NULL_ADDR) + inode->u.i_blkaddr += uniaddr_offset; + } else if (inode->datalayout == EROFS_INODE_FLAT_INLINE) { + erofs_blk_t nblocks = erofs_blknr(src_sbi, inode->i_size); + unsigned int inline_size = inode->i_size % erofs_blksiz(src_sbi); + + if (nblocks > 0 && inode->u.i_blkaddr != EROFS_NULL_ADDR) + inode->u.i_blkaddr += uniaddr_offset; + + inode->idata_size = inline_size; + if (inline_size > 0) { + struct erofs_vfile vf; + erofs_off_t tail_offset = erofs_pos(src_sbi, nblocks); + + inode->idata = malloc(inline_size); + if (!inode->idata) + return -ENOMEM; + err = erofs_iopen(&vf, inode); + if (err) { + free(inode->idata); + inode->idata = NULL; + return err; + } + err = erofs_pread(&vf, inode->idata, inline_size, + tail_offset); + if (err) { + free(inode->idata); + inode->idata = NULL; + return err; + } + } + } else if (inode->datalayout == EROFS_INODE_CHUNK_BASED) { + erofs_err("chunk-based files not yet supported: %s", + inode->i_srcpath); + err = -EOPNOTSUPP; + } else if (is_inode_layout_compression(inode)) { + erofs_err("compressed files not yet supported: %s", + inode->i_srcpath); + err = -EOPNOTSUPP; + } + return err; +} + static int erofs_rebuild_update_inode(struct erofs_sb_info *dst_sb, struct erofs_inode *inode, - enum erofs_rebuild_datamode datamode) + enum erofs_rebuild_datamode datamode, + erofs_blk_t uniaddr_offset) { int err = 0; @@ -265,6 +318,8 @@ static int erofs_rebuild_update_inode(struct erofs_sb_info *dst_sb, err = erofs_rebuild_write_blob_index(dst_sb, inode); else if (datamode == EROFS_REBUILD_DATA_RESVSP) inode->datasource = EROFS_INODE_DATA_SOURCE_RESVSP; + else if (datamode == EROFS_REBUILD_DATA_FULL) + err = erofs_rebuild_write_full_data(inode, uniaddr_offset); else err = -EOPNOTSUPP; break; @@ -387,7 +442,8 @@ static int erofs_rebuild_dirent_iter(struct erofs_dir_context *ctx) inode->i_nlink = 1; ret = erofs_rebuild_update_inode(&g_sbi, inode, - rctx->datamode); + rctx->datamode, + rctx->uniaddr_offset); if (ret) { erofs_iput(inode); goto out; @@ -425,6 +481,7 @@ int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi, { struct erofs_inode inode = {}; struct erofs_rebuild_dir_context ctx; + struct erofs_inode *mergedir; char uuid_str[37]; char *fsid = sbi->devname; int ret; @@ -447,16 +504,19 @@ int erofs_rebuild_load_tree(struct erofs_inode *root, struct erofs_sb_info *sbi, erofs_err("failed to read root inode of %s", fsid); return ret; } + + mergedir = root; inode.i_srcpath = strdup("/"); ctx = (struct erofs_rebuild_dir_context) { .ctx.dir = &inode, .ctx.cb = erofs_rebuild_dirent_iter, - .mergedir = root, + .mergedir = mergedir, .datamode = mode, .uniaddr_offset = uniaddr_offset, }; ret = erofs_iterate_dir(&ctx.ctx, false); + free(inode.i_srcpath); return ret; } @@ -556,3 +616,49 @@ int erofs_rebuild_load_basedir(struct erofs_inode *dir, u64 *nr_subdirs, }; return erofs_iterate_dir(&ctx.ctx, false); } + +int erofs_rebuild_copy_src(struct erofs_sb_info *sbi, + struct erofs_sb_info *src, + struct erofs_device_info *dev) +{ + erofs_blk_t cur = sbi->primarydevice_blocks; + u64 src_off = 0, dst_off, len; + int src_fd, dst_fd; + int ret; + + ret = erofs_read_superblock(src); + if (ret) { + erofs_err("failed to read superblock of %s: %s", + src->devname, erofs_strerror(ret)); + return ret; + } + + dev->blocks = src->primarydevice_blocks; + dev->uniaddr = cur; + + erofs_info("Copying %s: %u blocks at unified address %u", + src->devname, dev->blocks, cur); + + src_fd = src->bdev.fd; + dst_fd = sbi->bdev.fd; + if (src_fd < 0 || dst_fd < 0) { + erofs_err("failed to get file descriptors"); + return -EINVAL; + } + dst_off = erofs_pos(sbi, cur); + len = erofs_pos(src, dev->blocks); + while (len > 0) { + ssize_t copied = erofs_copy_file_range(src_fd, &src_off, + dst_fd, &dst_off, len); + if (copied < 0) { + erofs_err("failed to copy data from %s: %s", + src->devname, erofs_strerror(-copied)); + return copied; + } + if (copied == 0) + break; + len -= copied; + } + sbi->primarydevice_blocks += dev->blocks; + return 0; +} diff --git a/mkfs/main.c b/mkfs/main.c index 48da20f..4ac835f 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -15,9 +15,11 @@ #include <getopt.h> #include "erofs/config.h" #include "erofs/print.h" +#include "erofs/io.h" #include "erofs/importer.h" #include "erofs/diskbuf.h" #include "erofs/inode.h" +#include "erofs/dir.h" #include "erofs/tar.h" #include "erofs/dedupe.h" #include "erofs/xattr.h" @@ -30,6 +32,7 @@ #include "../lib/liberofs_metabox.h" #include "../lib/liberofs_oci.h" #include "../lib/liberofs_private.h" +#include "../lib/liberofs_cache.h" #include "../lib/liberofs_rebuild.h" #include "../lib/liberofs_s3.h" #include "../lib/liberofs_uuid.h" @@ -1717,7 +1720,7 @@ static int erofs_mkfs_rebuild_load_trees(struct erofs_inode *root) struct erofs_sb_info *src; unsigned int extra_devices = 0; erofs_blk_t nblocks; - int ret, idx; + int ret, idx = 0; enum erofs_rebuild_datamode datamode; switch (dataimport_mode) { @@ -1734,9 +1737,33 @@ static int erofs_mkfs_rebuild_load_trees(struct erofs_inode *root) return -EINVAL; } + if (datamode != EROFS_REBUILD_DATA_RESVSP) { + ret = erofs_mkfs_init_devices(&g_sbi, rebuild_src_count); + if (ret) { + erofs_err("failed to initialize devices: %s", + erofs_strerror(ret)); + return ret; + } + devs = g_sbi.devs; + } + list_for_each_entry(src, &rebuild_src_list, list) { + erofs_blk_t uniaddr = 0; + + if (datamode == EROFS_REBUILD_DATA_FULL) { + /* Copy source data blocks */ + ret = erofs_rebuild_copy_src(&g_sbi, src, &devs[idx]); + if (ret) + return ret; + + uniaddr = devs[idx].uniaddr; + + /* Advance buffer manager past copied data */ + erofs_bset_tail(g_sbi.bmgr, g_sbi.primarydevice_blocks); + } + src->xamgr = g_sbi.xamgr; - ret = erofs_rebuild_load_tree(root, src, datamode, 0); + ret = erofs_rebuild_load_tree(root, src, datamode, uniaddr); src->xamgr = NULL; if (ret) { erofs_err("failed to load %s", src->devname); @@ -1748,9 +1775,10 @@ static int erofs_mkfs_rebuild_load_trees(struct erofs_inode *root) return -EOPNOTSUPP; } extra_devices += src->extra_devices; + idx++; } - if (datamode != EROFS_REBUILD_DATA_BLOB_INDEX) + if (datamode == EROFS_REBUILD_DATA_RESVSP) return 0; /* Each blob has either no extra device or only one device for TarFS */ @@ -1760,11 +1788,6 @@ static int erofs_mkfs_rebuild_load_trees(struct erofs_inode *root) return -EOPNOTSUPP; } - ret = erofs_mkfs_init_devices(&g_sbi, rebuild_src_count); - if (ret) - return ret; - - devs = g_sbi.devs; list_for_each_entry(src, &rebuild_src_list, list) { u8 *tag = NULL; @@ -1775,14 +1798,15 @@ static int erofs_mkfs_rebuild_load_trees(struct erofs_inode *root) tag = src->devs[0].tag; } else { nblocks = src->primarydevice_blocks; - devs[idx].src_path = strdup(src->devname); + if (datamode == EROFS_REBUILD_DATA_BLOB_INDEX) + devs[idx].src_path = strdup(src->devname); } devs[idx].blocks = nblocks; if (tag && *tag) memcpy(devs[idx].tag, tag, sizeof(devs[0].tag)); else /* convert UUID of the source image to a hex string */ - erofs_uuid_unparse_as_tag(src->uuid, (char *)g_sbi.devs[idx].tag); + erofs_uuid_unparse_as_tag(src->uuid, (char *)devs[idx].tag); } return 0; } -- Git-155)
