Hoi Richard,
and thanks for the feedback!
your other mail is still being processed, but to get already back to you on
this one:
>> set the package-database of a "lower image" to unpack and build upon when
>> installing packages for the current image. This way a lean image will be
>> created, which only holds the packages that are not already present in the
>> lower
>> image, that then could be used with overlayfs or systemd-sysext to extend the
>> "lower image" on demand; for development purposes on an RO lower image for
>> example.
>>
>> Signed-off-by: Johannes Schneider <johannes.schnei...@leica-geosystems.com>
>> ---
>> meta/classes-recipe/image.bbclass | 10 +++++++-
>> meta/lib/oe/package_manager/deb/rootfs.py | 2 ++
>> meta/lib/oe/package_manager/ipk/rootfs.py | 6 +++--
>> meta/lib/oe/package_manager/rpm/rootfs.py | 7 ++++--
>> meta/lib/oe/rootfs.py | 29 +++++++++++++++++++++++
>> 5 files changed, 49 insertions(+), 5 deletions(-)
>>
>> diff --git a/meta/classes-recipe/image.bbclass
>> b/meta/classes-recipe/image.bbclass
>> index c688c39f15..b4a2460187 100644
>> --- a/meta/classes-recipe/image.bbclass
>> +++ b/meta/classes-recipe/image.bbclass
>> @@ -42,8 +42,16 @@ IMAGE_FEATURES ?= ""
>> IMAGE_FEATURES[type] = "list"
>> IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs
>> read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password
>> allow-empty-password allow-root-login serial-autologin-root
>> post-install-logging overlayfs-etc"
>>
>> +# Image layering:
>> +# a "base image" would create a snapshot of the package-database after the
>> +# installation of all packages into the rootfs is done. The next/other image
>> +# "layered on-top" of the former would then import that database and install
>> +# further packages; without reinstalling package dependencies that are
>> already
>> +# installed in the layer below.
>> # Generate snapshot of the package database?
>> IMAGE_GEN_PKGDBFS ?= "0"
>> +# Package-database of the base image, upon which to build up a new
>> image-layer
>> +IMAGE_BASE_PKGDB ?= ""
>>
>> # Generate companion debugfs?
>> IMAGE_GEN_DEBUGFS ?= "0"
>> @@ -134,7 +142,7 @@ def rootfs_variables(d):
>>
>> 'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS',
>> 'IMAGE_LINGUAS_COMPLEMENTARY', 'IMAGE_LOCALES_ARCHIVE',
>>
>> 'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS',
>>
>> 'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS',
>> - 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS',
>> 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR',
>> 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS',
>> + 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS',
>> 'IMAGE_BASE_PKGDB', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED',
>> 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY',
>> 'REPRODUCIBLE_TIMESTAMP_ROOTFS',
>> 'IMAGE_INSTALL_DEBUGFS']
>> variables.extend(rootfs_command_variables(d))
>> variables.extend(variable_depends(d))
>> diff --git a/meta/lib/oe/package_manager/deb/rootfs.py
>> b/meta/lib/oe/package_manager/deb/rootfs.py
>> index 43107c8663..71a21df09b 100644
>> --- a/meta/lib/oe/package_manager/deb/rootfs.py
>> +++ b/meta/lib/oe/package_manager/deb/rootfs.py
>> @@ -152,6 +152,8 @@ class PkgRootfs(DpkgOpkgRootfs):
>>
>> execute_pre_post_process(self.d, deb_pre_process_cmds)
>>
>> + self._unpack_pkg_db_rootfs(['/var/lib/dpkg'])
>> +
>> if self.progress_reporter:
>> self.progress_reporter.next_stage()
>> # Don't support incremental, so skip that
>> diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py
>> b/meta/lib/oe/package_manager/ipk/rootfs.py
>> index 64d9bc7969..408faa8030 100644
>> --- a/meta/lib/oe/package_manager/ipk/rootfs.py
>> +++ b/meta/lib/oe/package_manager/ipk/rootfs.py
>> @@ -276,12 +276,16 @@ class PkgRootfs(DpkgOpkgRootfs):
>> pkgs_to_install = self.manifest.parse_initial_manifest()
>> opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS')
>> opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS')
>> + opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
>> + opkg_dir = os.path.join(opkg_lib_dir, 'opkg')
>>
>> # update PM index files
>> self.pm.write_index()
>>
>> execute_pre_post_process(self.d, opkg_pre_process_cmds)
>>
>> + self._unpack_pkg_db_rootfs([opkg_dir])
>> +
>> if self.progress_reporter:
>> self.progress_reporter.next_stage()
>> # Steps are a bit different in order, skip next
>> @@ -317,8 +321,6 @@ class PkgRootfs(DpkgOpkgRootfs):
>> if self.progress_reporter:
>> self.progress_reporter.next_stage()
>>
>> - opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
>> - opkg_dir = os.path.join(opkg_lib_dir, 'opkg')
>> self._setup_pkg_db_rootfs([opkg_dir])
>> self._setup_dbg_rootfs([opkg_dir])
>>
>> diff --git a/meta/lib/oe/package_manager/rpm/rootfs.py
>> b/meta/lib/oe/package_manager/rpm/rootfs.py
>> index 673006c131..9986b13b5f 100644
>> --- a/meta/lib/oe/package_manager/rpm/rootfs.py
>> +++ b/meta/lib/oe/package_manager/rpm/rootfs.py
>> @@ -67,12 +67,15 @@ class PkgRootfs(Rootfs):
>> pkgs_to_install = self.manifest.parse_initial_manifest()
>> rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS')
>> rpm_post_process_cmds = self.d.getVar('RPM_POSTPROCESS_COMMANDS')
>> + package_paths = ['/etc/rpm', '/etc/rpmrc', '/etc/dnf',
>> '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf']
>>
>> # update PM index files
>> self.pm.write_index()
>>
>> execute_pre_post_process(self.d, rpm_pre_process_cmds)
>>
>> + self._unpack_pkg_db_rootfs(package_paths)
>> +
>> if self.progress_reporter:
>> self.progress_reporter.next_stage()
>>
>> @@ -110,8 +113,8 @@ class PkgRootfs(Rootfs):
>> if self.progress_reporter:
>> self.progress_reporter.next_stage()
>>
>> - self._setup_pkg_db_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf',
>> '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
>> - self._setup_dbg_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf',
>> '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
>> + self._setup_pkg_db_rootfs(package_paths)
>> + self._setup_dbg_rootfs(package_paths)
>>
>> execute_pre_post_process(self.d, rpm_post_process_cmds)
>>
>> diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
>> index 9268a02531..7ca574e6b1 100644
>> --- a/meta/lib/oe/rootfs.py
>> +++ b/meta/lib/oe/rootfs.py
>> @@ -106,6 +106,35 @@ class Rootfs(object, metaclass=ABCMeta):
>> def _cleanup(self):
>> pass
>>
>> + def _unpack_pkg_db_rootfs(self, package_paths):
>> + import tarfile
>> + gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB') or ''
>> + if gen_pkg_db_fs == '':
>> + return
>
>Can be simplified to:
>
>gen_pkg_db_fs = self.d.getVar('IMAGE_BASE_PKGDB')
>if not gen_pkg_db_fs:
> return
>
thanks! will be in V3
>
>> +
>> + fname = self.d.getVar('DEPLOY_DIR_IMAGE') + '/' +
>> self.d.getVar('IMAGE_BASE_PKGDB') + '-pkgdb.tar.gz'
>> + if not fname:
>> + bb.warn("PKGDB does not exit:", fname)
>> + return
>
>Shouldn't that be a hard error?
good point, i also need to figure out a way to get a task-dependency there, so
that the lower-layer-base-image:setup_pkg_db is triggered when only
upper-layer-image is built separatly - thoughts/suggestions?
>
>> +
>> + bb.note(" unpacking package database...")
>> + bb.utils.mkdirhier(self.image_rootfs + '-pkgdb')
>> + if fname.endswith("tar.gz"):
>> + tar = tarfile.open(fname, "r:gz")
>> + tar.extractall(path=self.image_rootfs + '-pkgdb')
>> + tar.close()
>
>Didn't you hardcode it to tar.gz above? FWIW the python tarfile module
>is also very slow.
>
>
>What happens if I set IMAGE_FSTYPES_PKGDBFS to something other than
>tar.gz?
>
that's true, would i get away with dropping the whole "configurable FSTYPE" and
just always use tar.gz instead?
since the pack+unpack artifacts are only used internally while building the
images,
therefor there is probably not much benefit in keeping it user-configurable
python tarfile: what alternatives are there?
is it ok to call the systems native tar?
looking around i see some uses of bb.compression in combination with tarfile,
but also others (spdx license packaging) that use the python tarfile
>> +
>> + bb.note(" Copying back package database...")
>> + for path in package_paths:
>> + try:
>> + bb.utils.mkdirhier(self.image_rootfs +
>> os.path.dirname(path))
>> + except:
>> + pass
>> + if os.path.isdir(self.image_rootfs + '-pkgdb' + path):
>> + shutil.copytree(self.image_rootfs + '-pkgdb' + path,
>> self.image_rootfs + path, symlinks=True, dirs_exist_ok=True)
>> + elif os.path.isfile(self.image_rootfs + '-pkgdb' + path):
>> + shutil.copyfile(self.image_rootfs + '-pkgdb' + path,
>> self.image_rootfs + path)
>> +
>> def _setup_pkg_db_rootfs(self, package_paths):
>> gen_pkg_db_fs = self.d.getVar('IMAGE_GEN_PKGDBFS') or '0'
>> if gen_pkg_db_fs != '1':
>
>
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#195903):
https://lists.openembedded.org/g/openembedded-core/message/195903
Mute This Topic: https://lists.openembedded.org/mt/104329519/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
