archive the package database after the rootfs has been put together as
*rootfs-pkdbfs.tar.gz, and put it into the deploy folder.
This creates a snapshot of the package mangers state at the point in
time when all dependencies have been resolved and installed; which
could be used by "extension images" to built upon.
Signed-off-by: Johannes Schneider <johannes.schnei...@leica-geosystems.com>
---
meta/classes-recipe/image.bbclass | 44 ++++++++++++++++-
meta/classes-recipe/image_types.bbclass | 1 +
meta/conf/documentation.conf | 1 +
meta/lib/oe/package_manager/deb/rootfs.py | 1 +
meta/lib/oe/package_manager/ipk/rootfs.py | 1 +
meta/lib/oe/package_manager/rpm/rootfs.py | 1 +
meta/lib/oe/rootfs.py | 20 ++++++++
meta/lib/oeqa/selftest/cases/imagefeatures.py | 47 +++++++++++++++++++
8 files changed, 115 insertions(+), 1 deletion(-)
diff --git a/meta/classes-recipe/image.bbclass
b/meta/classes-recipe/image.bbclass
index 28be6c6362..3ccaaa17b8 100644
--- a/meta/classes-recipe/image.bbclass
+++ b/meta/classes-recipe/image.bbclass
@@ -42,6 +42,9 @@ IMAGE_FEATURES ?= ""
IMAGE_FEATURES[type] = "list"
IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs
read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password
allow-empty-password allow-root-login serial-autologin-root
post-install-logging overlayfs-etc"
+# Generate snapshot of the package database?
+IMAGE_GEN_PKGDBFS ?= "0"
+
# Generate companion debugfs?
IMAGE_GEN_DEBUGFS ?= "0"
@@ -131,7 +134,8 @@ def rootfs_variables(d):
'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS',
'IMAGE_LINGUAS_COMPLEMENTARY', 'IMAGE_LOCALES_ARCHIVE',
'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS',
'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS',
- 'CONVERSIONTYPES', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED',
'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY',
'REPRODUCIBLE_TIMESTAMP_ROOTFS', 'IMAGE_INSTALL_DEBUGFS']
+ 'CONVERSIONTYPES', 'IMAGE_GEN_PKGDBFS', 'IMAGE_GEN_DEBUGFS',
'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY',
'REPRODUCIBLE_TIMESTAMP_ROOTFS',
+ 'IMAGE_INSTALL_DEBUGFS']
variables.extend(rootfs_command_variables(d))
variables.extend(variable_depends(d))
return " ".join(variables)
@@ -337,6 +341,17 @@ python do_image_qa_setscene () {
}
addtask do_image_qa_setscene
+def setup_pkgdbfs_variables(d):
+ d.appendVar('IMAGE_ROOTFS', '-pkgdb')
+ if d.getVar('IMAGE_LINK_NAME'):
+ d.appendVar('IMAGE_LINK_NAME', '-pkgdb')
+ d.appendVar('IMAGE_NAME','-pkgdb')
+ d.setVar('IMAGE_FSTYPES', 'tar.gz')
+
+python setup_pkgdbfs () {
+ setup_pkgdbfs_variables(d)
+}
+
def setup_debugfs_variables(d):
d.appendVar('IMAGE_ROOTFS', '-dbg')
if d.getVar('IMAGE_LINK_NAME'):
@@ -381,6 +396,11 @@ python () {
alltypes = d.getVar('IMAGE_FSTYPES').split()
typedeps = {}
+ if d.getVar('IMAGE_GEN_PKGDBFS') == "1":
+ pkgdbfs_fstypes = ['tar.gz']
+ for t in pkgdbfs_fstypes:
+ alltypes.append("pkgdbfs_" + t)
+
if d.getVar('IMAGE_GEN_DEBUGFS') == "1":
debugfs_fstypes = d.getVar('IMAGE_FSTYPES_DEBUGFS').split()
for t in debugfs_fstypes:
@@ -393,6 +413,10 @@ python () {
basetypes[baset]= []
if t not in basetypes[baset]:
basetypes[baset].append(t)
+ pkgdb = ""
+ if t.startswith("pkgdbfs_"):
+ t = t[8:]
+ pkgdb = "pkgdbfs_"
debug = ""
if t.startswith("debugfs_"):
t = t[8:]
@@ -401,6 +425,13 @@ python () {
vardeps.add('IMAGE_TYPEDEP:' + t)
if baset not in typedeps:
typedeps[baset] = set()
+ deps = [pkgdb + dep for dep in deps]
+ for dep in deps:
+ if dep not in alltypes:
+ alltypes.append(dep)
+ _add_type(dep)
+ basedep = _image_base_type(dep)
+ typedeps[baset].add(basedep)
deps = [debug + dep for dep in deps]
for dep in deps:
if dep not in alltypes:
@@ -419,6 +450,7 @@ python () {
maskedtypes = (d.getVar('IMAGE_TYPES_MASKED') or "").split()
maskedtypes = [dbg + t for t in maskedtypes for dbg in ("", "debugfs_")]
+ maskedtypes = [pkgdb + t for t in maskedtypes for pkgdb in ("",
"pkgdbfs_")]
for t in basetypes:
vardeps = set()
@@ -430,6 +462,11 @@ python () {
continue
localdata = bb.data.createCopy(d)
+ pkgdb = ""
+ if t.startswith("pkgdbfs_"):
+ setup_pkgdbfs_variables(localdata)
+ pkgdb = "setup_pkgdbfs "
+ realt = t[8:]
debug = ""
if t.startswith("debugfs_"):
setup_debugfs_variables(localdata)
@@ -468,6 +505,8 @@ python () {
for ctype in sorted(ctypes):
if bt.endswith("." + ctype):
type = bt[0:-len(ctype) - 1]
+ if type.startswith("pkgdbfs_"):
+ type = type[8:]
if type.startswith("debugfs_"):
type = type[8:]
# Create input image first.
@@ -509,6 +548,9 @@ python () {
d.setVarFlag(task, 'fakeroot', '1')
d.appendVarFlag(task, 'prefuncs', ' ' + debug + ' set_image_size')
+ if pkgdb:
+ d.appendVarFlag(task, 'prefuncs', ' ' + pkgdb + ' set_image_size')
+
d.prependVarFlag(task, 'postfuncs', 'create_symlinks ')
d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages))
d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps))
diff --git a/meta/classes-recipe/image_types.bbclass
b/meta/classes-recipe/image_types.bbclass
index 3733bdfc20..39856a6368 100644
--- a/meta/classes-recipe/image_types.bbclass
+++ b/meta/classes-recipe/image_types.bbclass
@@ -25,6 +25,7 @@ def imagetypes_getdepends(d):
fstypes = set((d.getVar('IMAGE_FSTYPES') or "").split())
fstypes |= set((d.getVar('IMAGE_FSTYPES_DEBUGFS') or "").split())
+ fstypes |= set('tar.gz') # hardcoded for 'pkgdbfs'
deprecated = set()
deps = set()
diff --git a/meta/conf/documentation.conf b/meta/conf/documentation.conf
index 90d8e82932..7ec6f07dfc 100644
--- a/meta/conf/documentation.conf
+++ b/meta/conf/documentation.conf
@@ -215,6 +215,7 @@ IMAGE_FEATURES[doc] = "The primary list of features to
include in an image. Conf
IMAGE_FSTYPES[doc] = "Formats of root filesystem images that you want to have
created."
IMAGE_FSTYPES_DEBUGFS[doc] = "Formats of the debug root filesystem images that
you want to have created."
IMAGE_GEN_DEBUGFS[doc] = "When set to '1', generate a companion debug
object/source filesystem image."
+IMAGE_GEN_PKGDBFS[doc] = "When set to '1', create a snapshot of the
package-manager state after the rootfs has been assembled."
IMAGE_INSTALL[doc] = "Specifies the packages to install into an image. Image
recipes set IMAGE_INSTALL to specify the packages to install into an image
through image.bbclass."
IMAGE_LINGUAS[doc] = "Specifies the list of locales to install into the image
during the root filesystem construction process."
IMAGE_NAME[doc] = "The name of the output image files minus the extension."
diff --git a/meta/lib/oe/package_manager/deb/rootfs.py
b/meta/lib/oe/package_manager/deb/rootfs.py
index 1e25b64ed9..43107c8663 100644
--- a/meta/lib/oe/package_manager/deb/rootfs.py
+++ b/meta/lib/oe/package_manager/deb/rootfs.py
@@ -178,6 +178,7 @@ class PkgRootfs(DpkgOpkgRootfs):
if self.progress_reporter:
self.progress_reporter.next_stage()
+ self._setup_pkg_db_rootfs(['/var/lib/dpkg'])
self._setup_dbg_rootfs(['/var/lib/dpkg'])
self.pm.fix_broken_dependencies()
diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py
b/meta/lib/oe/package_manager/ipk/rootfs.py
index ba93eb62ea..64d9bc7969 100644
--- a/meta/lib/oe/package_manager/ipk/rootfs.py
+++ b/meta/lib/oe/package_manager/ipk/rootfs.py
@@ -319,6 +319,7 @@ class PkgRootfs(DpkgOpkgRootfs):
opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
opkg_dir = os.path.join(opkg_lib_dir, 'opkg')
+ self._setup_pkg_db_rootfs([opkg_dir])
self._setup_dbg_rootfs([opkg_dir])
execute_pre_post_process(self.d, opkg_post_process_cmds)
diff --git a/meta/lib/oe/package_manager/rpm/rootfs.py
b/meta/lib/oe/package_manager/rpm/rootfs.py
index 3ba5396320..673006c131 100644
--- a/meta/lib/oe/package_manager/rpm/rootfs.py
+++ b/meta/lib/oe/package_manager/rpm/rootfs.py
@@ -110,6 +110,7 @@ class PkgRootfs(Rootfs):
if self.progress_reporter:
self.progress_reporter.next_stage()
+ self._setup_pkg_db_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf',
'/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
self._setup_dbg_rootfs(['/etc/rpm', '/etc/rpmrc', '/etc/dnf',
'/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
execute_pre_post_process(self.d, rpm_post_process_cmds)
diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index 8cd48f9450..6d6e888a30 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -106,6 +106,26 @@ class Rootfs(object, metaclass=ABCMeta):
def _cleanup(self):
pass
+ def _setup_pkg_db_rootfs(self, package_paths):
+ gen_pkg_db_fs = bb.utils.to_boolean(self.d.getVar('IMAGE_GEN_PKGDBFS'))
+ if not gen_pkg_db_fs:
+ return
+
+ bb.note(" Creating pkg-db rootfs...")
+ try:
+ shutil.rmtree(self.image_rootfs + '-pkgdb')
+ except (FileNotFoundError, NotADirectoryError):
+ pass
+ bb.utils.mkdirhier(self.image_rootfs + "-pkgdb")
+
+ bb.note(" Copying back package database...")
+ for path in package_paths:
+ bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path))
+ if os.path.isdir(self.image_rootfs + path):
+ shutil.copytree(self.image_rootfs + path, self.image_rootfs +
'-pkgdb' + path, symlinks=True)
+ elif os.path.isfile(self.image_rootfs + path):
+ shutil.copyfile(self.image_rootfs + path, self.image_rootfs +
'-pkgdb' + path)
+
def _setup_dbg_rootfs(self, package_paths):
gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0'
if gen_debugfs != '1':
diff --git a/meta/lib/oeqa/selftest/cases/imagefeatures.py
b/meta/lib/oeqa/selftest/cases/imagefeatures.py
index dc88c222bd..4d4005e00c 100644
--- a/meta/lib/oeqa/selftest/cases/imagefeatures.py
+++ b/meta/lib/oeqa/selftest/cases/imagefeatures.py
@@ -302,6 +302,53 @@ SKIP_RECIPE[busybox] = "Don't build this"
result = runCmd('objdump --syms %s | grep debug' % t)
self.assertTrue("debug" in result.output, msg='Failed to find
debug symbol: %s' % result.output)
+ def test_image_gen_pkgdbfs(self):
+ """
+ Summary: Check pkgdb snapshot
+ Expected: 1. core-image-minimal can be build with IMAGE_GEN_PKGDBFS
variable set
+ 2. *rootfs-pkgdb.tar.gz snapshot of the package manager
state is created,
+ after the rootfs of the core-image-minimal is put
together
+ 3. check that the restored package index is usable by the
default package-manager
+ PACKAGE_CLASS=package_ipk -> opkg in this case.
+ """
+
+ image = 'core-image-minimal'
+ features = 'IMAGE_GEN_PKGDBFS = "1"\n'
+ self.write_config(features)
+
+ bitbake(image)
+
+ img_vars = get_bb_vars(['DEPLOY_DIR_IMAGE', 'IMAGE_LINK_NAME',
'MACHINE', 'DEFAULTTUNE'], image)
+
+ tar_file = os.path.join(img_vars['DEPLOY_DIR_IMAGE'],
"%s-pkgdb.tar.gz" % (img_vars['IMAGE_LINK_NAME']))
+ self.assertTrue(os.path.exists(tar_file), 'pkgdb filesystem not
generated at %s' % tar_file)
+ result = runCmd('cd %s; tar xvf %s' % (img_vars['DEPLOY_DIR_IMAGE'],
tar_file))
+ self.assertEqual(result.status, 0, msg='Failed to extract %s: %s' %
(tar_file, result.output))
+
self.assertTrue(os.path.exists(os.path.join(img_vars['DEPLOY_DIR_IMAGE'],
'var/lib/opkg/status')), 'opkg\'s status file was not present in: %s' %
tar_file)
+
+ bitbake("opkg-native")
+ bitbake("build-sysroots -c build_native_sysroot")
+
+ native_vars = get_bb_vars(["STAGING_DIR", "BUILD_ARCH",
"bindir_native", "libdir_native"])
+ stagingbindir = os.path.join(native_vars["STAGING_DIR"],
native_vars["BUILD_ARCH"], native_vars["bindir_native"].lstrip(os.sep))
+
+ # some 'add-arch' have '-' replaced by '_', some not; adding both does
not hurt
+ cmd = os.path.join(stagingbindir, 'opkg') + ' --volatile-cache
--offline-root=%s --add-arch %s:11 --add-arch %s:12 list-installed' %
(img_vars['DEPLOY_DIR_IMAGE'], img_vars["MACHINE"],
img_vars["MACHINE"].replace('-','_'))
+ self.logger.debug('running cmd: %s' % cmd)
+ result = runCmd(cmd)
+ self.logger.debug("list-installed:\n%s" % result.output)
+ self.assertEqual(result.status, 0, msg='Failed to run package manager
on unpacked pkgdb %s: %s' % (tar_file, result.output))
+ # check for a 'random, very architecture specific' package
+ self.assertTrue("kernel-image" in result.output, msg='Failed query
installed packages')
+
+ cmd = os.path.join(stagingbindir, 'opkg') + ' --volatile-cache
--offline-root=%s --add-arch %s:11 --add-arch %s:12 list-installed' %
(img_vars['DEPLOY_DIR_IMAGE'], img_vars["DEFAULTTUNE"],
img_vars["DEFAULTTUNE"].replace('-','_'))
+ self.logger.debug('running cmd: %s' % cmd)
+ result = runCmd(cmd)
+ self.logger.debug("list-installed:\n%s" % result.output)
+ self.assertEqual(result.status, 0, msg='Failed to run package manager
on unpacked pkgdb %s: %s' % (tar_file, result.output))
+ # check for a common base package, a random library for example:
+ self.assertTrue("libc6" in result.output, msg='Failed query installed
packages')
+
def test_empty_image(self):
"""Test creation of image with no packages"""
image = 'test-empty-image'
--
2.34.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#196000):
https://lists.openembedded.org/g/openembedded-core/message/196000
Mute This Topic: https://lists.openembedded.org/mt/104500430/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
