Re: [OE-core] [PATCH] oeqa/selftest/overlayfs: test read-only rootfs
Just checking if there's any feedback for the patch or something that
stops it from being merged.
Slava
On 17.03.2024 20:33, Vyacheslav Yurkov via lists.openembedded.org wrote:
From: Baruch Siach
Use the read-only squashfs filesystem to test the read-only case.
Signed-off-by: Baruch Siach
Signed-off-by: Vyacheslav Yurkov
---
meta-selftest/wic/overlayfs_etc.wks.in| 4 +--
meta/lib/oeqa/selftest/cases/overlayfs.py | 34 +++
2 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/meta-selftest/wic/overlayfs_etc.wks.in
b/meta-selftest/wic/overlayfs_etc.wks.in
index 1e1e5836e7..066cd35b15 100644
--- a/meta-selftest/wic/overlayfs_etc.wks.in
+++ b/meta-selftest/wic/overlayfs_etc.wks.in
@@ -1,4 +1,4 @@
part /boot --active --source bootimg-biosplusefi --ondisk sda
--sourceparams="loader=grub-efi" --align 1024
-part / --source rootfs --ondisk sda --fstype=ext4 --use-uuid --align 1024
+part / --source rootfs --ondisk sda --fstype=${OVERLAYFS_ROOTFS_TYPE}
--use-uuid --align 1024
part --ondisk sda --fstype=ext4 --size=5 --align 1024
-bootloader --ptable gpt --timeout=1 --append="rootfstype=ext4 console=ttyS0,115200
console=tty0 ${OVERLAYFS_INIT_OPTION}"
+bootloader --ptable gpt --timeout=1 --append="rootfstype=${OVERLAYFS_ROOTFS_TYPE}
console=ttyS0,115200 console=tty0 ${OVERLAYFS_INIT_OPTION}"
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index cd0dc60c64..e31063567b 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -353,6 +353,7 @@ EXTRA_IMAGE_FEATURES += "read-only-rootfs"
# Image configuration for overlayfs-etc
OVERLAYFS_ETC_MOUNT_POINT = "/data"
OVERLAYFS_ETC_DEVICE = "/dev/sda3"
+OVERLAYFS_ROOTFS_TYPE = "ext4"
"""
self.write_config(config)
@@ -367,13 +368,17 @@ OVERLAYFS_ETC_DEVICE = "/dev/sda3"
@skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently")
def test_sbin_init_preinit(self):
-self.run_sbin_init(False)
+self.run_sbin_init(False, "ext4")
@skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently")
def test_sbin_init_original(self):
-self.run_sbin_init(True)
+self.run_sbin_init(True, "ext4")
-def run_sbin_init(self, origInit):
+@skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently")
+def test_sbin_init_read_only(self):
+self.run_sbin_init(True, "squashfs")
+
+def run_sbin_init(self, origInit, rootfsType):
"""
Summary: Confirm we can replace original init and mount overlay on
top of /etc
Expected: Image is created successfully and /etc is mounted as an
overlay
@@ -384,7 +389,9 @@ OVERLAYFS_ETC_DEVICE = "/dev/sda3"
args = {
'OVERLAYFS_INIT_OPTION': "" if origInit else "init=/sbin/preinit",
-'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': int(origInit == True)
+'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': int(origInit == True),
+'OVERLAYFS_ROOTFS_TYPE': rootfsType,
+'OVERLAYFS_ETC_CREATE_MOUNT_DIRS': int(rootfsType == "ext4")
}
self.write_config(config.format(**args))
@@ -437,7 +444,9 @@ IMAGE_INSTALL:append = " overlayfs-user"
args = {
'OVERLAYFS_INIT_OPTION': "",
-'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': 1
+'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': 1,
+'OVERLAYFS_ROOTFS_TYPE': "ext4",
+'OVERLAYFS_ETC_CREATE_MOUNT_DIRS': 1
}
self.write_config(config.format(**args))
@@ -463,10 +472,14 @@ IMAGE_INSTALL:append = " overlayfs-user"
INIT_MANAGER = "systemd"
# enable overlayfs in the kernel
-KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
+KERNEL_EXTRA_FEATURES:append = " \
+features/overlayfs/overlayfs.scc \
+cfg/fs/squashfs.scc"
IMAGE_FSTYPES += "wic"
OVERLAYFS_INIT_OPTION = "{OVERLAYFS_INIT_OPTION}"
+OVERLAYFS_ROOTFS_TYPE = "{OVERLAYFS_ROOTFS_TYPE}"
+OVERLAYFS_ETC_CREATE_MOUNT_DIRS = "{OVERLAYFS_ETC_CREATE_MOUNT_DIRS}"
WKS_FILE = "overlayfs_etc.wks.in"
EXTRA_IMAGE_FEATURES += "read-only-rootfs"
@@ -477,4 +490,13 @@ OVERLAYFS_ETC_MOUNT_POINT = "/data"
OVERLAYFS_ETC_FSTYPE = "ext4"
OVERLAYFS_ETC_DEVICE = "/dev/sda3"
OVERLAYFS_ETC_USE_ORIG_INIT_NAME = "{OVERLAYFS_ETC_USE_ORIG_INIT_NAME}"
+
+ROOTFS_POSTPROCESS_COMMAND += "{OVERLAYFS_ROOTFS_TYPE}_rootfs"
+
+ext4_rootfs() {{
+}}
+
+squashfs_rootfs() {{
+mkdir -p ${
[OE-core] [PATCH] oeqa/selftest/overlayfs: test read-only rootfs
From: Baruch Siach
Use the read-only squashfs filesystem to test the read-only case.
Signed-off-by: Baruch Siach
Signed-off-by: Vyacheslav Yurkov
---
meta-selftest/wic/overlayfs_etc.wks.in| 4 +--
meta/lib/oeqa/selftest/cases/overlayfs.py | 34 +++
2 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/meta-selftest/wic/overlayfs_etc.wks.in
b/meta-selftest/wic/overlayfs_etc.wks.in
index 1e1e5836e7..066cd35b15 100644
--- a/meta-selftest/wic/overlayfs_etc.wks.in
+++ b/meta-selftest/wic/overlayfs_etc.wks.in
@@ -1,4 +1,4 @@
part /boot --active --source bootimg-biosplusefi --ondisk sda
--sourceparams="loader=grub-efi" --align 1024
-part / --source rootfs --ondisk sda --fstype=ext4 --use-uuid --align 1024
+part / --source rootfs --ondisk sda --fstype=${OVERLAYFS_ROOTFS_TYPE}
--use-uuid --align 1024
part --ondisk sda --fstype=ext4 --size=5 --align 1024
-bootloader --ptable gpt --timeout=1 --append="rootfstype=ext4
console=ttyS0,115200 console=tty0 ${OVERLAYFS_INIT_OPTION}"
+bootloader --ptable gpt --timeout=1
--append="rootfstype=${OVERLAYFS_ROOTFS_TYPE} console=ttyS0,115200 console=tty0
${OVERLAYFS_INIT_OPTION}"
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index cd0dc60c64..e31063567b 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -353,6 +353,7 @@ EXTRA_IMAGE_FEATURES += "read-only-rootfs"
# Image configuration for overlayfs-etc
OVERLAYFS_ETC_MOUNT_POINT = "/data"
OVERLAYFS_ETC_DEVICE = "/dev/sda3"
+OVERLAYFS_ROOTFS_TYPE = "ext4"
"""
self.write_config(config)
@@ -367,13 +368,17 @@ OVERLAYFS_ETC_DEVICE = "/dev/sda3"
@skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently")
def test_sbin_init_preinit(self):
-self.run_sbin_init(False)
+self.run_sbin_init(False, "ext4")
@skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently")
def test_sbin_init_original(self):
-self.run_sbin_init(True)
+self.run_sbin_init(True, "ext4")
-def run_sbin_init(self, origInit):
+@skipIfNotMachine("qemux86-64", "tests are qemux86-64 specific currently")
+def test_sbin_init_read_only(self):
+self.run_sbin_init(True, "squashfs")
+
+def run_sbin_init(self, origInit, rootfsType):
"""
Summary: Confirm we can replace original init and mount overlay on
top of /etc
Expected: Image is created successfully and /etc is mounted as an
overlay
@@ -384,7 +389,9 @@ OVERLAYFS_ETC_DEVICE = "/dev/sda3"
args = {
'OVERLAYFS_INIT_OPTION': "" if origInit else "init=/sbin/preinit",
-'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': int(origInit == True)
+'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': int(origInit == True),
+'OVERLAYFS_ROOTFS_TYPE': rootfsType,
+'OVERLAYFS_ETC_CREATE_MOUNT_DIRS': int(rootfsType == "ext4")
}
self.write_config(config.format(**args))
@@ -437,7 +444,9 @@ IMAGE_INSTALL:append = " overlayfs-user"
args = {
'OVERLAYFS_INIT_OPTION': "",
-'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': 1
+'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': 1,
+'OVERLAYFS_ROOTFS_TYPE': "ext4",
+'OVERLAYFS_ETC_CREATE_MOUNT_DIRS': 1
}
self.write_config(config.format(**args))
@@ -463,10 +472,14 @@ IMAGE_INSTALL:append = " overlayfs-user"
INIT_MANAGER = "systemd"
# enable overlayfs in the kernel
-KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
+KERNEL_EXTRA_FEATURES:append = " \
+features/overlayfs/overlayfs.scc \
+cfg/fs/squashfs.scc"
IMAGE_FSTYPES += "wic"
OVERLAYFS_INIT_OPTION = "{OVERLAYFS_INIT_OPTION}"
+OVERLAYFS_ROOTFS_TYPE = "{OVERLAYFS_ROOTFS_TYPE}"
+OVERLAYFS_ETC_CREATE_MOUNT_DIRS = "{OVERLAYFS_ETC_CREATE_MOUNT_DIRS}"
WKS_FILE = "overlayfs_etc.wks.in"
EXTRA_IMAGE_FEATURES += "read-only-rootfs"
@@ -477,4 +490,13 @@ OVERLAYFS_ETC_MOUNT_POINT = "/data"
OVERLAYFS_ETC_FSTYPE = "ext4"
OVERLAYFS_ETC_DEVICE = "/dev/sda3"
OVERLAYFS_ETC_USE_ORIG_INIT_NAME = "{OVERLAYFS_ETC_USE_ORIG_INIT_NAME}"
+
+ROOTFS_POSTPROCESS_COMMAND += "{OVERLAYFS_ROOTFS_TYPE}_rootfs"
+
+ext4_rootfs() {{
+}}
+
+squashfs_rootfs() {{
+mkdir -p ${{IMAGE_ROOTFS}}/data
+}}
"""
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#197270):
https://lists.openembedded.org/g/openembedded-core/message/197270
Mute This Topic: https://lists.openembedded.org/mt/104989652/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH v2] overlayfs-etc: add option to skip creation of mount dirs
Hi Baruch, That's a good question. Bruce, if we need a new kernel feature (squashfs), the only way would be first to contribute it to https://git.yoctoproject.org/yocto-kernel-cache/ ? Or there's another way to enable a specific kernel configuration in a selftest? Slava On 14.02.2024 07:54, Baruch Siach wrote: Makes sense. I added mkdirhier of OVERLAYFS_ETC_MOUNT_POINT for the case of OVERLAYFS_ETC_CREATE_MOUNT_DIRS == 0. Let me know if there is a better solution for read-only filesystems. The trouble with testing is that I could not find any existing feature in yocto-kernel-cache that enables either CONFIG_SQUASHFS or CONFIG_EROFS_FS. Is there another way to enable kernel features for the test? Thanks, baruch -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#195690): https://lists.openembedded.org/g/openembedded-core/message/195690 Mute This Topic: https://lists.openembedded.org/mt/104317307/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] Patchtest results for [PATCH v2 8/9] oeqa/selftest/recipetool: Move helper function to the class scope
I don't think so. git pw series list --submitter "Vyacheslav Yurkov" Shows the latest series 21353 from me. Regards, Slava On 22.01.2024 16:35, Lukas Funke wrote: Is this the reason why I cannot find the series on patchwork? This makes it kinda tricky to review/test IMHO. Please provide a commit message, so patchtest is happy. On 16.01.2024 09:45, Patchtest via lists.openembedded.org wrote: Thank you for your submission. Patchtest identified one or more issues with the patch. Please see the log below for more information: --- Testing patch /home/patchtest/share/mboxes/v2-8-9-oeqa-selftest-recipetool-Move-helper-function-to-the-class-scope.patch FAIL: test commit message presence: Please include a commit message on your patch explaining the change (test_mbox.TestMbox.test_commit_message_presence) PASS: pretest pylint (test_python_pylint.PyLint.pretest_pylint) PASS: test Signed-off-by presence (test_mbox.TestMbox.test_signed_off_by_presence) PASS: test author valid (test_mbox.TestMbox.test_author_valid) PASS: test max line length (test_metadata.TestMetadata.test_max_line_length) PASS: test mbox format (test_mbox.TestMbox.test_mbox_format) PASS: test non-AUH upgrade (test_mbox.TestMbox.test_non_auh_upgrade) PASS: test pylint (test_python_pylint.PyLint.test_pylint) PASS: test shortlog format (test_mbox.TestMbox.test_shortlog_format) PASS: test shortlog length (test_mbox.TestMbox.test_shortlog_length) PASS: test target mailing list (test_mbox.TestMbox.test_target_mailing_list) SKIP: pretest src uri left files: Patch cannot be merged (test_metadata.TestMetadata.pretest_src_uri_left_files) SKIP: test CVE check ignore: No modified recipes, skipping test (test_metadata.TestMetadata.test_cve_check_ignore) SKIP: test CVE tag format: No new CVE patches introduced (test_patch.TestPatch.test_cve_tag_format) SKIP: test Signed-off-by presence: No new CVE patches introduced (test_patch.TestPatch.test_signed_off_by_presence) SKIP: test Upstream-Status presence: No new CVE patches introduced (test_patch.TestPatch.test_upstream_status_presence_format) SKIP: test bugzilla entry format: No bug ID found (test_mbox.TestMbox.test_bugzilla_entry_format) SKIP: test lic files chksum modified not mentioned: No modified recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_modified_not_mentioned) SKIP: test lic files chksum presence: No added recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_presence) SKIP: test license presence: No added recipes, skipping test (test_metadata.TestMetadata.test_license_presence) SKIP: test series merge on head: Merge test is disabled for now (test_mbox.TestMbox.test_series_merge_on_head) SKIP: test src uri left files: Patch cannot be merged (test_metadata.TestMetadata.test_src_uri_left_files) SKIP: test summary presence: No added recipes, skipping test (test_metadata.TestMetadata.test_summary_presence) --- Please address the issues identified and submit a new revision of the patch, or alternatively, reply to this email with an explanation of why the patch should be accepted. If you believe these results are due to an error in patchtest, please submit a bug at https://bugzilla.yoctoproject.org/ (use the 'Patchtest' category under 'Yocto Project Subprojects'). For more information on specific failures, see: https://wiki.yoctoproject.org/wiki/Patchtest. Thank you! -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#194317): https://lists.openembedded.org/g/openembedded-core/message/194317 Mute This Topic: https://lists.openembedded.org/mt/103758513/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] Patchtest results for [PATCH v2 7/9] oeqa/selftest/recipetool: Move create_go test to a proper class
I also have the same opinion. What else am I supposed to write in the body if it were merely a refactoring (function moved, renamed, etc), and it's already described in the hearer? Slava On 16.01.2024 10:32, Alexander Kanavin wrote: Because making it strict would only result in useless noise in the commit logs. It's entirely valid to have changes that can fully be described with a commit header. I see it as a notice to consider if the commit message should be added, at the discretion of both submitter and committer. Alex On Tue, 16 Jan 2024 at 10:28, Alexandre Belloni wrote: On 16/01/2024 10:08:50+0100, Alexander Kanavin wrote: No. Why not? -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#193848): https://lists.openembedded.org/g/openembedded-core/message/193848 Mute This Topic: https://lists.openembedded.org/mt/103758512/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] Patchtest results for [PATCH v2 7/9] oeqa/selftest/recipetool: Move create_go test to a proper class
Is that a strict requirement now even for trivial patches? Slava On 16.01.2024 09:45, patcht...@automation.yoctoproject.org wrote: Thank you for your submission. Patchtest identified one or more issues with the patch. Please see the log below for more information: --- Testing patch /home/patchtest/share/mboxes/v2-7-9-oeqa-selftest-recipetool-Move-create_go-test-to-a-proper-class.patch FAIL: test commit message presence: Please include a commit message on your patch explaining the change (test_mbox.TestMbox.test_commit_message_presence) PASS: pretest pylint (test_python_pylint.PyLint.pretest_pylint) PASS: test Signed-off-by presence (test_mbox.TestMbox.test_signed_off_by_presence) PASS: test author valid (test_mbox.TestMbox.test_author_valid) PASS: test max line length (test_metadata.TestMetadata.test_max_line_length) PASS: test mbox format (test_mbox.TestMbox.test_mbox_format) PASS: test non-AUH upgrade (test_mbox.TestMbox.test_non_auh_upgrade) PASS: test pylint (test_python_pylint.PyLint.test_pylint) PASS: test shortlog format (test_mbox.TestMbox.test_shortlog_format) PASS: test shortlog length (test_mbox.TestMbox.test_shortlog_length) SKIP: pretest src uri left files: No modified recipes, skipping pretest (test_metadata.TestMetadata.pretest_src_uri_left_files) SKIP: test CVE check ignore: No modified recipes, skipping test (test_metadata.TestMetadata.test_cve_check_ignore) SKIP: test CVE tag format: No new CVE patches introduced (test_patch.TestPatch.test_cve_tag_format) SKIP: test Signed-off-by presence: No new CVE patches introduced (test_patch.TestPatch.test_signed_off_by_presence) SKIP: test Upstream-Status presence: No new CVE patches introduced (test_patch.TestPatch.test_upstream_status_presence_format) SKIP: test bugzilla entry format: No bug ID found (test_mbox.TestMbox.test_bugzilla_entry_format) SKIP: test lic files chksum modified not mentioned: No modified recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_modified_not_mentioned) SKIP: test lic files chksum presence: No added recipes, skipping test (test_metadata.TestMetadata.test_lic_files_chksum_presence) SKIP: test license presence: No added recipes, skipping test (test_metadata.TestMetadata.test_license_presence) SKIP: test series merge on head: Merge test is disabled for now (test_mbox.TestMbox.test_series_merge_on_head) SKIP: test src uri left files: No modified recipes, skipping pretest (test_metadata.TestMetadata.test_src_uri_left_files) SKIP: test summary presence: No added recipes, skipping test (test_metadata.TestMetadata.test_summary_presence) SKIP: test target mailing list: Series merged, no reason to check other mailing lists (test_mbox.TestMbox.test_target_mailing_list) --- Please address the issues identified and submit a new revision of the patch, or alternatively, reply to this email with an explanation of why the patch should be accepted. If you believe these results are due to an error in patchtest, please submit a bug at https://bugzilla.yoctoproject.org/ (use the 'Patchtest' category under 'Yocto Project Subprojects'). For more information on specific failures, see: https://wiki.yoctoproject.org/wiki/Patchtest. Thank you! -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#193841): https://lists.openembedded.org/g/openembedded-core/message/193841 Mute This Topic: https://lists.openembedded.org/mt/103758512/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH 1/3] recipetool: Don't fail on local go modules
On 10.01.2024 16:45, Lukas Funke wrote: On 10.01.2024 14:59, Richard Purdie wrote: On Wed, 2024-01-10 at 12:53 +0100, Vyacheslav Yurkov wrote: Local modules are usually referenced with a 'replace' directive in go.mod file. If that's the case, remove them from populating SRC_URI. Signed-off-by: Vyacheslav Yurkov --- scripts/lib/recipetool/create_go.py | 32 + 1 file changed, 19 insertions(+), 13 deletions(-) Do these changes mean we need to improve the test coverage in oe- selftest for recipetool? Test converage needs improvement, yes. I haven't considered the case for local modules since this seems to be rare(?). Are there public projects which use this mechanism? Best regards Lukas I added coverage for local modules in my v2. I'm not a golang expert, but it seems that local modules in golang is a means to "place a patched version into the project space until upstream fixes it" more or less like Yocto does with Upstream patches. It's used a lot by in-house projects, but you could also get a glimpse on github with this query https://github.com/search?q=path%3A%2Fgo.mod+NOT+is%3Afork+content%3A%2F%5Ereplace+.*%2F=code=content%3Ago.mod+replace=3 . Note, Github is able to display only first 100 search results (Github API restriction). One thing I don't like in my v2 is that I need to look up a public project to add required coverage. I would rather improve tests further by having everything in one repository and just one test, i.e. it has to be a small test project that could be used to test all go-related functionality. The question is though where it should be placed? I could create it on my Github account, but would that be a good option? Richard, any suggestions on where these projects for selftest coverage should be hosted? Slava -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#193836): https://lists.openembedded.org/g/openembedded-core/message/193836 Mute This Topic: https://lists.openembedded.org/mt/103639058/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 9/9] oeqa/selftest/recipetool: Add test coverage for local go modules
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/recipetool.py | 55 ++
1 file changed, 55 insertions(+)
diff --git a/meta/lib/oeqa/selftest/cases/recipetool.py
b/meta/lib/oeqa/selftest/cases/recipetool.py
index 83848d6170..0c296875b2 100644
--- a/meta/lib/oeqa/selftest/cases/recipetool.py
+++ b/meta/lib/oeqa/selftest/cases/recipetool.py
@@ -927,6 +927,61 @@ class RecipetoolCreateTests(RecipetoolBase):
self.assertTrue(os.path.isfile(deps_require_file))
self._test_recipe_contents(deps_require_file, checkvars, [])
+def test_recipetool_create_go_replace_modules(self):
+# Check handling of replaced modules
+temprecipe = os.path.join(self.tempdir, 'recipe')
+os.makedirs(temprecipe)
+
+recipefile = os.path.join(temprecipe, 'openapi-generator_git.bb')
+deps_require_file = os.path.join(temprecipe, 'openapi-generator',
'go-modules.inc')
+lics_require_file = os.path.join(temprecipe, 'openapi-generator',
'go-licenses.inc')
+modules_txt_file = os.path.join(temprecipe, 'openapi-generator',
'modules.txt')
+
+srcuri = 'https://github.com/OpenAPITools/openapi-generator.git'
+srcrev = "v7.2.0"
+srcbranch = "master"
+srcsubdir = "samples/openapi3/client/petstore/go"
+
+result = runCmd('recipetool create -o %s %s -S %s -B %s --src-subdir
%s' % (temprecipe, srcuri, srcrev, srcbranch, srcsubdir))
+
+self.maxDiff = None
+inherits = ['go-vendor']
+
+checkvars = {}
+checkvars['GO_IMPORT'] =
"github.com/OpenAPITools/openapi-generator/samples/openapi3/client/petstore/go"
+checkvars['SRC_URI'] =
{'git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https',
+'file://modules.txt'}
+
+self.assertNotIn('Traceback', result.output)
+self.assertIn('No license file was detected for the main module',
result.output)
+self.assertTrue(os.path.isfile(recipefile))
+self._test_recipe_contents(recipefile, checkvars, inherits)
+
+# make sure that dependencies don't mention local directory
./go-petstore
+dependencies = \
+[ ('github.com/stretchr/testify','v1.8.4', '', '', ''),
+('go.googlesource.com/oauth2','v0.10.0','golang.org/x/oauth2',
'', ''),
+('github.com/davecgh/go-spew','v1.1.1', '', '', ''),
+('github.com/golang/protobuf','v1.5.3', '', '', ''),
+('github.com/kr/pretty','v0.3.0', '', '', ''),
+('github.com/pmezard/go-difflib','v1.0.0', '', '', ''),
+('github.com/rogpeppe/go-internal','v1.9.0', '', '', ''),
+('go.googlesource.com/net','v0.12.0','golang.org/x/net', '',
''),
+
('github.com/golang/appengine','v1.6.7','google.golang.org/appengine', '', ''),
+
('go.googlesource.com/protobuf','v1.31.0','google.golang.org/protobuf', '', ''),
+('gopkg.in/check.v1','v1.0.0-20201130134442-10cb98267c6c', '',
'', ''),
+('gopkg.in/yaml.v3','v3.0.1', '', '', ''),
+]
+
+src_uri = set()
+for d in dependencies:
+src_uri.add(self._go_urifiy(*d))
+
+checkvars = {}
+checkvars['GO_DEPENDENCIES_SRC_URI'] = src_uri
+
+self.assertTrue(os.path.isfile(deps_require_file))
+self._test_recipe_contents(deps_require_file, checkvars, [])
class RecipetoolTests(RecipetoolBase):
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193832):
https://lists.openembedded.org/g/openembedded-core/message/193832
Mute This Topic: https://lists.openembedded.org/mt/103758372/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 7/9] oeqa/selftest/recipetool: Move create_go test to a proper class
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/recipetool.py | 24 ++
1 file changed, 11 insertions(+), 13 deletions(-)
diff --git a/meta/lib/oeqa/selftest/cases/recipetool.py
b/meta/lib/oeqa/selftest/cases/recipetool.py
index df15c80069..356bb4a746 100644
--- a/meta/lib/oeqa/selftest/cases/recipetool.py
+++ b/meta/lib/oeqa/selftest/cases/recipetool.py
@@ -744,19 +744,6 @@ class RecipetoolCreateTests(RecipetoolBase):
def test_recipetool_create_git_srcbranch(self):
self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;protocol=https',
'matchbox-keyboard-0-1')
-
-class RecipetoolTests(RecipetoolBase):
-
-@classmethod
-def setUpClass(cls):
-import sys
-
-super(RecipetoolTests, cls).setUpClass()
-bb_vars = get_bb_vars(['BBPATH'])
-cls.bbpath = bb_vars['BBPATH']
-libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib',
'recipetool')
-sys.path.insert(0, libpath)
-
def test_recipetool_create_go(self):
# Basic test to check go recipe generation
def urifiy(url, version, modulepath = None, pathmajor = None, subdir =
None):
@@ -941,6 +928,17 @@ class RecipetoolTests(RecipetoolBase):
self._test_recipe_contents(deps_require_file, checkvars, [])
+class RecipetoolTests(RecipetoolBase):
+
+@classmethod
+def setUpClass(cls):
+import sys
+
+super(RecipetoolTests, cls).setUpClass()
+bb_vars = get_bb_vars(['BBPATH'])
+cls.bbpath = bb_vars['BBPATH']
+libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib',
'recipetool')
+sys.path.insert(0, libpath)
def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths):
dstdir = basedstdir
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193830):
https://lists.openembedded.org/g/openembedded-core/message/193830
Mute This Topic: https://lists.openembedded.org/mt/103758370/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 8/9] oeqa/selftest/recipetool: Move helper function to the class scope
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/recipetool.py | 14 +++---
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/meta/lib/oeqa/selftest/cases/recipetool.py
b/meta/lib/oeqa/selftest/cases/recipetool.py
index 356bb4a746..83848d6170 100644
--- a/meta/lib/oeqa/selftest/cases/recipetool.py
+++ b/meta/lib/oeqa/selftest/cases/recipetool.py
@@ -744,14 +744,14 @@ class RecipetoolCreateTests(RecipetoolBase):
def test_recipetool_create_git_srcbranch(self):
self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;protocol=https',
'matchbox-keyboard-0-1')
+def _go_urifiy(self, url, version, modulepath = None, pathmajor = None,
subdir = None):
+modulepath = ",path='%s'" % modulepath if len(modulepath) else ''
+pathmajor = ",pathmajor='%s'" % pathmajor if len(pathmajor) else ''
+subdir = ",subdir='%s'" % subdir if len(subdir) else ''
+return "${@go_src_uri('%s','%s'%s%s%s)}" % (url, version, modulepath,
pathmajor, subdir)
+
def test_recipetool_create_go(self):
# Basic test to check go recipe generation
-def urifiy(url, version, modulepath = None, pathmajor = None, subdir =
None):
-modulepath = ",path='%s'" % modulepath if len(modulepath) else ''
-pathmajor = ",pathmajor='%s'" % pathmajor if len(pathmajor) else ''
-subdir = ",subdir='%s'" % subdir if len(subdir) else ''
-return "${@go_src_uri('%s','%s'%s%s%s)}" % (url, version,
modulepath, pathmajor, subdir)
-
temprecipe = os.path.join(self.tempdir, 'recipe')
os.makedirs(temprecipe)
@@ -919,7 +919,7 @@ class RecipetoolCreateTests(RecipetoolBase):
src_uri = set()
for d in dependencies:
-src_uri.add(urifiy(*d))
+src_uri.add(self._go_urifiy(*d))
checkvars = {}
checkvars['GO_DEPENDENCIES_SRC_URI'] = src_uri
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193831):
https://lists.openembedded.org/g/openembedded-core/message/193831
Mute This Topic: https://lists.openembedded.org/mt/103758371/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 6/9] recipetool: Disregard version in URL for replaced modules
Major module version is a part of name, but not necessary part of the
actual URL (See https://go.dev/ref/mod#module-path).
Nevertheless, name detection function can't handle that suffix, so get
rid of it to determine component name.
For replaced modules that name might be different that the actual module
name defined in go.mod file.
Signed-off-by: Vyacheslav Yurkov
---
scripts/lib/recipetool/create_go.py | 22 +-
1 file changed, 17 insertions(+), 5 deletions(-)
diff --git a/scripts/lib/recipetool/create_go.py
b/scripts/lib/recipetool/create_go.py
index 035ab9f7c1..c560831442 100644
--- a/scripts/lib/recipetool/create_go.py
+++ b/scripts/lib/recipetool/create_go.py
@@ -506,6 +506,7 @@ class GoRecipeHandler(RecipeHandler):
def __go_handle_dependencies(self, go_mod, srctree, localfilesdir,
extravalues, d):
+import re
src_uris = []
src_revs = []
@@ -555,7 +556,9 @@ class GoRecipeHandler(RecipeHandler):
src_uris.append(inline_fcn)
src_revs.append(generate_src_rev(path, version, commithash))
-pn, _ = determine_from_url(go_mod['Module']['Path'])
+# strip version part from module URL /vXX
+baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
+pn, _ = determine_from_url(baseurl)
go_mods_basename = "%s-modules.inc" % pn
go_mods_filename = os.path.join(localfilesdir, go_mods_basename)
@@ -636,7 +639,9 @@ class GoRecipeHandler(RecipeHandler):
lic_files_chksum.append(
'file://src/${GO_IMPORT}/vendor/%s;md5=%s' % (licvalue[1],
licvalue[2]))
-pn, _ = determine_from_url(go_mod['Module']['Path'])
+# strip version part from module URL /vXX
+baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
+pn, _ = determine_from_url(baseurl)
licenses_basename = "%s-licenses.inc" % pn
licenses_filename = os.path.join(localfilesdir, licenses_basename)
@@ -682,6 +687,13 @@ class GoRecipeHandler(RecipeHandler):
localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
extravalues.setdefault('extrafiles', {})
+
+# Use an explicit name determined from the module name because it
+# might differ from the actual URL for replaced modules
+# strip version part from module URL /vXX
+baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
+pn, _ = determine_from_url(baseurl)
+
# go.mod files with version < 1.17 may not include all indirect
# dependencies. Thus, we have to upgrade the go version.
if go_version_major == 1 and go_version_minor < 17:
@@ -699,18 +711,18 @@ class GoRecipeHandler(RecipeHandler):
# Write additional $BPN-modules.inc file
self.__go_mod_vendor(go_mod, srctree, localfilesdir, extravalues,
d)
lines_before.append("LICENSE += \" & ${GO_MOD_LICENSES}\"")
-lines_before.append("require ${BPN}-licenses.inc")
+lines_before.append("require %s-licenses.inc" % (pn))
self.__rewrite_src_uri(lines_before, ["file://modules.txt"])
self.__go_handle_dependencies(go_mod, srctree, localfilesdir,
extravalues, d)
-lines_before.append("require ${BPN}-modules.inc")
+lines_before.append("require %s-modules.inc" % (pn))
# Do generic license handling
handle_license_vars(srctree, lines_before, handled, extravalues, d)
self.__rewrite_lic_uri(lines_before)
-lines_before.append("GO_IMPORT = \"{}\"".format(go_import))
+lines_before.append("GO_IMPORT = \"{}\"".format(baseurl))
lines_before.append("SRCREV_FORMAT = \"${BPN}\"")
def __update_lines_before(self, updated, newlines, lines_before):
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193829):
https://lists.openembedded.org/g/openembedded-core/message/193829
Mute This Topic: https://lists.openembedded.org/mt/103758369/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 5/9] recipetool: Proceed even with a missing license file
Whenever the recipe uses a CLOSED license, the list is going to be
empty. It's a discouraged practice not to have a license, but proceed
anyway to finish recipe generation.
Signed-off-by: Vyacheslav Yurkov
---
scripts/lib/recipetool/create_go.py | 6 ++
1 file changed, 6 insertions(+)
diff --git a/scripts/lib/recipetool/create_go.py
b/scripts/lib/recipetool/create_go.py
index 668a24b8dd..035ab9f7c1 100644
--- a/scripts/lib/recipetool/create_go.py
+++ b/scripts/lib/recipetool/create_go.py
@@ -730,6 +730,12 @@ class GoRecipeHandler(RecipeHandler):
new_licenses = []
licenses = origvalue.split('\\')
for license in licenses:
+if not license:
+logger.warning("No license file was detected for the
main module!")
+# the license list of the main recipe must be empty
+# this can happen for example in case of CLOSED license
+# Fall through to complete recipe generation
+continue
license = license.strip()
uri, chksum = license.split(';', 1)
url = urllib.parse.urlparse(uri)
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193828):
https://lists.openembedded.org/g/openembedded-core/message/193828
Mute This Topic: https://lists.openembedded.org/mt/103758368/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 4/9] classes: go-vendor: Unlink vendor dir later
Vendor directory might still be required during install stage, so defer
the removal until later stage.
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/go-vendor.bbclass | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
index 48f4d1041c..1bbb99ac79 100644
--- a/meta/classes/go-vendor.bbclass
+++ b/meta/classes/go-vendor.bbclass
@@ -48,7 +48,7 @@ python do_vendor_unlink() {
os.unlink(linkname)
}
-addtask vendor_unlink before do_install after do_compile
+addtask vendor_unlink before do_package after do_install
python do_go_vendor() {
import shutil
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193827):
https://lists.openembedded.org/g/openembedded-core/message/193827
Mute This Topic: https://lists.openembedded.org/mt/103758367/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 3/9] classes: go-vendor: Handle modules from the same repo
Take into account module version when populating vendor directory,
because a module with the same URL but with a different version tag
could be used as an indirect dependency.
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/go-vendor.bbclass | 9 -
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
index b965428dd1..48f4d1041c 100644
--- a/meta/classes/go-vendor.bbclass
+++ b/meta/classes/go-vendor.bbclass
@@ -103,18 +103,17 @@ python do_go_vendor() {
pathMajor = fetcher.ud[url].parm.get('go_pathmajor')
pathMajor = None if not pathMajor else pathMajor.strip('/')
-if not repo in modules:
-modules[repo] = { "version": version,
+if not (repo, version) in modules:
+modules[(repo, version)] = {
"repo_path": os.path.join(import_dir, p),
"module_path": module_path,
"subdir": subdir,
"pathMajor": pathMajor }
-for module_key in sorted(modules):
+for module_key, module in modules.items():
# only take the version which is explicitly listed
# as a dependency in the go.mod
-module = modules[module_key]
module_path = module['module_path']
rootdir = module['repo_path']
subdir = module['subdir']
@@ -139,7 +138,7 @@ python do_go_vendor() {
dst = os.path.join(vendor_dir, module_path)
bb.debug(1, "cp %s --> %s" % (src, dst))
-shutil.copytree(src, dst, symlinks=True, \
+shutil.copytree(src, dst, symlinks=True, dirs_exist_ok=True, \
ignore=shutil.ignore_patterns(".git", \
"vendor", \
"*._test.go"))
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193826):
https://lists.openembedded.org/g/openembedded-core/message/193826
Mute This Topic: https://lists.openembedded.org/mt/103758366/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 1/9] recipetool: Don't fail on local go modules
Local modules are usually referenced with a 'replace' directive in
go.mod file. If that's the case, remove them from populating SRC_URI.
Signed-off-by: Vyacheslav Yurkov
---
scripts/lib/recipetool/create_go.py | 36 ++---
1 file changed, 23 insertions(+), 13 deletions(-)
diff --git a/scripts/lib/recipetool/create_go.py
b/scripts/lib/recipetool/create_go.py
index 21dcb41271..668a24b8dd 100644
--- a/scripts/lib/recipetool/create_go.py
+++ b/scripts/lib/recipetool/create_go.py
@@ -504,7 +504,7 @@ class GoRecipeHandler(RecipeHandler):
return inline_fcn, commit
-def __go_handle_dependencies(self, go_mod, localfilesdir, extravalues, d):
+def __go_handle_dependencies(self, go_mod, srctree, localfilesdir,
extravalues, d):
src_uris = []
src_revs = []
@@ -525,6 +525,27 @@ class GoRecipeHandler(RecipeHandler):
return src_rev
+# we first go over replacement list, because we are essentialy
+# interested only in the replaced path
+if go_mod['Replace']:
+for replacement in go_mod['Replace']:
+oldpath = replacement['Old']['Path']
+path = replacement['New']['Path']
+version = ''
+if 'Version' in replacement['New']:
+version = replacement['New']['Version']
+
+if os.path.exists(os.path.join(srctree, path)):
+# the module refers to the local path, remove it from
requirement list
+# because it's a local module
+go_mod['Require'][:] = [v for v in go_mod['Require'] if
v.get('Path') != oldpath]
+else:
+# Replace the path and the version, so we don't iterate
replacement list anymore
+for require in go_mod['Require']:
+if require['Path'] == oldpath:
+require.update({'Path': path, 'Version': version})
+break
+
for require in go_mod['Require']:
path = require['Path']
version = require['Version']
@@ -534,17 +555,6 @@ class GoRecipeHandler(RecipeHandler):
src_uris.append(inline_fcn)
src_revs.append(generate_src_rev(path, version, commithash))
-if go_mod['Replace']:
-for replacement in go_mod['Replace']:
-oldpath = replacement['Old']['Path']
-path = replacement['New']['Path']
-version = replacement['New']['Version']
-
-inline_fcn, commithash = self.__generate_srcuri_inline_fcn(
-path, version, oldpath)
-src_uris.append(inline_fcn)
-src_revs.append(generate_src_rev(path, version, commithash))
-
pn, _ = determine_from_url(go_mod['Module']['Path'])
go_mods_basename = "%s-modules.inc" % pn
@@ -693,7 +703,7 @@ class GoRecipeHandler(RecipeHandler):
self.__rewrite_src_uri(lines_before, ["file://modules.txt"])
-self.__go_handle_dependencies(go_mod, localfilesdir, extravalues,
d)
+self.__go_handle_dependencies(go_mod, srctree, localfilesdir,
extravalues, d)
lines_before.append("require ${BPN}-modules.inc")
# Do generic license handling
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193824):
https://lists.openembedded.org/g/openembedded-core/message/193824
Mute This Topic: https://lists.openembedded.org/mt/103758364/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 2/9] classes: go-vendor: Reference local modules
Create symlinks for local modules, which are usually not referenced in
the SRC_URI, but still expected to be found in the vendor directory
during the build.
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/go-vendor.bbclass | 18 +-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
index 2426bddfba..b965428dd1 100644
--- a/meta/classes/go-vendor.bbclass
+++ b/meta/classes/go-vendor.bbclass
@@ -169,6 +169,7 @@ python do_go_vendor() {
fetched_paths.remove('.')
vendored_paths = set()
+replaced_paths = dict()
with open(modules_txt_src) as f:
for line in f:
if not line.startswith("#"):
@@ -182,6 +183,15 @@ python do_go_vendor() {
vendored_paths.add(topdir)
topdir = os.path.dirname(topdir)
+else:
+replaced_module = line.split("=>")
+if len(replaced_module) > 1:
+# This module has been replaced, use a local path
+# we parse the line that has a pattern "# module-name
[module-version] => local-path
+actual_path = replaced_module[1].strip()
+vendored_name = replaced_module[0].split()[1]
+bb.debug(1, "added vendored name %s for actual path %s" %
(vendored_name, actual_path))
+replaced_paths[vendored_name] = actual_path
for path in fetched_paths:
if path not in vendored_paths:
@@ -189,7 +199,13 @@ python do_go_vendor() {
if os.path.exists(realpath):
shutil.rmtree(realpath)
-# Create a symlink the the actual directory
+for vendored_name, replaced_path in replaced_paths.items():
+symlink_target = os.path.join(source_dir, *['src', go_import,
replaced_path])
+symlink_name = os.path.join(vendor_dir, vendored_name)
+bb.debug(1, "vendored name %s, symlink name %s" % (vendored_name,
symlink_name))
+os.symlink(symlink_target, symlink_name)
+
+# Create a symlink to the actual directory
os.symlink(vendor_dir, linkname)
}
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193825):
https://lists.openembedded.org/g/openembedded-core/message/193825
Mute This Topic: https://lists.openembedded.org/mt/103758365/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 3/3] classes: go-vendor: Handle modules from the same repo
Take into account module version when populating vendor directory,
because a module with the same URL but with a different version tag
could be used as an indirect dependency.
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/go-vendor.bbclass | 9 -
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
index b965428dd1..48f4d1041c 100644
--- a/meta/classes/go-vendor.bbclass
+++ b/meta/classes/go-vendor.bbclass
@@ -103,18 +103,17 @@ python do_go_vendor() {
pathMajor = fetcher.ud[url].parm.get('go_pathmajor')
pathMajor = None if not pathMajor else pathMajor.strip('/')
-if not repo in modules:
-modules[repo] = { "version": version,
+if not (repo, version) in modules:
+modules[(repo, version)] = {
"repo_path": os.path.join(import_dir, p),
"module_path": module_path,
"subdir": subdir,
"pathMajor": pathMajor }
-for module_key in sorted(modules):
+for module_key, module in modules.items():
# only take the version which is explicitly listed
# as a dependency in the go.mod
-module = modules[module_key]
module_path = module['module_path']
rootdir = module['repo_path']
subdir = module['subdir']
@@ -139,7 +138,7 @@ python do_go_vendor() {
dst = os.path.join(vendor_dir, module_path)
bb.debug(1, "cp %s --> %s" % (src, dst))
-shutil.copytree(src, dst, symlinks=True, \
+shutil.copytree(src, dst, symlinks=True, dirs_exist_ok=True, \
ignore=shutil.ignore_patterns(".git", \
"vendor", \
"*._test.go"))
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193495):
https://lists.openembedded.org/g/openembedded-core/message/193495
Mute This Topic: https://lists.openembedded.org/mt/103639060/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/3] recipetool: Don't fail on local go modules
Local modules are usually referenced with a 'replace' directive in
go.mod file. If that's the case, remove them from populating SRC_URI.
Signed-off-by: Vyacheslav Yurkov
---
scripts/lib/recipetool/create_go.py | 32 +
1 file changed, 19 insertions(+), 13 deletions(-)
diff --git a/scripts/lib/recipetool/create_go.py
b/scripts/lib/recipetool/create_go.py
index 21dcb41271..bd20b07f91 100644
--- a/scripts/lib/recipetool/create_go.py
+++ b/scripts/lib/recipetool/create_go.py
@@ -504,7 +504,7 @@ class GoRecipeHandler(RecipeHandler):
return inline_fcn, commit
-def __go_handle_dependencies(self, go_mod, localfilesdir, extravalues, d):
+def __go_handle_dependencies(self, go_mod, srctree, localfilesdir,
extravalues, d):
src_uris = []
src_revs = []
@@ -525,6 +525,23 @@ class GoRecipeHandler(RecipeHandler):
return src_rev
+# we first go over replacement list, because we are essentialy
+# interested only in the replaced path
+if go_mod['Replace']:
+for replacement in go_mod['Replace']:
+oldpath = replacement['Old']['Path']
+path = replacement['New']['Path']
+if 'Version' in replacement['New']:
+version = replacement['New']['Version']
+
+if os.path.exists(os.path.join(srctree, path)):
+# the module refers to the local path, remove it from
requirement list
+# because it's a local module
+go_mod['Require'][:] = [v for v in go_mod['Require'] if
v.get('Path') != oldpath]
+else:
+# Replace the path and the version, so we don't iterate
replacement list anymore
+go_mod['Require']['Path'] = path
+
for require in go_mod['Require']:
path = require['Path']
version = require['Version']
@@ -534,17 +551,6 @@ class GoRecipeHandler(RecipeHandler):
src_uris.append(inline_fcn)
src_revs.append(generate_src_rev(path, version, commithash))
-if go_mod['Replace']:
-for replacement in go_mod['Replace']:
-oldpath = replacement['Old']['Path']
-path = replacement['New']['Path']
-version = replacement['New']['Version']
-
-inline_fcn, commithash = self.__generate_srcuri_inline_fcn(
-path, version, oldpath)
-src_uris.append(inline_fcn)
-src_revs.append(generate_src_rev(path, version, commithash))
-
pn, _ = determine_from_url(go_mod['Module']['Path'])
go_mods_basename = "%s-modules.inc" % pn
@@ -693,7 +699,7 @@ class GoRecipeHandler(RecipeHandler):
self.__rewrite_src_uri(lines_before, ["file://modules.txt"])
-self.__go_handle_dependencies(go_mod, localfilesdir, extravalues,
d)
+self.__go_handle_dependencies(go_mod, srctree, localfilesdir,
extravalues, d)
lines_before.append("require ${BPN}-modules.inc")
# Do generic license handling
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193493):
https://lists.openembedded.org/g/openembedded-core/message/193493
Mute This Topic: https://lists.openembedded.org/mt/103639058/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/3] classes: go-vendor: Reference local modules
Create symlinks for local modules, which are usually not referenced in
the SRC_URI, but still expected to be found in the vendor directory
during the build.
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/go-vendor.bbclass | 18 +-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
index 2426bddfba..b965428dd1 100644
--- a/meta/classes/go-vendor.bbclass
+++ b/meta/classes/go-vendor.bbclass
@@ -169,6 +169,7 @@ python do_go_vendor() {
fetched_paths.remove('.')
vendored_paths = set()
+replaced_paths = dict()
with open(modules_txt_src) as f:
for line in f:
if not line.startswith("#"):
@@ -182,6 +183,15 @@ python do_go_vendor() {
vendored_paths.add(topdir)
topdir = os.path.dirname(topdir)
+else:
+replaced_module = line.split("=>")
+if len(replaced_module) > 1:
+# This module has been replaced, use a local path
+# we parse the line that has a pattern "# module-name
[module-version] => local-path
+actual_path = replaced_module[1].strip()
+vendored_name = replaced_module[0].split()[1]
+bb.debug(1, "added vendored name %s for actual path %s" %
(vendored_name, actual_path))
+replaced_paths[vendored_name] = actual_path
for path in fetched_paths:
if path not in vendored_paths:
@@ -189,7 +199,13 @@ python do_go_vendor() {
if os.path.exists(realpath):
shutil.rmtree(realpath)
-# Create a symlink the the actual directory
+for vendored_name, replaced_path in replaced_paths.items():
+symlink_target = os.path.join(source_dir, *['src', go_import,
replaced_path])
+symlink_name = os.path.join(vendor_dir, vendored_name)
+bb.debug(1, "vendored name %s, symlink name %s" % (vendored_name,
symlink_name))
+os.symlink(symlink_target, symlink_name)
+
+# Create a symlink to the actual directory
os.symlink(vendor_dir, linkname)
}
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193494):
https://lists.openembedded.org/g/openembedded-core/message/193494
Mute This Topic: https://lists.openembedded.org/mt/103639059/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-Core][PATCH v3 0/4] recipetool: Add handler to create go recipes
Hi Lukas,
Sorry for the late reply.
I think I see what's going on.
The module I'm working on is already "vendored", which means there's no
modules.txt created for my project by recipetool, since it's already
provided in the repository itself.
I guess this has pros and cons, and I'm currently not sure which
approach is better. On one hand, offline vendoring (no by the
recipetool) has an advantage for SDK builds, when people don't use the
BSP directly. On the other hand, Yocto style vendoring (when modules.txt
is stored in the BSP) is more transparent for dependency management.
I.e. I would need to update the recipe when my dependencies change.
Slava
On 15.11.2023 08:34, Vyacheslav Yurkov via lists.openembedded.org wrote:
On 15.11.2023 08:30, Lukas Funke wrote:
Hi Slava,
No problem. Party is still going on. The 'modules.txt' is intendet to
be in your ${WORKDIR} as it is part of the SRC_URI, generated by
the recipetool. My guess would be, that there is some
missconfiguration in your SRC_URI?
Yes, I noticed that it should be there in recipetool tests. But
generated recipe didn't have It in SRC_URI. I tried to use generated
recipe as is. I'll try to double check later.
Slava
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#193015):
https://lists.openembedded.org/g/openembedded-core/message/193015
Mute This Topic: https://lists.openembedded.org/mt/102345308/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core][master][PATCH] ptest-packagelists: Add sdbus-c++
On 21.11.2023 14:24, Alexander Kanavin wrote: On Tue, 21 Nov 2023 at 14:22, Poonam Jadhav wrote: Add sdbus-c++ in ptest-packagelists to run ptest sdbus-c++ is not in oe-core, and so cannot be added. You need to add it to the list via .bbappend or a custom image recipe. Alex It's in meta-oe. Please resend it to openembedded-de...@lists.openembedded.org Slava -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#190979): https://lists.openembedded.org/g/openembedded-core/message/190979 Mute This Topic: https://lists.openembedded.org/mt/102727832/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-Core][PATCH v3 0/4] recipetool: Add handler to create go recipes
On 15.11.2023 08:30, Lukas Funke wrote:
Hi Slava,
No problem. Party is still going on. The 'modules.txt' is intendet to
be in your ${WORKDIR} as it is part of the SRC_URI, generated by
the recipetool. My guess would be, that there is some
missconfiguration in your SRC_URI?
Yes, I noticed that it should be there in recipetool tests. But
generated recipe didn't have It in SRC_URI. I tried to use generated
recipe as is. I'll try to double check later.
Slava
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#190558):
https://lists.openembedded.org/g/openembedded-core/message/190558
Mute This Topic: https://lists.openembedded.org/mt/102345308/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-Core][PATCH v3 0/4] recipetool: Add handler to create go recipes
Hi Lukas,
Thanks for the v3. I know I'm late to the party, because it's already
merged, but my testing results are below anyway.
The series seems to have handled my ssh URL correctly.
I've got an error in do_go_vendor though.
File: ''/meta/classes/go-vendor.bbclass', lineno: 166, function:
do_go_vendor
0162:
0163: # Copy vendor manifest
0164: modules_txt_src = os.path.join(d.getVar('WORKDIR'),
"modules.txt")
0165: bb.debug(1, "cp %s --> %s" % (modules_txt_src, vendor_dir))
*** 0166: shutil.copy2(modules_txt_src, vendor_dir)
0167:
0168: # Clean up vendor dir
0169: # We only require the modules in the modules_txt file
0170: fetched_paths = set([os.path.relpath(x[0], vendor_dir)
for x in os.walk(vendor_dir)])
File: '/usr/lib64/python3.9/shutil.py', lineno: 444, function: copy2
0440: resembles GNU's "cp -P src dst".
0441: """
0442: if os.path.isdir(dst):
0443: dst = os.path.join(dst, os.path.basename(src))
*** 0444: copyfile(src, dst, follow_symlinks=follow_symlinks)
0445: copystat(src, dst, follow_symlinks=follow_symlinks)
0446: return dst
0447:
0448:def ignore_patterns(*patterns):
File: '/usr/lib64/python3.9/shutil.py', lineno: 264, function: copyfile
0260:
0261: if not follow_symlinks and _islink(src):
0262: os.symlink(os.readlink(src), dst)
0263: else:
*** 0264: with open(src, 'rb') as fsrc:
0265: try:
0266: with open(dst, 'wb') as fdst:
0267: # macOS
0268: if _HAS_FCOPYFILE:
Exception: FileNotFoundError: [Errno 2] No such file or directory:
'/modules.txt'
I've located "modules.txt" in my $WORKDIR/$GO_IMPORT/vendor directory
though, which seems to be consistent with
https://go.dev/ref/mod#go-mod-file-go . Am I missing something or I can
send a fixup?
Slava
On 02.11.2023 16:53, lukas.funke-...@weidmueller.com wrote:
From: Lukas Funke
This patch series adds a recipetool handler in order to create 'go' recipes.
Each recipe contains a list of dependencies in their SRC_URI.
Dependencies are derived from the projects `go.mod` file. For each
dependency the corresponding license file uri/hash is added.
The recipe may not work ad-hoc, but is a good starting point to create
a working recipe and have a working offline-build.
In addition to the main recipe three additional files will be generated:
- $pn-modules.inc
- $pn-license.inc
- modules.txt
Changes from v2:
- Generate separate *.inc for go dependencies and licenses
- Adapted oe-selftest according to change above
- Incorparate community suggestions
Lukas Funke (4):
classes: go-vendor: Add go-vendor class
selftest: recipetool: Add test for go recipe handler
recipetool: Ignore *.go files while scanning for licenses
recipetool: Add handler to create go recipes
meta/classes/go-vendor.bbclass | 135
meta/lib/oeqa/selftest/cases/recipetool.py | 163 +
scripts/lib/recipetool/create.py | 2 +-
scripts/lib/recipetool/create_go.py| 730 +
4 files changed, 1029 insertions(+), 1 deletion(-)
create mode 100644 meta/classes/go-vendor.bbclass
create mode 100644 scripts/lib/recipetool/create_go.py
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#190555):
https://lists.openembedded.org/g/openembedded-core/message/190555
Mute This Topic: https://lists.openembedded.org/mt/102345308/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2] staging.bbclass: Inlcude native dirs for nativesdk build
Some build systems (like cmake) check whether target binaries are
present even if they are not used during compilation. Excluded ${bindir}
causes such recipes to fail to build nativesdk variant. The good
example of that is protobuf/grpc nativesdk build from meta-oe layer.
For more details refer this long discussion:
https://lists.openembedded.org/g/openembedded-devel/topic/101679410#105284
Signed-off-by: Vyacheslav Yurkov
---
meta/classes-global/staging.bbclass | 1 +
1 file changed, 1 insertion(+)
diff --git a/meta/classes-global/staging.bbclass
b/meta/classes-global/staging.bbclass
index d229f40107..72c811d609 100644
--- a/meta/classes-global/staging.bbclass
+++ b/meta/classes-global/staging.bbclass
@@ -26,6 +26,7 @@ SYSROOT_DIRS_NATIVE = " \
${localstatedir} \
"
SYSROOT_DIRS:append:class-native = " ${SYSROOT_DIRS_NATIVE}"
+SYSROOT_DIRS:append:class-nativesdk = " ${SYSROOT_DIRS_NATIVE}"
SYSROOT_DIRS:append:class-cross = " ${SYSROOT_DIRS_NATIVE}"
SYSROOT_DIRS:append:class-crosssdk = " ${SYSROOT_DIRS_NATIVE}"
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#190400):
https://lists.openembedded.org/g/openembedded-core/message/190400
Mute This Topic: https://lists.openembedded.org/mt/102502647/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] staging.bbclass: Inlcude native dirs for nativesdk build
Some build systems (like cmake) check whether target binaries are
present even if they are not used during compilation. Excluded ${bindir}
causes such recipes to faile to build nativesdk variant. The good
example of that is protobuf/grps nativesdk build.
For more details refer this long discussion:
https://lists.openembedded.org/g/openembedded-devel/topic/101679410#105284
Signed-off-by: Vyacheslav Yurkov
---
meta/classes-global/staging.bbclass | 1 +
1 file changed, 1 insertion(+)
diff --git a/meta/classes-global/staging.bbclass
b/meta/classes-global/staging.bbclass
index d229f40107..72c811d609 100644
--- a/meta/classes-global/staging.bbclass
+++ b/meta/classes-global/staging.bbclass
@@ -26,6 +26,7 @@ SYSROOT_DIRS_NATIVE = " \
${localstatedir} \
"
SYSROOT_DIRS:append:class-native = " ${SYSROOT_DIRS_NATIVE}"
+SYSROOT_DIRS:append:class-nativesdk = " ${SYSROOT_DIRS_NATIVE}"
SYSROOT_DIRS:append:class-cross = " ${SYSROOT_DIRS_NATIVE}"
SYSROOT_DIRS:append:class-crosssdk = " ${SYSROOT_DIRS_NATIVE}"
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#190399):
https://lists.openembedded.org/g/openembedded-core/message/190399
Mute This Topic: https://lists.openembedded.org/mt/102502625/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] lib/oe/path: Deploy files can start only with a dot
From: Vyacheslav Yurkov
There might be only hidden files deployed. In that case we don't need a
generic wildcard present in copy command, otherwise it fails with:
Exception: subprocess.CalledProcessError: Command 'cp -afl --preserve=xattr
./.??* ./* /tmp/deploy/images/qemux86-64' returned non-zero exit
status 1.
Subprocess output:
cp: cannot stat './*': No such file or directory
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oe/path.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py
index 0dc8f172d5..e2f1913a35 100644
--- a/meta/lib/oe/path.py
+++ b/meta/lib/oe/path.py
@@ -125,7 +125,8 @@ def copyhardlinktree(src, dst):
if os.path.isdir(src):
if len(glob.glob('%s/.??*' % src)) > 0:
source = './.??* '
-source += './*'
+if len(glob.glob('%s/**' % src)) > 0:
+source += './*'
s_dir = src
else:
source = src
--
2.42.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#190076):
https://lists.openembedded.org/g/openembedded-core/message/190076
Mute This Topic: https://lists.openembedded.org/mt/102339026/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-Core][PATCH v2 0/4] recipetool: Add handler to create go recipes
On 24.10.2023 08:19, Lukas Funke wrote: - Could please clarify where does the version from go.mod hide? Is it taken directly from go.mod? I'm trying to understand what should be the workflow when a module version should be bumped up in the go.mod. Will that be reflected in the recipe in any way? No, currently the go-version doesn't play a role ATM. Except one case when you have a go.mod file with go < 1.17. These go.mod files don't include indirect dependencies. Still trying to wrap my head around... When there's no version at parsing stage, how this will affect reproducibility? If it's not known, then whenever the version is bumped up in go.mod, a manual 'clean all' will be required? (It's probably the same as now though). Maybe I don't understand the problem: Is it required for the go module to have the *same* version as the golang package in yocto? In my understanding, when the golang version is greater-equal to the go.mod version we're good? I think I mixed up with revisions here a bit. What I meant is how the bitbake would know if versions of dependent components in go.mod have been updated. The easy answer I guess is that the revision of the main recipe (that contains go.mod) needs to be updated for that, and I hope that bitbake would refetch new versions from go.mod, but I didn't check it yet. The more complicated scenario, what if I use a devtool workflow? Will the fetcher be able to reparse go.mod in this case? Slava -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#189639): https://lists.openembedded.org/g/openembedded-core/message/189639 Mute This Topic: https://lists.openembedded.org/mt/102017388/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-Core][PATCH v2 0/4] recipetool: Add handler to create go recipes
Hey Lukas
On 24.10.2023 08:33, Lukas Funke wrote:
- I placed the correct URL into SRC_URI, but do_go_vendor still
failed with following stacktrace:
File:
'/home/uvv/projects/yocto-lorch-mapro/openembedded-core/meta/classes/go-vendor.bbclass',
lineno: 86, function: do_go_vendor
0082: # path = github.com/foo/bar
0083: # version = v1.2.3
0084:
0085: p = destsuffix[len(default_destsuffix)+1:]
*** 0086: path, version = p.split('@')
0087:
0088: subdir = fetcher.ud[url].parm.get('go_subdir')
0089: subdir = "" if not subdir else subdir
0090:
Exception: ValueError: not enough values to unpack (expected 2, got 1)
The reason is that my go.mod name does not have a version component.
If I understood the convention https://go.dev/ref/mod#introduction,
it's not a required component, so this should be taken into account.
This error could happen if your dependencies don't have a version.
I've never seen this in my experiments. Maybe check your go.mod file
for the missing version info.
I debugged it a bit and see that the error is actually caused by my URL
modification. The URL that works for me looks like
SRC_URI = git://git@${GO_IMPORT}.git;...
The parsing expects the version after "@", which is not right anymore.
Slava
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#189638):
https://lists.openembedded.org/g/openembedded-core/message/189638
Mute This Topic: https://lists.openembedded.org/mt/102017388/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-Core][PATCH v2 0/4] recipetool: Add handler to create go recipes
On 23.10.2023 14:18, Lukas Funke wrote:
Hi Slava,
On 22.10.2023 20:34, Vyacheslav Yurkov wrote:
Hey Lukas,
Thanks a lot for the patch. A few questions/comments from my initial
test below.
- I tried it with a go-based backend I have by providing ssh URL to
github. It seems like the GO_IMPORT is set to a module name from
go.mod of my project, which of course fails to fetch it like that,
because it's not a valid URL. How is it supposed to be used?
Your assumption is correct: the GO_IMPORT is taken from the go.mod file.
I've not considered the case where the repo URL is unequal to the
module path. This may require manual rework of the recipe. Another
solution would be to take the source URL from the recipetool. I think
there is no correct solution to this problem, because probably most
people might want to have the original solution, since they are
creating recipes from oss components (i.e. from github).
Some more results from my tests.
- I refactored module name to contain a valid URL... It seems though
that current version of go_src_uri does not handle ssh URLs, and all
required info from URL was lost (git@ component, ssh protocol, .git suffix).
- I placed the correct URL into SRC_URI, but do_go_vendor still failed
with following stacktrace:
File:
'/home/uvv/projects/yocto-lorch-mapro/openembedded-core/meta/classes/go-vendor.bbclass',
lineno: 86, function: do_go_vendor
0082: # path = github.com/foo/bar
0083: # version = v1.2.3
0084:
0085: p = destsuffix[len(default_destsuffix)+1:]
*** 0086: path, version = p.split('@')
0087:
0088: subdir = fetcher.ud[url].parm.get('go_subdir')
0089: subdir = "" if not subdir else subdir
0090:
Exception: ValueError: not enough values to unpack (expected 2, got 1)
The reason is that my go.mod name does not have a version component. If
I understood the convention https://go.dev/ref/mod#introduction, it's
not a required component, so this should be taken into account.
Thanks,
Slava
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#189631):
https://lists.openembedded.org/g/openembedded-core/message/189631
Mute This Topic: https://lists.openembedded.org/mt/102017388/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-Core][PATCH v2 0/4] recipetool: Add handler to create go recipes
On 23.10.2023 14:18, Lukas Funke wrote: Hi Slava, On 22.10.2023 20:34, Vyacheslav Yurkov wrote: Hey Lukas, Thanks a lot for the patch. A few questions/comments from my initial test below. - I tried it with a go-based backend I have by providing ssh URL to github. It seems like the GO_IMPORT is set to a module name from go.mod of my project, which of course fails to fetch it like that, because it's not a valid URL. How is it supposed to be used? Your assumption is correct: the GO_IMPORT is taken from the go.mod file. I've not considered the case where the repo URL is unequal to the module path. This may require manual rework of the recipe. Another solution would be to take the source URL from the recipetool. I think there is no correct solution to this problem, because probably most people might want to have the original solution, since they are creating recipes from oss components (i.e. from github). Ah, it could be that module name is not used correctly in our backend. I'll try to put a proper URL there, thanks. The file will explode anyway. Thanks 'go'. Adding more intelligence to the underlying license detection could be an idea: in your first example, the license file contains actually two licenses: MIT and Apache. Yocto has no way to detect those cases. Yeah, I only mentioned 2 out of 20 in my email to not make it huge ;) There are indeed around 20. I'll try to prepare the list and send the patch to include them. - Could please clarify where does the version from go.mod hide? Is it taken directly from go.mod? I'm trying to understand what should be the workflow when a module version should be bumped up in the go.mod. Will that be reflected in the recipe in any way? No, currently the go-version doesn't play a role ATM. Except one case when you have a go.mod file with go < 1.17. These go.mod files don't include indirect dependencies. Still trying to wrap my head around... When there's no version at parsing stage, how this will affect reproducibility? If it's not known, then whenever the version is bumped up in go.mod, a manual 'clean all' will be required? (It's probably the same as now though). Thanks, Slava -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#189625): https://lists.openembedded.org/g/openembedded-core/message/189625 Mute This Topic: https://lists.openembedded.org/mt/102017388/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-Core][PATCH v2 0/4] recipetool: Add handler to create go recipes
Hey Lukas, Thanks a lot for the patch. A few questions/comments from my initial test below. - I tried it with a go-based backend I have by providing ssh URL to github. It seems like the GO_IMPORT is set to a module name from go.mod of my project, which of course fails to fetch it like that, because it's not a valid URL. How is it supposed to be used? - I've got about 20 lines like: INFO: Please add the following line for 'vendor/google.golang.org/protobuf/LICENSE' to a 'lib/recipetool/licenses.csv' and replace `Unknown` with the license: 02d4002e9171d41a8fad93aa7faf3956,Unknown INFO: Please add the following line for 'vendor/gopkg.in/yaml.v3/LICENSE' to a 'lib/recipetool/licenses.csv' and replace `Unknown` with the license: 3c91c17266710e16afdbb2b6d15c761c,Unknown I believe I have to go through all of them manually and set a proper license in the recipe. Still the questions arises, what should be a better way to reduce amount of this work? Adding more hashes/licenses to lib/recipetool/licenses.csv would be a way to go? I'm just afraid that file might explode if we use it like that... - Could please clarify where does the version from go.mod hide? Is it taken directly from go.mod? I'm trying to understand what should be the workflow when a module version should be bumped up in the go.mod. Will that be reflected in the recipe in any way? Thanks, Slava On 17.10.2023 15:26, lukas.funke-...@weidmueller.com wrote: From: Lukas Funke This patch series adds a recipetool handler in order to create 'go' recipes. Each recipe contains a list of dependencies in their SRC_URI variable which are derived from the projects `go.mod` file. For each dependency the corresponding license file uri/hash is added. The recipe may not work ad-hoc, but is a good starting point to create a working recipe and have a working offline-build. Lukas Funke (4): classes: go-vendor: Add go-vendor class selftest: recipetool: Add test for go recipe handler recipetool: Ignore *.go files while scanning for licenses recipetool: Add handler to create go recipes meta/classes/go-vendor.bbclass | 135 meta/lib/oeqa/selftest/cases/recipetool.py | 163 + scripts/lib/recipetool/create.py | 2 +- scripts/lib/recipetool/create_go.py| 730 + 4 files changed, 1029 insertions(+), 1 deletion(-) create mode 100644 meta/classes/go-vendor.bbclass create mode 100644 scripts/lib/recipetool/create_go.py -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#189611): https://lists.openembedded.org/g/openembedded-core/message/189611 Mute This Topic: https://lists.openembedded.org/mt/102017388/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] Dependency on packages
Hey everyone. As you know, DEPENDS operates on a recipe name basis, and RDEPENDS on a package name basis. I'm working on a dependency issue in meta-openembedded, where I need to indicate a package dependency during a build stage. Any advice how should I do it? (if any details necessary it's nativesdk-grpc/nativesdk-protobuf recipes) Regards, Slava -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#189324): https://lists.openembedded.org/g/openembedded-core/message/189324 Mute This Topic: https://lists.openembedded.org/mt/102017321/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] kirkstone boost upgrade?
I would be cautious with boost upgrade in stable branch. Boost is known for having breaking changes almost each new version. I only had a glance at 1.79-1.81 https://www.boost.org/users/history/ and there are a couple of them. Are you affected by some particular bug or vulnerability in 1.78? Vyacheslav On 21.07.2023 06:49, Thomas Roos via lists.openembedded.org wrote: Hi, we are hitting an issue with boost version 1.78 and like to ask if and how there is a boost upgrade on kirkstone branch? Cheers, Thomas Amazon Web Services EMEA SARL 38 avenue John F. Kennedy, L-1855 Luxembourg Sitz der Gesellschaft: L-1855 Luxemburg eingetragen im Luxemburgischen Handelsregister unter R.C.S. B186284 Amazon Web Services EMEA SARL, Niederlassung Deutschland Marcel-Breuer-Str. 12, D-80807 Muenchen Sitz der Zweigniederlassung: Muenchen eingetragen im Handelsregister des Amtsgerichts Muenchen unter HRB 242240, USt-ID DE317013094 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#184668): https://lists.openembedded.org/g/openembedded-core/message/184668 Mute This Topic: https://lists.openembedded.org/mt/100271099/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] systemd service@ bug
Hey, Usually Steve Sakoman cherry-picks patches from master. If it's not the case, it was probably not applied clearly or simply overlooked. You can submit a backport with the branch tag. Vyacheslav On 19.07.2023 03:48, Yuta Hayama wrote: Hi, This issue has been fixed in master. https://git.openembedded.org/openembedded-core/commit/?id=d18b939fb08b37380ce95934da38e6522392621c But, yes. This patch has not yet been backported to the any stable branch. I don't know about the maintenance process for the stable branch, but I expect that patch will probably be queued and backported to dunfell in a month or so. Please let me know if anyone knows anything about this. Should we simply wait? Or do I have to submit a backport request? Regards, Yuta Hayama -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#184567): https://lists.openembedded.org/g/openembedded-core/message/184567 Mute This Topic: https://lists.openembedded.org/mt/100140897/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [langdale][kirkstone][PATCH] go-mod.bbclass: Allow network in do_compile
On 26.12.2022 10:59, Martin Jansa wrote:
Please try recipetool changes on your go-mod recipes and provide feedback.
Coincidentally, I also needed grpc-web recipe, which original patchset
was tested on.
I applied the series on my kirkstone branch of poky and tried to run the
command:
% recipetool create -S c1971f71a28950d212b7ddc8df07b5563723031d -B
master https://github.com/improbable-eng/grpc-web.git
Unfortunately this failed with the following log:
WARNING: go.mod files generated by Go < 1.17 might have incomplete
indirect dependencies.
Traceback (most recent call last):
File "/home/uvv/projects/poky/scripts/recipetool", line 111, in
ret = main()
File "/home/uvv/projects/poky/scripts/recipetool", line 100, in main
ret = args.func(args)
File "/home/uvv/projects/poky/scripts/lib/recipetool/create.py", line
746, in create_recipe
handler.process(srctree_use, classes, lines_before, lines_after,
handled, extravalues)
File "/home/uvv/projects/poky/scripts/lib/recipetool/create_go.py",
line 332, in process
go_mod = self._go_mod_patch(patchfile, go_import, srctree,
localfilesdir, extravalues, d)
File "/home/uvv/projects/poky/scripts/lib/recipetool/create_go.py",
line 291, in _go_mod_patch
bb.process.run("go mod tidy", stderr=subprocess.STDOUT, env=runenv,
shell=True, cwd=srctree)
File "/home/uvv/projects/poky/bitbake/lib/bb/process.py", line 189,
in run
raise ExecutionError(cmd, pipe.returncode, stdout, stderr)
bb.process.ExecutionError: Execution of 'go mod tidy' failed with exit
code 1:
go: finding module for package google.golang.org/grpc/metadata
go: finding module for package google.golang.org/grpc
go: finding module for package github.com/golang/protobuf/protoc-gen-go
go: finding module for package golang.org/x/net/context
go: finding module for package google.golang.org/grpc/codes
go: finding module for package google.golang.org/grpc/grpclog
go: downloading golang.org/x/net v0.4.0
go: downloading google.golang.org/grpc v1.51.0
go: downloading github.com/golang/protobuf v1.5.2
go: finding module for package github.com/cenkalti/backoff/v4
go: downloading github.com/cenkalti/backoff/v4 v4.2.0
go: finding module for package github.com/desertbit/timer
go: downloading github.com/cenkalti/backoff v2.2.1+incompatible
go: finding module for package github.com/rs/cors
go: finding module for package golang.org/x/net/http2
go: finding module for package google.golang.org/grpc/health/grpc_health_v1
go: finding module for package google.golang.org/grpc/status
go: finding module for package nhooyr.io/websocket
go: downloading github.com/desertbit/timer
v0.0.0-20180107155436-c41aec40b27f
go: finding module for package github.com/grpc-ecosystem/go-grpc-middleware
go: finding module for package
github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus
go: downloading github.com/rs/cors v1.8.3
go: finding module for package github.com/grpc-ecosystem/go-grpc-prometheus
go: finding module for package github.com/mwitkow/go-conntrack
go: downloading nhooyr.io/websocket v1.8.7
go: finding module for package github.com/mwitkow/go-conntrack/connhelpers
go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
go: finding module for package github.com/mwitkow/grpc-proxy/proxy
go: finding module for package
github.com/prometheus/client_golang/prometheus/promhttp
go: downloading github.com/mwitkow/go-conntrack
v0.0.0-20190716064945-2f068394615f
go: finding module for package github.com/sirupsen/logrus
go: finding module for package github.com/spf13/pflag
go: finding module for package golang.org/x/net/trace
go: finding module for package google.golang.org/grpc/credentials
go: finding module for package github.com/golang/protobuf/ptypes/empty
go: downloading github.com/mwitkow/grpc-proxy
v0.0.0-20220126150247-db34e7bfee32
go: finding module for package github.com/golang/protobuf/proto
go: downloading github.com/spf13/pflag v1.0.5
go: downloading github.com/sirupsen/logrus v1.9.0
go: finding module for package github.com/stretchr/testify/assert
go: finding module for package github.com/stretchr/testify/require
go: finding module for package github.com/stretchr/testify/suite
go: finding module for package google.golang.org/grpc/health
go: finding module for package
google.golang.org/protobuf/reflect/protoreflect
go: downloading github.com/prometheus/client_golang v1.14.0
go: downloading github.com/stretchr/testify v1.8.1
go: finding module for package google.golang.org/protobuf/runtime/protoimpl
go: finding module for package
google.golang.org/protobuf/types/known/emptypb
go: downloading google.golang.org/protobuf v1.28.1
go: found github.com/golang/protobuf/protoc-gen-go in
github.com/golang/protobuf v1.5.2
go: found golang.org/x/net/context in golang.org/x/net v0.4.0
go: found google.golang.org/grpc in google.golang.org/grpc v1.51.0
go: found google.golang.org/grpc/codes in
Re: [OE-core] [langdale][kirkstone][PATCH] go-mod.bbclass: Allow network in do_compile
Martin, I did see that patch series, but I was not sure about its state,
therefore I didn't try it. If I understood correctly, that partially
solves the issue, but the proper solution should involve the fetcher,
that's why it's not yet merged to master. Am I missing something?
Regards,
Slava
On 22.12.2022 20:08, Martin Jansa wrote:
Have you tried the changes submitted by Lukas/Stefan in:
https://lists.openembedded.org/g/openembedded-architecture/message/1539
?
It's not perfect, I was testing it on
https://github.com/influxdata/telegraf/blob/master/go.mod and I've
found some corner cases where it failed.
But it seems like very good start and we should work with Lukas/Stefan
to get it merged in master. Then all branches could consume recipes
created in master and only the exceptions would need to have network
access (instead of the bbclass enabling it for every go-mod user -
go-vendor inherits go-mod as well).
Lukas/Stefan: is there something we can do to help with v2 of your
patches?
I wanted to submit better review on submitted patches after more
debugging, but here it is:
1) to resolve 2 deps:
{'Path': 'gonum.org/v1/gonum ', 'Version':
'v0.12.0'} due to 404 "ERROR: Error while fetching redirect page: HTTP
Error 404: Not Found"
{'Path': 'code.cloudfoundry.org/clock
', 'Version': 'v1.0.0',
'Indirect': True} due to this being redirect to github.com
and failing with "ERROR: Error while fetching
redirect page: certificate verify failed: Hostname mismatch, certificate is not valid
for 'code.cloudfoundry.org '. (_ssl.c:996)>"
parse
2) some issues with LICENSEs:
INFO: Please add the following line for
'docs/LICENSE_OF_DEPENDENCIES.md' to a 'lib/recipetool/licenses.csv'
and replace `Unknown` with the license:
c4807c5b058a56c02f592207b4e056b1,Unknown
Traceback (most recent call last):
File "/OE/build/oe-core/openembedded-core/scripts/recipetool", line
111, in
ret = main()
^^
File "/OE/build/oe-core/openembedded-core/scripts/recipetool", line
100, in main
ret = args.func(args)
^^^
File
"/OE/layers/openembedded-core/scripts/lib/recipetool/create.py", line
746, in create_recipe
handler.process(srctree_use, classes, lines_before, lines_after,
handled, extravalues)
File
"/OE/layers/openembedded-core/scripts/lib/recipetool/create_go.py",
line 349, in process
self._rewrite_lic_uri(lines_before)
File
"/OE/layers/openembedded-core/scripts/lib/recipetool/create_go.py",
line 380, in _rewrite_lic_uri
updated, newlines = bb.utils.edit_metadata(lines_before,
['LIC_FILES_CHKSUM'], varfunc)
^^^
File "/OE/build/oe-core/bitbake/lib/bb/utils.py", line 1382, in
edit_metadata
if handle_var_end():
File "/OE/build/oe-core/bitbake/lib/bb/utils.py", line 1280, in
handle_var_end
(newvalue, newop, indent, minbreak) = varfunc(in_var, full_value,
op, newlines)
^
File
"/OE/layers/openembedded-core/scripts/lib/recipetool/create_go.py",
line 372, in varfunc
uri, chksum = license.split(';', 1)
^^^
ValueError: not enough values to unpack (expected 2, got 1)
(the license was just '/')
3) some SRCREVs set to None leading to parsing failure:
File "/OE/layers/bitbake/lib/bb/fetch2/git.py", line 762, in
_latest_revision
raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git
repository in git ls-remote output for %s" % \
bb.data_smart.ExpansionError: Failure expanding variable SRCPV,
expression was ${@bb.fetch2.get_srcrev(d)} which triggered exception
FetchError: Fetcher failure: Unable to resolve 'None' in upstream git
repository in git ls-remote output for
github.com/googleapis/google-cloud-go
The variable dependency chain for the failure is: SRCPV -> PV -> BP ->
FILESPATH
ERROR: Parsing newly created recipe failed, moving recipe to
/OE/build/oe-core/workspace/recipes/telegraf/telegraf_git.bb.parsefailed
for reference. If this looks to be caused by the recipe itself, please
report this error.
It takes long time to fetch all telegraf dependencies, so I'll create
some test recipe to more easily debug these 3 issues in isolation.
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#175016):
https://lists.openembedded.org/g/openembedded-core/message/175016
Mute This Topic: https://lists.openembedded.org/mt/95827446/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [langdale][kirkstone][PATCH] go-mod.bbclass: Allow network in do_compile
On 22.12.2022 17:48, Bruce Ashfield wrote: On Thu, Dec 22, 2022 at 11:20 AM Richard Purdie wrote: This patch is not in master and is not a backport therefore not eligible for kirkstone/langdale. My bad, the intention was to have it in all three branches: master, kirkstone, and langdale. 'nor should it go to master. If someone wants to allow go to fetch sources during builds, it needs to be done in their own layers. Bruce Probably I misunderstood the outcome of the discussion in my first link. Bruce and Richard, you both suggested to have it in go-mod with the warning. I agree that it breaks reproducibility, and is considered a bad practice in general (I'm aware of a few more build systems with the same shortcoming). My point is, until a proper solution is in place, it should be at least documented somewhere that this should be done in own layer/recipe. Any suggestions where this can be documented if not in OE-Core? Vyacheslav -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#174970): https://lists.openembedded.org/g/openembedded-core/message/174970 Mute This Topic: https://lists.openembedded.org/mt/95827446/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [langdale][kirkstone][PATCH] go-mod.bbclass: Allow network in do_compile
This is a workaround to allow using go-mod until a proper solution is in
place. Possible solutions were discussued ealier this year in these
threads:
https://lists.openembedded.org/g/openembedded-core/topic/89464905
https://lists.openembedded.org/g/openembedded-architecture/topic/90782883
Signed-off-by: Vyacheslav Yurkov
---
meta/classes-recipe/go-mod.bbclass | 5 +
1 file changed, 5 insertions(+)
diff --git a/meta/classes-recipe/go-mod.bbclass
b/meta/classes-recipe/go-mod.bbclass
index 61571596bc..da9a583288 100644
--- a/meta/classes-recipe/go-mod.bbclass
+++ b/meta/classes-recipe/go-mod.bbclass
@@ -24,3 +24,8 @@ inherit go
GO_WORKDIR ?= "${GO_IMPORT}"
do_compile[dirs] += "${B}/src/${GO_WORKDIR}"
+do_compile[network] = "1"
+
+do_compile:prepend() {
+bbwarn "Using go-mod enables network during compilation to be able to
fetch dependencies. This breaks reproducibility of the build!"
+}
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#174966):
https://lists.openembedded.org/g/openembedded-core/message/174966
Mute This Topic: https://lists.openembedded.org/mt/95827446/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [kirkstone][PATCH] overlayfs: Allow not used mount points
The patch is now in master. Just a reminder to queue it to kirkstone as
well.
Thanks,
Vyacheslav
On 12.10.2022 18:40, Steve Sakoman wrote:
On Tue, Oct 11, 2022 at 7:23 AM Vyacheslav Yurkov wrote:
Just checking if something is missing to include this fix to kirkstone.
Yes, it needs to make it into the master branch first :-) I will then
cherry-pick for langdale and kirstone.
Richard is on holiday until 24 Oct, so it will occur shortly
thereafter. It is already queued for the master branch, so there is
nothing else you need to do!
Steve
On 05.10.2022 13:17, Vyacheslav Yurkov via lists.openembedded.org wrote:
From: Vyacheslav Yurkov
When machine configuration defines a mount point, which is not used in
any recipe, allow to fall through and only report a note in the logs.
This can be expected behavior, when a mount point is defined for several
machines, but not used in all of them
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/overlayfs.bbclass | 6 +-
meta/lib/oe/overlayfs.py | 6 +-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass
index f7069edd41..c3564b6ec1 100644
--- a/meta/classes/overlayfs.bbclass
+++ b/meta/classes/overlayfs.bbclass
@@ -96,7 +96,11 @@ python do_create_overlayfs_units() {
overlayMountPoints = d.getVarFlags("OVERLAYFS_MOUNT_POINT")
for mountPoint in overlayMountPoints:
bb.debug(1, "Process variable flag %s" % mountPoint)
-for lower in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS',
mountPoint).split():
+lowerList = d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint)
+if not lowerList:
+bb.note("No mount points defined for %s flag, skipping" %
(mountPoint))
+continue
+for lower in lowerList.split():
bb.debug(1, "Prepare mount unit for %s with data mount point %s"
%
(lower, d.getVarFlag('OVERLAYFS_MOUNT_POINT',
mountPoint)))
prepareUnits(d.getVarFlag('OVERLAYFS_MOUNT_POINT', mountPoint),
lower)
diff --git a/meta/lib/oe/overlayfs.py b/meta/lib/oe/overlayfs.py
index b5d5e88e80..590c0de58a 100644
--- a/meta/lib/oe/overlayfs.py
+++ b/meta/lib/oe/overlayfs.py
@@ -38,7 +38,11 @@ def unitFileList(d):
bb.fatal("Missing required mount point for OVERLAYFS_MOUNT_POINT[%s]
in your MACHINE configuration" % mountPoint)
for mountPoint in overlayMountPoints:
-for path in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS',
mountPoint).split():
+mountPointList = d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint)
+if not mountPointList:
+bb.debug(1, "No mount points defined for %s flag, don't add to file
list", mountPoint)
+continue
+for path in mountPointList.split():
fileList.append(mountUnitName(path))
fileList.append(helperUnitName(path))
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#172326):
https://lists.openembedded.org/g/openembedded-core/message/172326
Mute This Topic: https://lists.openembedded.org/mt/94132926/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [kirkstone][PATCH 1/2] files: overlayfs-etc: refactor preinit template
From: Vyacheslav Yurkov
Signed-off-by: Vyacheslav Yurkov
Signed-off-by: Alexandre Belloni
Signed-off-by: Richard Purdie
---
meta/files/overlayfs-etc-preinit.sh.in | 14 +-
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/meta/files/overlayfs-etc-preinit.sh.in
b/meta/files/overlayfs-etc-preinit.sh.in
index 43c9b04eb9..0e80849f12 100644
--- a/meta/files/overlayfs-etc-preinit.sh.in
+++ b/meta/files/overlayfs-etc-preinit.sh.in
@@ -15,19 +15,23 @@ mount -t sysfs sysfs /sys
[ -z "$CONSOLE" ] && CONSOLE="/dev/console"
+BASE_OVERLAY_ETC_DIR={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc
+UPPER_DIR=$BASE_OVERLAY_ETC_DIR/upper
+WORK_DIR=$BASE_OVERLAY_ETC_DIR/work
+
mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}
if mount -n -t {OVERLAYFS_ETC_FSTYPE} \
-o {OVERLAYFS_ETC_MOUNT_OPTIONS} \
{OVERLAYFS_ETC_DEVICE} {OVERLAYFS_ETC_MOUNT_POINT}
then
-mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper
-mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/work
+mkdir -p $UPPER_DIR
+mkdir -p $WORK_DIR
mount -n -t overlay \
--o upperdir={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper \
+-o upperdir=$UPPER_DIR \
-o lowerdir=/etc \
--o workdir={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/work \
+-o workdir=$WORK_DIR \
-o index=off,xino=off,redirect_dir=off,metacopy=off \
-{OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper /etc || \
+$UPPER_DIR /etc || \
echo "PREINIT: Mounting etc-overlay failed!"
else
echo "PREINIT: Mounting failed!"
--
2.30.2
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#171960):
https://lists.openembedded.org/g/openembedded-core/message/171960
Mute This Topic: https://lists.openembedded.org/mt/94425835/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [kirkstone][PATCH 2/2] classes: files: Extend overlayfs-etc class
From: Vyacheslav Yurkov
Add the ability to expose the lower layer of /etc when mounting overlay.
This is the similar to what overlayroot script from initramfs-framework does.
By default, this option is turned off to keep an old behavior intact.
Signed-off-by: Vyacheslav Yurkov
Signed-off-by: Alexandre Belloni
Signed-off-by: Richard Purdie
---
meta/classes/overlayfs-etc.bbclass | 5 -
meta/files/overlayfs-etc-preinit.sh.in | 9 +
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/meta/classes/overlayfs-etc.bbclass
b/meta/classes/overlayfs-etc.bbclass
index 91afee695c..40116e4c6e 100644
--- a/meta/classes/overlayfs-etc.bbclass
+++ b/meta/classes/overlayfs-etc.bbclass
@@ -34,6 +34,7 @@ OVERLAYFS_ETC_DEVICE ??= ""
OVERLAYFS_ETC_USE_ORIG_INIT_NAME ??= "1"
OVERLAYFS_ETC_MOUNT_OPTIONS ??= "defaults"
OVERLAYFS_ETC_INIT_TEMPLATE ??=
"${COREBASE}/meta/files/overlayfs-etc-preinit.sh.in"
+OVERLAYFS_ETC_EXPOSE_LOWER ??= "0"
python create_overlayfs_etc_preinit() {
overlayEtcMountPoint = d.getVar("OVERLAYFS_ETC_MOUNT_POINT")
@@ -54,13 +55,15 @@ python create_overlayfs_etc_preinit() {
preinitPath = oe.path.join(d.getVar("IMAGE_ROOTFS"),
d.getVar("base_sbindir"), "preinit")
initBaseName = oe.path.join(d.getVar("base_sbindir"), "init")
origInitNameSuffix = ".orig"
+exposeLower = oe.types.boolean(d.getVar('OVERLAYFS_ETC_EXPOSE_LOWER'))
args = {
'OVERLAYFS_ETC_MOUNT_POINT': overlayEtcMountPoint,
'OVERLAYFS_ETC_MOUNT_OPTIONS': d.getVar('OVERLAYFS_ETC_MOUNT_OPTIONS'),
'OVERLAYFS_ETC_FSTYPE': overlayEtcFsType,
'OVERLAYFS_ETC_DEVICE': overlayEtcDevice,
-'SBIN_INIT_NAME': initBaseName + origInitNameSuffix if useOrigInit
else initBaseName
+'SBIN_INIT_NAME': initBaseName + origInitNameSuffix if useOrigInit
else initBaseName,
+'OVERLAYFS_ETC_EXPOSE_LOWER': "true" if exposeLower else "false"
}
if useOrigInit:
diff --git a/meta/files/overlayfs-etc-preinit.sh.in
b/meta/files/overlayfs-etc-preinit.sh.in
index 0e80849f12..8db076f4ba 100644
--- a/meta/files/overlayfs-etc-preinit.sh.in
+++ b/meta/files/overlayfs-etc-preinit.sh.in
@@ -18,6 +18,7 @@ mount -t sysfs sysfs /sys
BASE_OVERLAY_ETC_DIR={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc
UPPER_DIR=$BASE_OVERLAY_ETC_DIR/upper
WORK_DIR=$BASE_OVERLAY_ETC_DIR/work
+LOWER_DIR=$BASE_OVERLAY_ETC_DIR/lower
mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}
if mount -n -t {OVERLAYFS_ETC_FSTYPE} \
@@ -26,6 +27,14 @@ if mount -n -t {OVERLAYFS_ETC_FSTYPE} \
then
mkdir -p $UPPER_DIR
mkdir -p $WORK_DIR
+
+if {OVERLAYFS_ETC_EXPOSE_LOWER}; then
+mkdir -p $LOWER_DIR
+
+# provide read-only access to original /etc content
+mount -o bind,ro /etc $LOWER_DIR
+fi
+
mount -n -t overlay \
-o upperdir=$UPPER_DIR \
-o lowerdir=/etc \
--
2.30.2
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#171961):
https://lists.openembedded.org/g/openembedded-core/message/171961
Mute This Topic: https://lists.openembedded.org/mt/94425836/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [kirkstone][PATCH] overlayfs: Allow not used mount points
Just checking if something is missing to include this fix to kirkstone.
Thanks,
Vyacheslav
On 05.10.2022 13:17, Vyacheslav Yurkov via lists.openembedded.org wrote:
From: Vyacheslav Yurkov
When machine configuration defines a mount point, which is not used in
any recipe, allow to fall through and only report a note in the logs.
This can be expected behavior, when a mount point is defined for several
machines, but not used in all of them
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/overlayfs.bbclass | 6 +-
meta/lib/oe/overlayfs.py | 6 +-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass
index f7069edd41..c3564b6ec1 100644
--- a/meta/classes/overlayfs.bbclass
+++ b/meta/classes/overlayfs.bbclass
@@ -96,7 +96,11 @@ python do_create_overlayfs_units() {
overlayMountPoints = d.getVarFlags("OVERLAYFS_MOUNT_POINT")
for mountPoint in overlayMountPoints:
bb.debug(1, "Process variable flag %s" % mountPoint)
-for lower in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS',
mountPoint).split():
+lowerList = d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint)
+if not lowerList:
+bb.note("No mount points defined for %s flag, skipping" %
(mountPoint))
+continue
+for lower in lowerList.split():
bb.debug(1, "Prepare mount unit for %s with data mount point %s" %
(lower, d.getVarFlag('OVERLAYFS_MOUNT_POINT',
mountPoint)))
prepareUnits(d.getVarFlag('OVERLAYFS_MOUNT_POINT', mountPoint),
lower)
diff --git a/meta/lib/oe/overlayfs.py b/meta/lib/oe/overlayfs.py
index b5d5e88e80..590c0de58a 100644
--- a/meta/lib/oe/overlayfs.py
+++ b/meta/lib/oe/overlayfs.py
@@ -38,7 +38,11 @@ def unitFileList(d):
bb.fatal("Missing required mount point for OVERLAYFS_MOUNT_POINT[%s]
in your MACHINE configuration" % mountPoint)
for mountPoint in overlayMountPoints:
-for path in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS',
mountPoint).split():
+mountPointList = d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint)
+if not mountPointList:
+bb.debug(1, "No mount points defined for %s flag, don't add to file
list", mountPoint)
+continue
+for path in mountPointList.split():
fileList.append(mountUnitName(path))
fileList.append(helperUnitName(path))
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#171634):
https://lists.openembedded.org/g/openembedded-core/message/171634
Mute This Topic: https://lists.openembedded.org/mt/94132926/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [kirkstone][PATCH] overlayfs: Allow not used mount points
From: Vyacheslav Yurkov
When machine configuration defines a mount point, which is not used in
any recipe, allow to fall through and only report a note in the logs.
This can be expected behavior, when a mount point is defined for several
machines, but not used in all of them
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/overlayfs.bbclass | 6 +-
meta/lib/oe/overlayfs.py | 6 +-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass
index f7069edd41..c3564b6ec1 100644
--- a/meta/classes/overlayfs.bbclass
+++ b/meta/classes/overlayfs.bbclass
@@ -96,7 +96,11 @@ python do_create_overlayfs_units() {
overlayMountPoints = d.getVarFlags("OVERLAYFS_MOUNT_POINT")
for mountPoint in overlayMountPoints:
bb.debug(1, "Process variable flag %s" % mountPoint)
-for lower in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS',
mountPoint).split():
+lowerList = d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint)
+if not lowerList:
+bb.note("No mount points defined for %s flag, skipping" %
(mountPoint))
+continue
+for lower in lowerList.split():
bb.debug(1, "Prepare mount unit for %s with data mount point %s" %
(lower, d.getVarFlag('OVERLAYFS_MOUNT_POINT',
mountPoint)))
prepareUnits(d.getVarFlag('OVERLAYFS_MOUNT_POINT', mountPoint),
lower)
diff --git a/meta/lib/oe/overlayfs.py b/meta/lib/oe/overlayfs.py
index b5d5e88e80..590c0de58a 100644
--- a/meta/lib/oe/overlayfs.py
+++ b/meta/lib/oe/overlayfs.py
@@ -38,7 +38,11 @@ def unitFileList(d):
bb.fatal("Missing required mount point for
OVERLAYFS_MOUNT_POINT[%s] in your MACHINE configuration" % mountPoint)
for mountPoint in overlayMountPoints:
-for path in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS',
mountPoint).split():
+mountPointList = d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint)
+if not mountPointList:
+bb.debug(1, "No mount points defined for %s flag, don't add to
file list", mountPoint)
+continue
+for path in mountPointList.split():
fileList.append(mountUnitName(path))
fileList.append(helperUnitName(path))
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#171457):
https://lists.openembedded.org/g/openembedded-core/message/171457
Mute This Topic: https://lists.openembedded.org/mt/94132926/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] overlayfs: Allow not used mount points
From: Vyacheslav Yurkov
When machine configuration defines a mount point, which is not used in
any recipe, allow to fall through and only report a note in the logs.
This can be expected behavior, when a mount point is defined for several
machines, but not used in all of them
Signed-off-by: Vyacheslav Yurkov
---
meta/classes-recipe/overlayfs.bbclass | 6 +-
meta/lib/oe/overlayfs.py | 6 +-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/meta/classes-recipe/overlayfs.bbclass
b/meta/classes-recipe/overlayfs.bbclass
index bdc6dd9d57..53d65d7531 100644
--- a/meta/classes-recipe/overlayfs.bbclass
+++ b/meta/classes-recipe/overlayfs.bbclass
@@ -102,7 +102,11 @@ python do_create_overlayfs_units() {
overlayMountPoints = d.getVarFlags("OVERLAYFS_MOUNT_POINT")
for mountPoint in overlayMountPoints:
bb.debug(1, "Process variable flag %s" % mountPoint)
-for lower in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS',
mountPoint).split():
+lowerList = d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint)
+if not lowerList:
+bb.note("No mount points defined for %s flag, skipping" %
(mountPoint))
+continue
+for lower in lowerList.split():
bb.debug(1, "Prepare mount unit for %s with data mount point %s" %
(lower, d.getVarFlag('OVERLAYFS_MOUNT_POINT',
mountPoint)))
prepareUnits(d.getVarFlag('OVERLAYFS_MOUNT_POINT', mountPoint),
lower)
diff --git a/meta/lib/oe/overlayfs.py b/meta/lib/oe/overlayfs.py
index 8d7a047125..8b88900f71 100644
--- a/meta/lib/oe/overlayfs.py
+++ b/meta/lib/oe/overlayfs.py
@@ -40,7 +40,11 @@ def unitFileList(d):
bb.fatal("Missing required mount point for
OVERLAYFS_MOUNT_POINT[%s] in your MACHINE configuration" % mountPoint)
for mountPoint in overlayMountPoints:
-for path in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS',
mountPoint).split():
+mountPointList = d.getVarFlag('OVERLAYFS_WRITABLE_PATHS', mountPoint)
+if not mountPointList:
+bb.debug(1, "No mount points defined for %s flag, don't add to
file list", mountPoint)
+continue
+for path in mountPointList.split():
fileList.append(mountUnitName(path))
fileList.append(helperUnitName(path))
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#171456):
https://lists.openembedded.org/g/openembedded-core/message/171456
Mute This Topic: https://lists.openembedded.org/mt/94132869/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 5/5] oeqa/selftest: Add lower layer test for overlayfs-etc
From: Vyacheslav Yurkov
Place a test file on the /etc by means of overlayfs-user recipe.
Perform QA checks to make sure that:
- When lower layer is exposed, that it's read-only to avoid undefined behavior
- By default lower layer is not exposed
Signed-off-by: Vyacheslav Yurkov
---
.../overlayfs-user/overlayfs-user.bb | 7 +-
meta/lib/oeqa/selftest/cases/overlayfs.py | 90 ++-
2 files changed, 74 insertions(+), 23 deletions(-)
diff --git a/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb
b/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb
index 913a4d1fdb..50cba9514b 100644
--- a/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb
+++ b/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb
@@ -12,6 +12,11 @@ OVERLAYFS_WRITABLE_PATHS[mnt-overlay] +=
"/usr/share/my-application"
do_install() {
install -d ${D}/usr/share/my-application
+install -d ${D}${sysconfdir}
+echo "Original file in /etc" >> ${D}${sysconfdir}/lower-layer-test.txt
}
-FILES:${PN} += "/usr"
+FILES:${PN} += "\
+${exec_prefix} \
+${sysconfdir \
+"
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index f550015b4b..57a8c8bdb6 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -381,28 +381,7 @@ OVERLAYFS_ETC_DEVICE = "/dev/sda3"
Author:Vyacheslav Yurkov
"""
-config = """
-DISTRO_FEATURES:append = " systemd"
-
-# Use systemd as init manager
-VIRTUAL-RUNTIME_init_manager = "systemd"
-
-# enable overlayfs in the kernel
-KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
-
-IMAGE_FSTYPES += "wic"
-OVERLAYFS_INIT_OPTION = "{OVERLAYFS_INIT_OPTION}"
-WKS_FILE = "overlayfs_etc.wks.in"
-
-EXTRA_IMAGE_FEATURES += "read-only-rootfs"
-# Image configuration for overlayfs-etc
-EXTRA_IMAGE_FEATURES += "overlayfs-etc"
-IMAGE_FEATURES:remove = "package-management"
-OVERLAYFS_ETC_MOUNT_POINT = "/data"
-OVERLAYFS_ETC_FSTYPE = "ext4"
-OVERLAYFS_ETC_DEVICE = "/dev/sda3"
-OVERLAYFS_ETC_USE_ORIG_INIT_NAME = "{OVERLAYFS_ETC_USE_ORIG_INIT_NAME}"
-"""
+config = self.get_working_config()
args = {
'OVERLAYFS_INIT_OPTION': "" if origInit else "init=/sbin/preinit",
@@ -423,6 +402,11 @@ OVERLAYFS_ETC_USE_ORIG_INIT_NAME =
"{OVERLAYFS_ETC_USE_ORIG_INIT_NAME}"
line = getline_qemu(output, "upperdir=/data/overlay-etc/upper")
self.assertTrue(line and line.startswith("/data/overlay-etc/upper
on /etc type overlay"), msg=output)
+# check that lower layer is not available
+status, output = qemu.run_serial("ls -1 /data/overlay-etc/lower")
+line = getline_qemu(output, "No such file or directory")
+self.assertTrue(line, msg=output)
+
status, output = qemu.run_serial("touch " + testFile)
status, output = qemu.run_serial("sync")
status, output = qemu.run_serial("ls -1 " + testFile)
@@ -434,3 +418,65 @@ OVERLAYFS_ETC_USE_ORIG_INIT_NAME =
"{OVERLAYFS_ETC_USE_ORIG_INIT_NAME}"
status, output = qemu.run_serial("ls -1 " + testFile)
line = getline_qemu(output, testFile)
self.assertTrue(line and line.startswith(testFile), msg=output)
+
+def test_lower_layer_access(self):
+"""
+Summary: Test that lower layer of /etc is available read-only when
configured
+Expected: Can't write to lower layer. The files on lower and upper
different after
+ modification
+Author:Vyacheslav Yurkov
+"""
+
+config = self.get_working_config()
+
+configLower = """
+OVERLAYFS_ETC_EXPOSE_LOWER = "1"
+IMAGE_INSTALL:append = " overlayfs-user"
+"""
+testFile = "lower-layer-test.txt"
+
+args = {
+'OVERLAYFS_INIT_OPTION': "",
+'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': 1
+}
+
+self.write_config(config.format(**args))
+
+self.append_config(configLower)
+bitbake('core-image-minimal')
+
+with runqemu('core-image-minimal', image_fstype='wic') as qemu:
+status, output = qemu.run_serial("echo \"Modified in upper\" >
/etc/" + testFile)
+status, output = qemu.run_serial("diff /etc/" + testFile + "
/data/overlay-etc/lower/" + testFile)
+line = getline_qemu(output, "Modified in upper")
+self.asse
[OE-core] [PATCH 4/5] classes: files: Extend overlayfs-etc class
From: Vyacheslav Yurkov
Add the ability to expose the lower layer of /etc when mounting overlay.
This is the similar to what overlayroot script from initramfs-framework does.
By default, this option is turned off to keep an old behavior intact.
Signed-off-by: Vyacheslav Yurkov
---
meta/classes-recipe/overlayfs-etc.bbclass | 5 -
meta/files/overlayfs-etc-preinit.sh.in| 9 +
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/meta/classes-recipe/overlayfs-etc.bbclass
b/meta/classes-recipe/overlayfs-etc.bbclass
index d0bc3ecfac..f8343106f3 100644
--- a/meta/classes-recipe/overlayfs-etc.bbclass
+++ b/meta/classes-recipe/overlayfs-etc.bbclass
@@ -40,6 +40,7 @@ OVERLAYFS_ETC_DEVICE ??= ""
OVERLAYFS_ETC_USE_ORIG_INIT_NAME ??= "1"
OVERLAYFS_ETC_MOUNT_OPTIONS ??= "defaults"
OVERLAYFS_ETC_INIT_TEMPLATE ??=
"${COREBASE}/meta/files/overlayfs-etc-preinit.sh.in"
+OVERLAYFS_ETC_EXPOSE_LOWER ??= "0"
python create_overlayfs_etc_preinit() {
overlayEtcMountPoint = d.getVar("OVERLAYFS_ETC_MOUNT_POINT")
@@ -60,13 +61,15 @@ python create_overlayfs_etc_preinit() {
preinitPath = oe.path.join(d.getVar("IMAGE_ROOTFS"),
d.getVar("base_sbindir"), "preinit")
initBaseName = oe.path.join(d.getVar("base_sbindir"), "init")
origInitNameSuffix = ".orig"
+exposeLower = oe.types.boolean(d.getVar('OVERLAYFS_ETC_EXPOSE_LOWER'))
args = {
'OVERLAYFS_ETC_MOUNT_POINT': overlayEtcMountPoint,
'OVERLAYFS_ETC_MOUNT_OPTIONS': d.getVar('OVERLAYFS_ETC_MOUNT_OPTIONS'),
'OVERLAYFS_ETC_FSTYPE': overlayEtcFsType,
'OVERLAYFS_ETC_DEVICE': overlayEtcDevice,
-'SBIN_INIT_NAME': initBaseName + origInitNameSuffix if useOrigInit
else initBaseName
+'SBIN_INIT_NAME': initBaseName + origInitNameSuffix if useOrigInit
else initBaseName,
+'OVERLAYFS_ETC_EXPOSE_LOWER': "true" if exposeLower else "false"
}
if useOrigInit:
diff --git a/meta/files/overlayfs-etc-preinit.sh.in
b/meta/files/overlayfs-etc-preinit.sh.in
index 0e80849f12..8db076f4ba 100644
--- a/meta/files/overlayfs-etc-preinit.sh.in
+++ b/meta/files/overlayfs-etc-preinit.sh.in
@@ -18,6 +18,7 @@ mount -t sysfs sysfs /sys
BASE_OVERLAY_ETC_DIR={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc
UPPER_DIR=$BASE_OVERLAY_ETC_DIR/upper
WORK_DIR=$BASE_OVERLAY_ETC_DIR/work
+LOWER_DIR=$BASE_OVERLAY_ETC_DIR/lower
mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}
if mount -n -t {OVERLAYFS_ETC_FSTYPE} \
@@ -26,6 +27,14 @@ if mount -n -t {OVERLAYFS_ETC_FSTYPE} \
then
mkdir -p $UPPER_DIR
mkdir -p $WORK_DIR
+
+if {OVERLAYFS_ETC_EXPOSE_LOWER}; then
+mkdir -p $LOWER_DIR
+
+# provide read-only access to original /etc content
+mount -o bind,ro /etc $LOWER_DIR
+fi
+
mount -n -t overlay \
-o upperdir=$UPPER_DIR \
-o lowerdir=/etc \
--
2.30.2
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#170429):
https://lists.openembedded.org/g/openembedded-core/message/170429
Mute This Topic: https://lists.openembedded.org/mt/93533202/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 3/5] files: overlayfs-etc: refactor preinit template
From: Vyacheslav Yurkov
Signed-off-by: Vyacheslav Yurkov
---
meta/files/overlayfs-etc-preinit.sh.in | 14 +-
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/meta/files/overlayfs-etc-preinit.sh.in
b/meta/files/overlayfs-etc-preinit.sh.in
index 43c9b04eb9..0e80849f12 100644
--- a/meta/files/overlayfs-etc-preinit.sh.in
+++ b/meta/files/overlayfs-etc-preinit.sh.in
@@ -15,19 +15,23 @@ mount -t sysfs sysfs /sys
[ -z "$CONSOLE" ] && CONSOLE="/dev/console"
+BASE_OVERLAY_ETC_DIR={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc
+UPPER_DIR=$BASE_OVERLAY_ETC_DIR/upper
+WORK_DIR=$BASE_OVERLAY_ETC_DIR/work
+
mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}
if mount -n -t {OVERLAYFS_ETC_FSTYPE} \
-o {OVERLAYFS_ETC_MOUNT_OPTIONS} \
{OVERLAYFS_ETC_DEVICE} {OVERLAYFS_ETC_MOUNT_POINT}
then
-mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper
-mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/work
+mkdir -p $UPPER_DIR
+mkdir -p $WORK_DIR
mount -n -t overlay \
--o upperdir={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper \
+-o upperdir=$UPPER_DIR \
-o lowerdir=/etc \
--o workdir={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/work \
+-o workdir=$WORK_DIR \
-o index=off,xino=off,redirect_dir=off,metacopy=off \
-{OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper /etc || \
+$UPPER_DIR /etc || \
echo "PREINIT: Mounting etc-overlay failed!"
else
echo "PREINIT: Mounting failed!"
--
2.30.2
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#170428):
https://lists.openembedded.org/g/openembedded-core/message/170428
Mute This Topic: https://lists.openembedded.org/mt/93533201/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/5] oeqa/selftest: drop image_feature test from overlayfs
From: Vyacheslav Yurkov
The test checked the incorrect class use with INHERIT. This
functionality is now covered by bitbake
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/overlayfs.py | 13 +
1 file changed, 1 insertion(+), 12 deletions(-)
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index bff22f21b6..f550015b4b 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -333,24 +333,14 @@ EXTRA_IMAGE_FEATURES += "package-management"
self.assertTrue("overlayfs-etc" in res.output, msg=res.output)
self.assertTrue("package-management" in res.output, msg=res.output)
-def test_image_feature_is_missing_class_included(self):
-configAppend = """
-INHERIT += "overlayfs-etc"
-"""
-self.run_check_image_feature(configAppend)
-
def test_image_feature_is_missing(self):
-self.run_check_image_feature()
-
-def run_check_image_feature(self, appendToConfig=""):
"""
Summary: Overlayfs-etc class is not applied when image feature is
not set
- even if we inherit it directly,
Expected: Image is created successfully but /etc is not an overlay
Author:Vyacheslav Yurkov
"""
-config = f"""
+config = """
DISTRO_FEATURES:append = " systemd"
# Use systemd as init manager
@@ -366,7 +356,6 @@ EXTRA_IMAGE_FEATURES += "read-only-rootfs"
# Image configuration for overlayfs-etc
OVERLAYFS_ETC_MOUNT_POINT = "/data"
OVERLAYFS_ETC_DEVICE = "/dev/sda3"
-{appendToConfig}
"""
self.write_config(config)
--
2.30.2
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#170426):
https://lists.openembedded.org/g/openembedded-core/message/170426
Mute This Topic: https://lists.openembedded.org/mt/93533199/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/5] classes: Update overlayfs classes to use new bitbake functionality
From: Vyacheslav Yurkov
OverlayFS classes belong to a recipe scope
Signed-off-by: Vyacheslav Yurkov
---
meta/{classes => classes-recipe}/overlayfs-etc.bbclass | 0
meta/{classes => classes-recipe}/overlayfs.bbclass | 0
2 files changed, 0 insertions(+), 0 deletions(-)
rename meta/{classes => classes-recipe}/overlayfs-etc.bbclass (100%)
rename meta/{classes => classes-recipe}/overlayfs.bbclass (100%)
diff --git a/meta/classes/overlayfs-etc.bbclass
b/meta/classes-recipe/overlayfs-etc.bbclass
similarity index 100%
rename from meta/classes/overlayfs-etc.bbclass
rename to meta/classes-recipe/overlayfs-etc.bbclass
diff --git a/meta/classes/overlayfs.bbclass
b/meta/classes-recipe/overlayfs.bbclass
similarity index 100%
rename from meta/classes/overlayfs.bbclass
rename to meta/classes-recipe/overlayfs.bbclass
--
2.30.2
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#170427):
https://lists.openembedded.org/g/openembedded-core/message/170427
Mute This Topic: https://lists.openembedded.org/mt/93533200/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/2] files: rootfs-postcommands: move helper commands to script
From: Vyacheslav Yurkov
OverlayFS systemd helper unit might require more pre-processing
commands. It gets more complicated to embed them in a unit file, because
systemd shell subset is limited and might require additional escaping.
Move the command to a separate script, thus simplifying systemd unit.
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/rootfs-postcommands.bbclass| 13 -
meta/files/overlayfs-create-dirs.service.in | 3 +--
meta/files/overlayfs-create-dirs.sh | 8
3 files changed, 21 insertions(+), 3 deletions(-)
create mode 100644 meta/files/overlayfs-create-dirs.sh
diff --git a/meta/classes/rootfs-postcommands.bbclass
b/meta/classes/rootfs-postcommands.bbclass
index d302c23cf4..3f9fdb602d 100644
--- a/meta/classes/rootfs-postcommands.bbclass
+++ b/meta/classes/rootfs-postcommands.bbclass
@@ -39,7 +39,7 @@ ROOTFS_POSTPROCESS_COMMAND +=
'${@bb.utils.contains("DISTRO_FEATURES", "systemd"
ROOTFS_POSTPROCESS_COMMAND += 'empty_var_volatile;'
-ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("DISTRO_FEATURES",
"overlayfs", "overlayfs_qa_check;", "", d)}'
+ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("DISTRO_FEATURES",
"overlayfs", "overlayfs_qa_check; overlayfs_postprocess;", "", d)}'
inherit image-artifact-names
@@ -422,3 +422,14 @@ python overlayfs_qa_check() {
if not allUnitExist:
bb.fatal('Not all mount paths and units are installed in the image')
}
+
+python overlayfs_postprocess() {
+import shutil
+
+# install helper script
+helperScriptName = "overlayfs-create-dirs.sh"
+helperScriptSource = oe.path.join(d.getVar("COREBASE"), "meta/files",
helperScriptName)
+helperScriptDest = oe.path.join(d.getVar("IMAGE_ROOTFS"), "/usr/sbin/",
helperScriptName)
+shutil.copyfile(helperScriptSource, helperScriptDest)
+os.chmod(helperScriptDest, 0o755)
+}
diff --git a/meta/files/overlayfs-create-dirs.service.in
b/meta/files/overlayfs-create-dirs.service.in
index 61b2b9321b..c949a6dc73 100644
--- a/meta/files/overlayfs-create-dirs.service.in
+++ b/meta/files/overlayfs-create-dirs.service.in
@@ -6,8 +6,7 @@ DefaultDependencies=no
[Service]
Type=oneshot
-ExecStart=mkdir -p {DATA_MOUNT_POINT}/upper{LOWERDIR}
-ExecStart=mkdir -p {DATA_MOUNT_POINT}/workdir{LOWERDIR}
+ExecStart=/usr/sbin/overlayfs-create-dirs.sh {LOWERDIR} {DATA_MOUNT_POINT}
RemainAfterExit=true
StandardOutput=journal
diff --git a/meta/files/overlayfs-create-dirs.sh
b/meta/files/overlayfs-create-dirs.sh
new file mode 100644
index 00..06c7587a6b
--- /dev/null
+++ b/meta/files/overlayfs-create-dirs.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+# This script is intended to be used sorely by overlayfs-create-dirs.service
+# Usage: overlayfs-create-dirs.sh
+
+lowerdir=$1
+datamountpoint=$2
+mkdir -p ${datamountpoint}/upper${lowerdir}
+mkdir -p ${datamountpoint}/workdir${lowerdir}
--
2.30.2
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#166439):
https://lists.openembedded.org/g/openembedded-core/message/166439
Mute This Topic: https://lists.openembedded.org/mt/91484585/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/2] files: respect overlayfs owner from lower layer
From: Vyacheslav Yurkov
Permission model of overlayfs uses permissions/ownership from the upper
layer after mounting. Fix up UID/GID of the upper layer, when lower
layer already uses something custom.
Signed-off-by: Vyacheslav Yurkov
---
meta/files/overlayfs-create-dirs.sh | 3 +++
1 file changed, 3 insertions(+)
diff --git a/meta/files/overlayfs-create-dirs.sh
b/meta/files/overlayfs-create-dirs.sh
index 06c7587a6b..9954c34352 100644
--- a/meta/files/overlayfs-create-dirs.sh
+++ b/meta/files/overlayfs-create-dirs.sh
@@ -6,3 +6,6 @@ lowerdir=$1
datamountpoint=$2
mkdir -p ${datamountpoint}/upper${lowerdir}
mkdir -p ${datamountpoint}/workdir${lowerdir}
+if [ -d "$lowerdir" ]; then
+chown $(stat -c "%U:%G" ${lowerdir}) ${datamountpoint}/upper${lowerdir}
+fi
--
2.30.2
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#166440):
https://lists.openembedded.org/g/openembedded-core/message/166440
Mute This Topic: https://lists.openembedded.org/mt/91484586/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH v3] classes: rootfs-postcommands: add skip option to overlayfs_qa_check
On 03.05.2022 10:22, Claudius Heine wrote:
The overlayfs_qa_check checks if the current root file system has a
mount configured for each overlayfs, when the overlayfs class is used.
However there are certain instances where this mount point is created at
runtime and not static in a fstab entry or systemd mount unit.
One such case would be if overlayfs-etc is used, where the device is
mounted in the preinit script and not via a mount unit or fstab entry.
However there are other possibilities for this as well, like startup
scripts that support a dynamic partition layout. For instance when
systemd-repart is used.
This adds the `OVERLAYFS_QA_SKIP` variable, which allows to define QA
skips via its flags. In principle it supports multiple QA skip flags
separated by whitespace, but only one (`mount-configured`) is
implemented here. To skip this QA check simply add `mount-configured` to
the flag of `OVERLAYFS_QA_SKIP` with the same name. For instance if a
overlayfs is configured as:
OVERLAYFS_MOUNT_POINT[data] = "/data"
Skipping this QA check can be done by setting:
OVERLAYFS_QA_SKIP[data] = "mount-configured"
Also add a testcase and fix a typo (fstat -> fstab).
Signed-off-by: Claudius Heine
---
Changes from v2:
- Fix typo 'fstat' -> 'fstab' in testcase
- Add testcase for successful build when QA check is skipped.
Changes from v1:
- `getVarFag` -> `getVarFlag`
---
meta/classes/rootfs-postcommands.bbclass | 10 --
meta/lib/oeqa/selftest/cases/overlayfs.py | 20 +++-
2 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/meta/classes/rootfs-postcommands.bbclass
b/meta/classes/rootfs-postcommands.bbclass
index 7b92df69c5..9b6824043c 100644
--- a/meta/classes/rootfs-postcommands.bbclass
+++ b/meta/classes/rootfs-postcommands.bbclass
@@ -398,6 +398,10 @@ python overlayfs_qa_check() {
allUnitExist = True;
for mountPoint in overlayMountPoints:
+qaSkip = (d.getVarFlag("OVERLAYFS_QA_SKIP", mountPoint) or "").split()
+if "mount-configured" in qaSkip:
+continue
+
mountPath = d.getVarFlag('OVERLAYFS_MOUNT_POINT', mountPoint)
if mountPath in fstabDevices:
continue
@@ -407,8 +411,10 @@ python overlayfs_qa_check() {
for dirpath in searchpaths):
continue
-bb.warn('Mount path %s not found in fstat and unit %s not found '
-'in systemd unit directories' % (mountPath, mountUnit))
+bb.warn(f'Mount path {mountPath} not found in fstab and unit '
+f'{mountUnit} not found in systemd unit directories.')
+bb.warn(f'Skip this check by setting OVERLAYFS_QA_SKIP[{mountPoint}] =
'
+'"mount-configured"')
allUnitExist = False;
if not allUnitExist:
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index 472746a64f..ce1d2f1ec3 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -62,11 +62,29 @@ DISTRO_FEATURES += "systemd overlayfs"
self.add_overlay_conf_to_machine()
res = bitbake('core-image-minimal', ignore_status=True)
-line = getline(res, " Mount path /mnt/overlay not found in fstat and unit
mnt-overlay.mount not found in systemd unit directories")
+line = getline(res, " Mount path /mnt/overlay not found in fstab and unit
mnt-overlay.mount not found in systemd unit directories")
self.assertTrue(line and line.startswith("WARNING:"), msg=res.output)
line = getline(res, "Not all mount paths and units are installed in the
image")
self.assertTrue(line and line.startswith("ERROR:"), msg=res.output)
+def test_not_all_units_installed_but_qa_skipped(self):
+"""
+Summary: Test skipping the QA check
+Expected: Image is created successfully
+Author:Claudius Heine
+"""
+
+config = """
+IMAGE_INSTALL:append = " overlayfs-user"
+DISTRO_FEATURES += "systemd overlayfs"
+OVERLAYFS_QA_SKIP[mnt-overlay] = "mount-configured"
+"""
+
+self.write_config(config)
+self.add_overlay_conf_to_machine()
+
+bitbake('core-image-minimal')
+
def test_mount_unit_not_set(self):
"""
Summary: Test whether mount unit was set properly
Hi Claudius,
Thanks a lot for the patch
Could you please also extend the header of overlayfs.bbclass to describe
the use case? I will update the documentation accordingly after that.
Thanks,
Vyacheslav
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#165384):
https://lists.openembedded.org/g/openembedded-core/message/165384
Mute This Topic: https://lists.openembedded.org/mt/90854996/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
Re: [OE-core] [PATCH v2 0/4] Patches and bugfixes for overlayfs
On 10.03.2022 14:13, Claudius Heine wrote:
Hi,
I bundled my patches to overlayfs and overlayfs-etc togehter in this
patchset and fixed the whitespace issues.
regards,
Claudius
--
Changes from v1:
- fixed whitespace issues: converted tabs to 4 spaces.
- changed git commit title of patch 4
Claudius Heine (4):
overlayfs-etc: add condition to package-management feature conflict
overlayfs: add systemd unit path prefix to FILES:${PN} array
files: overlayfs-etc: wrap long lines of preinit file
files: overlayfs-etc: add overlay mount options to preinit
meta/classes/overlayfs-etc.bbclass | 3 ++-
meta/classes/overlayfs.bbclass | 3 ++-
meta/files/overlayfs-etc-preinit.sh.in | 12 ++--
3 files changed, 14 insertions(+), 4 deletions(-)
Hi Claudius,
Thanks for the patches. It seems like the first 2 are already in master
branch or I'm missing something?
Cheers,
Vyacheslav
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#163029):
https://lists.openembedded.org/g/openembedded-core/message/163029
Mute This Topic: https://lists.openembedded.org/mt/89685524/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [RFC] support for multi project toolchain-cmake
On 07.02.2022 16:30, Tobias Neumann wrote: Because a SDK is specific for a hardware platform, but not for a project using that hardware platform, it is readonly and can be used for all projects with this hardware. This however requires that the compiled libraries generated with this SDK during the user side build process will be installed at a different location, which would be the second sysroot. Until now we only build a cmake project with external dependencies to the installed SDK using this workflow. Further external libraries, not contained in the SDK, had to be integrated in this cmake build, either as subdirectory or via external builds (using e.g. cmake-superbuilds). We now want to extend this workflow by supporting multiple cmake projects with dependencies to each other, thus the need for this second sysroot in the build. best regards Tobias Hi Tobias, Any reason you set SDK to be read-only? Perhaps your use case can be covered by extensible SDK? I mean if you have some shared libraries between project, isn't that enough to extend your SDK to solve the original problem? Cheers, Vyacheslav -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#161487): https://lists.openembedded.org/g/openembedded-core/message/161487 Mute This Topic: https://lists.openembedded.org/mt/88743247/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH v2] initramfs-framework: Add overlayroot module
Hi Alejandro On 02.02.2022 07:58, Alejandro Hernandez wrote: Hey Vyacheslav, ...I definitely tried extending the overlay-etc class but it simply does not work for this, since the rootfs becomes inaccessible once the system has booted, ... That's what I meant. What if we added another parameter, let's say OVERLAY_OLDROOT_MOUNT. If it's set, then lower layer can bind mounted to that mount point. In that case -etc part of overlayfs-etc has to be changed for something else, of course, to make it more general purpose class. Perhaps overlayfs.bbclass can be renamed to overlayfs-systemd.bbclass then? What do you think? ... but it is a different use case and boot flow so I believe both can co-exist for different workflows depending on the user needs... My concern here is code duplication :) If we find an issue in one place, we would need to change it in two places. Thanks, Vyacheslav Cheers, Alejandro -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#161183): https://lists.openembedded.org/g/openembedded-core/message/161183 Mute This Topic: https://lists.openembedded.org/mt/88773634/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH v2] initramfs-framework: Add overlayroot module
Hi Alejandro,
Thanks for the v2, but my questions from v1 are still left unanswered.
1. Do you really need the whole rootfs to be in overlay? Perhaps you
have another use case in mind, but the more scope overlay has, the more
migration effort you need in order to update upper layer on image upgrade.
2. I was thinking if perhaps that would be better to extend
files/overlayfs-etc-preinit.sh.in to suit your needs and re-use common
parts rather then having a good part of it copied to initramfs-framework?
Cheers,
Vyacheslav
On 29.01.2022 20:48, Alejandro Hernandez Samaniego wrote:
When installed, this module mounts a read-write (RW) overlay on
top of a root filesystem, which is kept read-only (RO), free
from modifications by the user, this might prove to be useful
if we want to access or restore the original unmodified rootfs.
The existing overlay-etc.bbclass does something similar, it
mounts an overlay on top of the /etc directory, however doing
the same for root causes the original root to be inaccessible
once the system is booted, hence why this module is added to
the initramfs boot flow, allowing us to mount the RW overlay,
while keeping the original rootfs mounted at /rofs once the
system finishes booting. This script is loosely based on that
class.
This module requires rootrw= to be passed as a kernel
parameter to specify the device/partition to be used as RW by the
overlay and has a dependency on overlayfs support being present
in the running kernel.
It does not require the read-only IMAGE_FEATURE to be enabled.
The module needs to be executed after the initramfs-module-rootfs
since it relies on it to mount the filesystem at initramfs startup
but before the finish module which normally switches root.
After overlayroot is executed the usual boot flow continues from
the real init process.
If something goes wrong while running this module, the rootfs
is still mounted RO (with no overlay) and the finish module is
executed to continue booting normally.
Its worth noting that, on purpose, this isnt installed by default
on any images that use initramfs-framework to keep the boot flow
unmodified, only when a user manually requests to install it,
then it becomes functional.
Signed-off-by: Alejandro Enedino Hernandez Samaniego
---
.../initramfs-framework/overlayroot | 112 ++
.../initrdscripts/initramfs-framework_1.0.bb | 9 ++
2 files changed, 121 insertions(+)
create mode 100644
meta/recipes-core/initrdscripts/initramfs-framework/overlayroot
diff --git a/meta/recipes-core/initrdscripts/initramfs-framework/overlayroot
b/meta/recipes-core/initrdscripts/initramfs-framework/overlayroot
new file mode 100644
index 00..d40342dc59
--- /dev/null
+++ b/meta/recipes-core/initrdscripts/initramfs-framework/overlayroot
@@ -0,0 +1,112 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MIT
+#
+# Copyright 2022 (C), Microsoft Corporation
+
+# Simple initramfs module intended to mount a read-write (RW)
+# overlayfs on top of /, keeping the original root filesystem
+# as read-only (RO), free from modifications by the user.
+#
+# NOTE: The read-only IMAGE_FEATURE is not required for this to work
+#
+# This script is based on the overlay-etc.bbclass, which sets up
+# an overlay on top of the /etc directory, but in this case allows
+# accessing the original, unmodified rootfs at /rofs after boot.
+#
+# It relies on the initramfs-module-rootfs to mount the original
+# root filesystem, and requires 'rootrw=' to be passed as a
+# kernel parameter, specifying the device/partition intended to
+# use as RW.
+#
+# This module needs to be executed after the initramfs-module-rootfs
+# since it relies on it to mount the filesystem at initramfs startup
+# but before the finish module which normally switches root.
+# After overlayroot is executed the usual boot flow continues from
+# the real init process.
+#
+# If something goes wrong while running this module, the rootfs
+# is still mounted RO (with no overlay) and the finish module is
+# executed to continue booting normally.
+#
+# It also has a dependency on overlayfs being enabled in the
+# running kernel via KERNEL_FEATURES (kmeta) or any other means.
+
+
+PATH=/sbin:/bin:/usr/sbin:/usr/bin
+
+# We get OLDROOT from the rootfs module
+OLDROOT="/rootfs"
+
+NEWROOT="${RWMOUNT}/root"
+RWMOUNT="/overlay"
+ROMOUNT="${RWMOUNT}/rofs"
+UPPER_DIR="${RWMOUNT}/upper"
+WORK_DIR="${RWMOUNT}/work"
+
+MODULES_DIR=/init.d
+
+# Something went wrong, make sure / is mounted as read only anyway.
+exit_gracefully() {
+echo $1 >/dev/console
+echo >/dev/console
+echo "OverlayRoot mounting failed, starting system as read-only"
>/dev/console
+echo >/dev/console
+
+# The following is borrowed from rootfs-postcommands.bbclass
+# This basically looks at the real rootfs mounting options and
+# replaces them with "ro"
+
+# Tweak the mount option and fs_passno for rootfs in fstab
+if [ -f ${OLDROOT}/etc/fstab ]; then
+
Re: [OE-core] [PATCH] initramfs-framework: Add overlayroot module
On 26.01.2022 19:09, Alejandro Hernandez Samaniego wrote: Oops! completely forgot!, I'll add it. This is loosely based on the overlay-etc.bbclass the difference is that the class only works for overlaying /etc and it doesnt require an initramfs, but, while its possible to tinker it in such a way that it overlays /, its not possible to access the original (RO) / after booting the system, this is the reason why this has to be done from initramfs and why this doesnt patch the overlay-etc.bbclass instead. I believe Ubuntu has this feature, coming from the cloud-initramfs package, functionally it should be similar although no code was borrowed from there. I'll add a note about the overlay-etc class though. I missed this part when I answered to your first email this morning. I generally against expanding the scope of overlayfs (i.e. using it for the whole / ), because it creates a bigger migration effort when you need to update the files on a persistent partition. I see the only difference in your code is that you provide access to original lower layer fs. Why would you want to do that? Thanks, Vyacheslav -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#161046): https://lists.openembedded.org/g/openembedded-core/message/161046 Mute This Topic: https://lists.openembedded.org/mt/88691930/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH] initramfs-framework: Add overlayroot module
Hi Alejandro,
Thanks for your patch.
I recently submitted two classes overlayfs and overlayfs-etc to do the
same thing you want to achieve. Could you please take a look if you can
use them instead? If not, perhaps we could adapt it to suit your needs?
Regards,
Vyacheslav
On 26.01.2022 08:22, Alejandro Hernandez Samaniego wrote:
When installed, this module mounts a read-write (RW) overlay on
top of a root filesystem, which is kept read-only (RO).
It needs to be executed after the initramfs-module-rootfs since
it relies on it to mount the filesystem at initramfs startup but
before the finish module which normally switches root.
It requires rootrw= to be passed as a kernel parameter to
specify the device/partition to be used as RW by the overlay and
has a dependency on overlayfs support being present in the
running kernel.
It does not require the read-only IMAGE_FEATURE to be enabled.
Signed-off-by: Alejandro Enedino Hernandez Samaniego
---
.../initramfs-framework/overlayroot | 93 +++
.../initrdscripts/initramfs-framework_1.0.bb | 9 ++
2 files changed, 102 insertions(+)
create mode 100644
meta/recipes-core/initrdscripts/initramfs-framework/overlayroot
diff --git a/meta/recipes-core/initrdscripts/initramfs-framework/overlayroot
b/meta/recipes-core/initrdscripts/initramfs-framework/overlayroot
new file mode 100644
index 00..ec5700e8fc
--- /dev/null
+++ b/meta/recipes-core/initrdscripts/initramfs-framework/overlayroot
@@ -0,0 +1,93 @@
+#!/bin/sh
+
+# Simple initramfs module intended to mount a read-write (RW)
+# overlayfs on top of /, keeping the original root filesystem
+# as read-only (RO).
+#
+# NOTE: The read-only IMAGE_FEATURE is not required for this to work
+#
+# It relies on the initramfs-module-rootfs to mount the original
+# root filesystem, and requires 'rootrw=' to be passed as a
+# kernel parameter, specifying the device/partition intended to
+# use as RW.
+#
+# It also has a dependency on overlayfs being enabled in the
+# running kernel via KERNEL_FEATURES (kmeta) or any other means.
+#
+# The RO root filesystem remains accessible by the system, mounted
+# at /rofs
+
+PATH=/sbin:/bin:/usr/sbin:/usr/bin
+
+# We get OLDROOT from the rootfs module
+OLDROOT="/rootfs"
+
+NEWROOT="${RWMOUNT}/root"
+RWMOUNT="/overlay"
+ROMOUNT="${RWMOUNT}/rofs"
+UPPER_DIR="${RWMOUNT}/upper"
+WORK_DIR="${RWMOUNT}/work"
+
+MODULES_DIR=/init.d
+
+exit_gracefully() {
+echo $1 >/dev/console
+echo >/dev/console
+echo "OverlayRoot mounting failed, starting system as read-only"
>/dev/console
+echo >/dev/console
+
+# Make sure / is mounted as read only anyway.
+# Borrowed from rootfs-postcommands.bbclass
+# Tweak the mount option and fs_passno for rootfs in fstab
+if [ -f ${OLDROOT}/etc/fstab ]; then
+sed -i -e
'/^[#[:space:]]*\/dev\/root/{s/defaults/ro/;s/\([[:space:]]*[[:digit:]]\)\([[:space:]]*\)[[:digit:]]$/\1\20/}'
${OLDROOT}/etc/fstab
+fi
+
+# Tweak the "mount -o remount,rw /" command in busybox-inittab inittab
+if [ -f ${OLDROOT}/etc/inittab ]; then
+sed -i 's|/bin/mount -o remount,rw /|/bin/mount -o remount,ro /|'
${OLDROOT}/etc/inittab
+fi
+
+# Continue as if the overlayroot module didn't exist
+. $MODULES_DIR/99-finish
+eval "finish_run"
+}
+
+
+if [ -z "$bootparam_rootrw" ]; then
+exit_gracefully "rootrw= kernel parameter doesn't exist and its required to
mount the overlayfs"
+fi
+
+mkdir -p ${RWMOUNT}
+
+# Mount RW device
+if mount -n -t ${bootparam_rootfstype:-ext4} -o
${bootparam_rootflags:-defaults} ${bootparam_rootrw} ${RWMOUNT}
+then
+# Set up overlay directories
+mkdir -p ${UPPER_DIR}
+mkdir -p ${WORK_DIR}
+mkdir -p ${NEWROOT}
+mkdir -p ${ROMOUNT}
+
+# Remount OLDROOT as read-only
+mount -o bind ${OLDROOT} ${ROMOUNT}
+mount -o remount,ro ${ROMOUNT}
+
+# Mount RW overlay
+mount -t overlay overlay -o
lowerdir=${ROMOUNT},upperdir=${UPPER_DIR},workdir=${WORK_DIR} ${NEWROOT} ||
exit_gracefully "initramfs-overlayroot: Mounting overlay failed"
+else
+exit_gracefully "initramfs-overlayroot: Mounting RW device failed"
+fi
+
+# Set up filesystems on overlay
+mkdir -p ${NEWROOT}/proc
+mkdir -p ${NEWROOT}/dev
+mkdir -p ${NEWROOT}/sys
+mkdir -p ${NEWROOT}/rofs
+
+mount -n --move ${ROMOUNT} ${NEWROOT}/rofs
+mount -n --move /proc ${NEWROOT}/proc
+mount -n --move /sys ${NEWROOT}/sys
+mount -n --move /dev ${NEWROOT}/dev
+
+exec chroot ${NEWROOT}/ ${bootparam_init:-/sbin/init} || exit_gracefully "Couldn't
chroot into overlay"
diff --git a/meta/recipes-core/initrdscripts/initramfs-framework_1.0.bb
b/meta/recipes-core/initrdscripts/initramfs-framework_1.0.bb
index 9e8c1dc3ab..4e76e20026 100644
--- a/meta/recipes-core/initrdscripts/initramfs-framework_1.0.bb
+++ b/meta/recipes-core/initrdscripts/initramfs-framework_1.0.bb
@@ -18,6 +18,7 @@ SRC_URI = "file://init \
file://e2fs \
file://debug \
file://lvm
[OE-core] [PATCH 8/8] overlayfs: move templates to files directory
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/overlayfs.bbclass | 52
meta/files/overlayfs-all-overlays.service.in | 12 +
meta/files/overlayfs-create-dirs.service.in | 14 ++
meta/files/overlayfs-unit.mount.in | 13 +
4 files changed, 49 insertions(+), 42 deletions(-)
create mode 100644 meta/files/overlayfs-all-overlays.service.in
create mode 100644 meta/files/overlayfs-create-dirs.service.in
create mode 100644 meta/files/overlayfs-unit.mount.in
diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass
index f1b8086ea8..4a860f7308 100644
--- a/meta/classes/overlayfs.bbclass
+++ b/meta/classes/overlayfs.bbclass
@@ -37,51 +37,19 @@ REQUIRED_DISTRO_FEATURES += "systemd overlayfs"
inherit systemd features_check
+OVERLAYFS_CREATE_DIRS_TEMPLATE ??=
"${COREBASE}/meta/files/overlayfs-create-dirs.service.in"
+OVERLAYFS_MOUNT_UNIT_TEMPLATE ??=
"${COREBASE}/meta/files/overlayfs-unit.mount.in"
+OVERLAYFS_ALL_OVERLAYS_TEMPLATE ??=
"${COREBASE}/meta/files/overlayfs-all-overlays.service.in"
+
python do_create_overlayfs_units() {
from oe.overlayfs import mountUnitName
-CreateDirsUnitTemplate = """[Unit]
-Description=Overlayfs directories setup
-Requires={DATA_MOUNT_UNIT}
-After={DATA_MOUNT_UNIT}
-DefaultDependencies=no
-
-[Service]
-Type=oneshot
-ExecStart=mkdir -p {DATA_MOUNT_POINT}/workdir{LOWERDIR} && mkdir -p
{DATA_MOUNT_POINT}/upper{LOWERDIR}
-RemainAfterExit=true
-StandardOutput=journal
-
-[Install]
-WantedBy=multi-user.target
-"""
-MountUnitTemplate = """[Unit]
-Description=Overlayfs mount unit
-Requires={CREATE_DIRS_SERVICE}
-After={CREATE_DIRS_SERVICE}
-
-[Mount]
-What=overlay
-Where={LOWERDIR}
-Type=overlay
-Options=lowerdir={LOWERDIR},upperdir={DATA_MOUNT_POINT}/upper{LOWERDIR},workdir={DATA_MOUNT_POINT}/workdir{LOWERDIR}
-
-[Install]
-WantedBy=multi-user.target
-"""
-AllOverlaysTemplate = """[Unit]
-Description=Groups all overlays required by {PN} in one unit
-After={ALL_OVERLAYFS_UNITS}
-Requires={ALL_OVERLAYFS_UNITS}
-
-[Service]
-Type=oneshot
-ExecStart=/bin/true
-RemainAfterExit=true
-
-[Install]
-WantedBy=local-fs.target
-"""
+with open(d.getVar("OVERLAYFS_CREATE_DIRS_TEMPLATE"), "r") as f:
+CreateDirsUnitTemplate = f.read()
+with open(d.getVar("OVERLAYFS_MOUNT_UNIT_TEMPLATE"), "r") as f:
+MountUnitTemplate = f.read()
+with open(d.getVar("OVERLAYFS_ALL_OVERLAYS_TEMPLATE"), "r") as f:
+AllOverlaysTemplate = f.read()
def prepareUnits(data, lower):
from oe.overlayfs import helperUnitName
diff --git a/meta/files/overlayfs-all-overlays.service.in
b/meta/files/overlayfs-all-overlays.service.in
new file mode 100644
index 00..74ee4e90ae
--- /dev/null
+++ b/meta/files/overlayfs-all-overlays.service.in
@@ -0,0 +1,12 @@
+[Unit]
+Description=Groups all overlays required by {PN} in one unit
+After={ALL_OVERLAYFS_UNITS}
+Requires={ALL_OVERLAYFS_UNITS}
+
+[Service]
+Type=oneshot
+ExecStart=/bin/true
+RemainAfterExit=true
+
+[Install]
+WantedBy=local-fs.target
diff --git a/meta/files/overlayfs-create-dirs.service.in
b/meta/files/overlayfs-create-dirs.service.in
new file mode 100644
index 00..17204145f2
--- /dev/null
+++ b/meta/files/overlayfs-create-dirs.service.in
@@ -0,0 +1,14 @@
+[Unit]
+Description=Overlayfs directories setup
+Requires={DATA_MOUNT_UNIT}
+After={DATA_MOUNT_UNIT}
+DefaultDependencies=no
+
+[Service]
+Type=oneshot
+ExecStart=mkdir -p {DATA_MOUNT_POINT}/workdir{LOWERDIR} && mkdir -p
{DATA_MOUNT_POINT}/upper{LOWERDIR}
+RemainAfterExit=true
+StandardOutput=journal
+
+[Install]
+WantedBy=multi-user.target
diff --git a/meta/files/overlayfs-unit.mount.in
b/meta/files/overlayfs-unit.mount.in
new file mode 100644
index 00..1d33b7e39c
--- /dev/null
+++ b/meta/files/overlayfs-unit.mount.in
@@ -0,0 +1,13 @@
+[Unit]
+Description=Overlayfs mount unit
+Requires={CREATE_DIRS_SERVICE}
+After={CREATE_DIRS_SERVICE}
+
+[Mount]
+What=overlay
+Where={LOWERDIR}
+Type=overlay
+Options=lowerdir={LOWERDIR},upperdir={DATA_MOUNT_POINT}/upper{LOWERDIR},workdir={DATA_MOUNT_POINT}/workdir{LOWERDIR}
+
+[Install]
+WantedBy=multi-user.target
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159517):
https://lists.openembedded.org/g/openembedded-core/message/159517
Mute This Topic: https://lists.openembedded.org/mt/87635154/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 5/8] oeqa/selftest: overlayfs helper function
Move helper functions out of class scope so they can be used in other tests
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/overlayfs.py | 32 +++
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index 84242a1605..43415778ce 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -5,16 +5,16 @@
from oeqa.selftest.case import OESelftestTestCase
from oeqa.utils.commands import runCmd, bitbake, get_bb_var, runqemu
-class OverlayFSTests(OESelftestTestCase):
-"""Overlayfs class usage tests"""
+def getline_qemu(out, line):
+for l in out.split('\n'):
+if line in l:
+return l
-def getline_qemu(self, out, line):
-for l in out.split('\n'):
-if line in l:
-return l
+def getline(res, line):
+return getline_qemu(res.output, line)
-def getline(self, res, line):
-return self.getline_qemu(res.output, line)
+class OverlayFSTests(OESelftestTestCase):
+"""Overlayfs class usage tests"""
def add_overlay_conf_to_machine(self):
machine_inc = """
@@ -40,7 +40,7 @@ inherit overlayfs
self.write_recipeinc('overlayfs-user', overlayfs_recipe_append)
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "overlayfs-user was skipped: missing required
distro features")
+line = getline(res, "overlayfs-user was skipped: missing required
distro features")
self.assertTrue("overlayfs" in res.output, msg=res.output)
self.assertTrue("systemd" in res.output, msg=res.output)
self.assertTrue("ERROR: Required build target 'core-image-minimal' has
no buildable providers." in res.output, msg=res.output)
@@ -61,9 +61,9 @@ DISTRO_FEATURES += "systemd overlayfs"
self.add_overlay_conf_to_machine()
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "Unit name mnt-overlay.mount not found in
systemd unit directories")
+line = getline(res, "Unit name mnt-overlay.mount not found in systemd
unit directories")
self.assertTrue(line and line.startswith("WARNING:"), msg=res.output)
-line = self.getline(res, "Not all mount units are installed by the
BSP")
+line = getline(res, "Not all mount units are installed by the BSP")
self.assertTrue(line and line.startswith("ERROR:"), msg=res.output)
def test_mount_unit_not_set(self):
@@ -81,7 +81,7 @@ DISTRO_FEATURES += "systemd overlayfs"
self.write_config(config)
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "A recipe uses overlayfs class but there is
no OVERLAYFS_MOUNT_POINT set in your MACHINE configuration")
+line = getline(res, "A recipe uses overlayfs class but there is no
OVERLAYFS_MOUNT_POINT set in your MACHINE configuration")
self.assertTrue(line and line.startswith("Parsing recipes...ERROR:"),
msg=res.output)
def test_wrong_mount_unit_set(self):
@@ -104,7 +104,7 @@ OVERLAYFS_MOUNT_POINT[usr-share-overlay] =
"/usr/share/overlay"
self.set_machine_config(wrong_machine_config)
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "Missing required mount point for
OVERLAYFS_MOUNT_POINT[mnt-overlay] in your MACHINE configuration")
+line = getline(res, "Missing required mount point for
OVERLAYFS_MOUNT_POINT[mnt-overlay] in your MACHINE configuration")
self.assertTrue(line and line.startswith("Parsing recipes...ERROR:"),
msg=res.output)
def test_correct_image(self):
@@ -201,11 +201,11 @@ EOT
# /usr/share/my-application as an overlay (see overlayfs-user
recipe)
status, output = qemu.run_serial("/bin/mount -t tmpfs,overlay")
-line = self.getline_qemu(output, "on /mnt/overlay")
+line = getline_qemu(output, "on /mnt/overlay")
self.assertTrue(line and line.startswith("tmpfs"), msg=output)
-line = self.getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/my-application")
+line = getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/my-application")
self.assertTrue(line and line.startswith("overlay"), msg=output)
-line = self.getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/another-overlay-mount")
+line = getline_qemu(output,
"upperdir=/mnt/o
[OE-core] [PATCH 6/8] oeqa/selftest: unit tests for overlayfs-etc
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/overlayfs.py | 184 ++
1 file changed, 184 insertions(+)
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index 43415778ce..82007fade7 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -209,3 +209,187 @@ EOT
line = getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/another-overlay-mount")
self.assertTrue(line and line.startswith("overlay"), msg=output)
+
+class OverlayFSEtcRunTimeTests(OESelftestTestCase):
+"""overlayfs-etc class tests"""
+
+def test_all_required_variables_set(self):
+"""
+Summary: Check that required variables are set
+Expected: Fail when any of required variables is missing
+Author:Vyacheslav Yurkov
+"""
+
+configBase = """
+DISTRO_FEATURES += "systemd"
+
+# Use systemd as init manager
+VIRTUAL-RUNTIME_init_manager = "systemd"
+
+# enable overlayfs in the kernel
+KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
+
+# Image configuration for overlayfs-etc
+EXTRA_IMAGE_FEATURES += "overlayfs-etc"
+IMAGE_FEATURES:remove = "package-management"
+"""
+configMountPoint = """
+OVERLAYFS_ETC_MOUNT_POINT = "/data"
+"""
+configDevice = """
+OVERLAYFS_ETC_DEVICE = "/dev/mmcblk0p1"
+"""
+
+self.write_config(configBase)
+res = bitbake('core-image-minimal', ignore_status=True)
+line = getline(res, "OVERLAYFS_ETC_MOUNT_POINT must be set in your
MACHINE configuration")
+self.assertTrue(line, msg=res.output)
+
+self.append_config(configMountPoint)
+res = bitbake('core-image-minimal', ignore_status=True)
+line = getline(res, "OVERLAYFS_ETC_DEVICE must be set in your MACHINE
configuration")
+self.assertTrue(line, msg=res.output)
+
+self.append_config(configDevice)
+res = bitbake('core-image-minimal', ignore_status=True)
+line = getline(res, "OVERLAYFS_ETC_FSTYPE should contain a valid file
system type on /dev/mmcblk0p1")
+self.assertTrue(line, msg=res.output)
+
+def test_image_feature_conflict(self):
+"""
+Summary: Overlayfs-etc is not allowed to be used with
package-management
+Expected: Feature conflict
+Author:Vyacheslav Yurkov
+"""
+
+config = """
+DISTRO_FEATURES += "systemd"
+
+# Use systemd as init manager
+VIRTUAL-RUNTIME_init_manager = "systemd"
+
+# enable overlayfs in the kernel
+KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
+EXTRA_IMAGE_FEATURES += "overlayfs-etc"
+EXTRA_IMAGE_FEATURES += "package-management"
+"""
+
+self.write_config(config)
+
+res = bitbake('core-image-minimal', ignore_status=True)
+line = getline(res, "contains conflicting IMAGE_FEATURES")
+self.assertTrue("overlayfs-etc" in res.output, msg=res.output)
+self.assertTrue("package-management" in res.output, msg=res.output)
+
+def test_image_feature_is_missing_class_included(self):
+configAppend = """
+INHERIT += "overlayfs-etc"
+"""
+self.run_check_image_feature(configAppend)
+
+def test_image_feature_is_missing(self):
+self.run_check_image_feature()
+
+def run_check_image_feature(self, appendToConfig=""):
+"""
+Summary: Overlayfs-etc class is not applied when image feature is
not set
+ even if we inherit it directly,
+Expected: Image is created successfully but /etc is not an overlay
+Author:Vyacheslav Yurkov
+"""
+
+config = f"""
+DISTRO_FEATURES += "systemd"
+
+# Use systemd as init manager
+VIRTUAL-RUNTIME_init_manager = "systemd"
+
+# enable overlayfs in the kernel
+KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
+
+IMAGE_FSTYPES += "wic"
+WKS_FILE = "overlayfs_etc.wks.in"
+
+EXTRA_IMAGE_FEATURES += "read-only-rootfs"
+# Image configuration for overlayfs-etc
+OVERLAYFS_ETC_MOUNT_POINT = "/data"
+OVERLAYFS_ETC_DEVICE = "/dev/sda3"
+{appendToConfig}
+"""
+
+self.write_config(config)
+
+bitbake('core-image-minimal')
+
+with runqemu('core-image-minimal', image_fstype='wic') as qemu:
+ status, output = qemu.run_serial("/bin/mount
[OE-core] [PATCH 7/8] overlayfs: update notes on /etc
Signed-off-by: Vyacheslav Yurkov --- meta/classes/overlayfs.bbclass | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass index 3c0f4dc882..f1b8086ea8 100644 --- a/meta/classes/overlayfs.bbclass +++ b/meta/classes/overlayfs.bbclass @@ -31,6 +31,7 @@ # OVERLAYFS_WRITABLE_PATHS[mnt-overlay] = "/usr/share/another-application" # # Note: the class does not support /etc directory itself, because systemd depends on it +# For /etc directory use overlayfs-etc class REQUIRED_DISTRO_FEATURES += "systemd overlayfs" -- 2.28.0 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#159516): https://lists.openembedded.org/g/openembedded-core/message/159516 Mute This Topic: https://lists.openembedded.org/mt/87635152/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 4/8] image: add overlayfs-etc image feature
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/image.bbclass | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 8a46b4852c..2b0ce4a988 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -15,6 +15,7 @@ IMGCLASSES += "${@bb.utils.contains('IMAGE_FSTYPES',
'container', 'image-contain
IMGCLASSES += "image_types_wic"
IMGCLASSES += "rootfs-postcommands"
IMGCLASSES += "image-postinst-intercepts"
+IMGCLASSES += "overlayfs-etc"
inherit ${IMGCLASSES}
TOOLCHAIN_TARGET_TASK += "${PACKAGE_INSTALL}"
@@ -33,7 +34,7 @@ INHIBIT_DEFAULT_DEPS = "1"
# IMAGE_FEATURES may contain any available package group
IMAGE_FEATURES ?= ""
IMAGE_FEATURES[type] = "list"
-IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs
read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password
allow-empty-password allow-root-login post-install-logging"
+IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs
read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password
allow-empty-password allow-root-login post-install-logging overlayfs-etc"
# Generate companion debugfs?
IMAGE_GEN_DEBUGFS ?= "0"
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159513):
https://lists.openembedded.org/g/openembedded-core/message/159513
Mute This Topic: https://lists.openembedded.org/mt/87635148/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 3/8] wic: image for overlayfs-etc tests
Introduce wic image for overlayfs-etc tests with a dedicated /data
partition and configurable kernel parameters
Signed-off-by: Vyacheslav Yurkov
---
meta-selftest/wic/overlayfs_etc.wks.in | 4
1 file changed, 4 insertions(+)
create mode 100644 meta-selftest/wic/overlayfs_etc.wks.in
diff --git a/meta-selftest/wic/overlayfs_etc.wks.in
b/meta-selftest/wic/overlayfs_etc.wks.in
new file mode 100644
index 00..1e1e5836e7
--- /dev/null
+++ b/meta-selftest/wic/overlayfs_etc.wks.in
@@ -0,0 +1,4 @@
+part /boot --active --source bootimg-biosplusefi --ondisk sda
--sourceparams="loader=grub-efi" --align 1024
+part / --source rootfs --ondisk sda --fstype=ext4 --use-uuid --align 1024
+part --ondisk sda --fstype=ext4 --size=5 --align 1024
+bootloader --ptable gpt --timeout=1 --append="rootfstype=ext4
console=ttyS0,115200 console=tty0 ${OVERLAYFS_INIT_OPTION}"
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159512):
https://lists.openembedded.org/g/openembedded-core/message/159512
Mute This Topic: https://lists.openembedded.org/mt/87635147/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/8] overlayfs-etc: mount etc as overlayfs
This class provides an image feature that mounts /etc as an overlayfs
file system. This is an extension for existing overlayfs class, which
doesn't support /etc
Signed-off-by: Alfred Schapansky
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/overlayfs-etc.bbclass | 76 ++
1 file changed, 76 insertions(+)
create mode 100644 meta/classes/overlayfs-etc.bbclass
diff --git a/meta/classes/overlayfs-etc.bbclass
b/meta/classes/overlayfs-etc.bbclass
new file mode 100644
index 00..4ced07ba11
--- /dev/null
+++ b/meta/classes/overlayfs-etc.bbclass
@@ -0,0 +1,76 @@
+# Class for setting up /etc in overlayfs
+#
+# In order to have /etc directory in overlayfs a special handling at early
boot stage is required
+# The idea is to supply a custom init script that mounts /etc before launching
actual init program,
+# because the latter already requires /etc to be mounted
+#
+# The configuration must be machine specific. You should at least set these
three variables:
+# OVERLAYFS_ETC_MOUNT_POINT ?= "/data"
+# OVERLAYFS_ETC_FSTYPE ?= "ext4"
+# OVERLAYFS_ETC_DEVICE ?= "/dev/mmcblk0p2"
+#
+# To control more mount options you should consider setting mount options:
+# OVERLAYFS_ETC_MOUNT_OPTIONS ?= "defaults"
+#
+# The class provides two options for /sbin/init generation
+# 1. Default option is to rename original /sbin/init to /sbin/init.orig and
place generated init under
+#original name, i.e. /sbin/init. It has an advantage that you won't need
to change any kernel
+#parameters in order to make it work, but it poses a restriction that
package-management can't
+#be used, becaause updating init manager would remove generated script
+# 2. If you are would like to keep original init as is, you can set
+#OVERLAYFS_ETC_USE_ORIG_INIT_NAME = "0"
+#Then generated init will be named /sbin/preinit and you would need to
extend you kernel parameters
+#manually in your bootloader configuration.
+#
+# Regardless which mode you choose, update and migration strategy of
configuration files under /etc
+# overlay is out of scope of this class
+
+ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("IMAGE_FEATURES",
"overlayfs-etc", "create_overlayfs_etc_preinit;", "", d)}'
+IMAGE_FEATURES_CONFLICTS_overlayfs-etc = "package-management"
+
+OVERLAYFS_ETC_MOUNT_POINT ??= ""
+OVERLAYFS_ETC_FSTYPE ??= ""
+OVERLAYFS_ETC_DEVICE ??= ""
+OVERLAYFS_ETC_USE_ORIG_INIT_NAME ??= "1"
+OVERLAYFS_ETC_MOUNT_OPTIONS ??= "defaults"
+OVERLAYFS_ETC_INIT_TEMPLATE ??=
"${COREBASE}/meta/files/overlayfs-etc-preinit.sh.in"
+
+python create_overlayfs_etc_preinit() {
+overlayEtcMountPoint = d.getVar("OVERLAYFS_ETC_MOUNT_POINT")
+overlayEtcFsType = d.getVar("OVERLAYFS_ETC_FSTYPE")
+overlayEtcDevice = d.getVar("OVERLAYFS_ETC_DEVICE")
+
+if not overlayEtcMountPoint:
+bb.fatal("OVERLAYFS_ETC_MOUNT_POINT must be set in your MACHINE
configuration")
+if not overlayEtcDevice:
+bb.fatal("OVERLAYFS_ETC_DEVICE must be set in your MACHINE
configuration")
+if not overlayEtcFsType:
+bb.fatal("OVERLAYFS_ETC_FSTYPE should contain a valid file system type
on {0}".format(overlayEtcDevice))
+
+with open(d.getVar("OVERLAYFS_ETC_INIT_TEMPLATE"), "r") as f:
+PreinitTemplate = f.read()
+
+useOrigInit =
oe.types.boolean(d.getVar('OVERLAYFS_ETC_USE_ORIG_INIT_NAME'))
+preinitPath = oe.path.join(d.getVar("IMAGE_ROOTFS"),
d.getVar("base_sbindir"), "preinit")
+initBaseName = oe.path.join(d.getVar("base_sbindir"), "init")
+origInitNameSuffix = ".orig"
+
+args = {
+'OVERLAYFS_ETC_MOUNT_POINT': overlayEtcMountPoint,
+'OVERLAYFS_ETC_MOUNT_OPTIONS': d.getVar('OVERLAYFS_ETC_MOUNT_OPTIONS'),
+'OVERLAYFS_ETC_FSTYPE': overlayEtcFsType,
+'OVERLAYFS_ETC_DEVICE': overlayEtcDevice,
+'SBIN_INIT_NAME': initBaseName + origInitNameSuffix if useOrigInit
else initBaseName
+}
+
+if useOrigInit:
+# rename original /sbin/init
+origInit = oe.path.join(d.getVar("IMAGE_ROOTFS"), initBaseName)
+bb.debug(1, "rootfs path %s, init path %s, test %s" %
(d.getVar('IMAGE_ROOTFS'), origInit, d.getVar("IMAGE_ROOTFS")))
+bb.utils.rename(origInit, origInit + origInitNameSuffix)
+preinitPath = origInit
+
+with open(preinitPath, 'w') as f:
+f.write(PreinitTemplate.format(**args))
+os.chmod(preinitPath, 0o755)
+}
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159511):
https://lists.openembedded.org/g/openembedded-core/message/159511
Mute This Topic: https://lists.openembedded.org/mt/87635146/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/8] files: add overlayfs-etc-preinit.sh.in
A template init script for overlayfs-etc class
Signed-off-by: Vyacheslav Yurkov
---
meta/files/overlayfs-etc-preinit.sh.in | 29 ++
1 file changed, 29 insertions(+)
create mode 100644 meta/files/overlayfs-etc-preinit.sh.in
diff --git a/meta/files/overlayfs-etc-preinit.sh.in
b/meta/files/overlayfs-etc-preinit.sh.in
new file mode 100644
index 00..2ebb6c9224
--- /dev/null
+++ b/meta/files/overlayfs-etc-preinit.sh.in
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+echo "PREINIT: Start"
+
+PATH=/sbin:/bin:/usr/sbin:/usr/bin
+mount -o remount,rw /
+
+mkdir -p /proc
+mkdir -p /sys
+mkdir -p /run
+mkdir -p /var/run
+
+mount -t proc proc /proc
+mount -t sysfs sysfs /sys
+
+[ -z "$CONSOLE" ] && CONSOLE="/dev/console"
+
+mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}
+if mount -n -t {OVERLAYFS_ETC_FSTYPE} -o {OVERLAYFS_ETC_MOUNT_OPTIONS}
{OVERLAYFS_ETC_DEVICE} {OVERLAYFS_ETC_MOUNT_POINT}
+then
+mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper
+mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/work
+mount -n -t overlay -o
upperdir={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper,lowerdir=/etc,workdir={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/work
{OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper /etc || echo "PREINIT: Mounting
etc-overlay failed!"
+else
+echo "PREINIT: Mounting failed!"
+fi
+
+echo "PREINIT: done; starting "
+exec {SBIN_INIT_NAME}
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159510):
https://lists.openembedded.org/g/openembedded-core/message/159510
Mute This Topic: https://lists.openembedded.org/mt/87635144/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v3 0/8] Pull request (cover letter only)
This is a V3 of overlayfs-etc image feature implementation, that allows to setup the whole /etc under overlayfs. Please review and merge if you find it OK The following changes since commit 1a6c2a7345199d77ad5aeac8ad337ed80a8aa39b: build-appliance-image: Update to master head revision (2021-12-09 13:56:07 +) are available in the Git repository at: git://github.com/UVV-gh/openembedded-core feature/overlayfs-etc https://github.com/UVV-gh/openembedded-core/tree/feature/overlayfs-etc Vyacheslav Yurkov (8): files: add overlayfs-etc-preinit.sh.in overlayfs-etc: mount etc as overlayfs wic: image for overlayfs-etc tests image: add overlayfs-etc image feature oeqa/selftest: overlayfs helper function oeqa/selftest: unit tests for overlayfs-etc overlayfs: update notes on /etc overlayfs: move templates to files directory meta-selftest/wic/overlayfs_etc.wks.in | 4 + meta/classes/image.bbclass | 3 +- meta/classes/overlayfs-etc.bbclass | 76 +++ meta/classes/overlayfs.bbclass | 53 + meta/files/overlayfs-all-overlays.service.in | 12 ++ meta/files/overlayfs-create-dirs.service.in | 14 ++ meta/files/overlayfs-etc-preinit.sh.in | 29 +++ meta/files/overlayfs-unit.mount.in | 13 ++ meta/lib/oeqa/selftest/cases/overlayfs.py| 216 +-- 9 files changed, 361 insertions(+), 59 deletions(-) create mode 100644 meta-selftest/wic/overlayfs_etc.wks.in create mode 100644 meta/classes/overlayfs-etc.bbclass create mode 100644 meta/files/overlayfs-all-overlays.service.in create mode 100644 meta/files/overlayfs-create-dirs.service.in create mode 100644 meta/files/overlayfs-etc-preinit.sh.in create mode 100644 meta/files/overlayfs-unit.mount.in -- 2.28.0 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#159509): https://lists.openembedded.org/g/openembedded-core/message/159509 Mute This Topic: https://lists.openembedded.org/mt/87635143/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 7/8] overlayfs: update notes on /etc
Signed-off-by: Vyacheslav Yurkov --- meta/classes/overlayfs.bbclass | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass index 3c0f4dc882..f1b8086ea8 100644 --- a/meta/classes/overlayfs.bbclass +++ b/meta/classes/overlayfs.bbclass @@ -31,6 +31,7 @@ # OVERLAYFS_WRITABLE_PATHS[mnt-overlay] = "/usr/share/another-application" # # Note: the class does not support /etc directory itself, because systemd depends on it +# For /etc directory use overlayfs-etc class REQUIRED_DISTRO_FEATURES += "systemd overlayfs" -- 2.28.0 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#159505): https://lists.openembedded.org/g/openembedded-core/message/159505 Mute This Topic: https://lists.openembedded.org/mt/87634123/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 6/8] oeqa/selftest: unit tests for overlayfs-etc
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/overlayfs.py | 184 ++
1 file changed, 184 insertions(+)
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index 43415778ce..82007fade7 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -209,3 +209,187 @@ EOT
line = getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/another-overlay-mount")
self.assertTrue(line and line.startswith("overlay"), msg=output)
+
+class OverlayFSEtcRunTimeTests(OESelftestTestCase):
+"""overlayfs-etc class tests"""
+
+def test_all_required_variables_set(self):
+"""
+Summary: Check that required variables are set
+Expected: Fail when any of required variables is missing
+Author:Vyacheslav Yurkov
+"""
+
+configBase = """
+DISTRO_FEATURES += "systemd"
+
+# Use systemd as init manager
+VIRTUAL-RUNTIME_init_manager = "systemd"
+
+# enable overlayfs in the kernel
+KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
+
+# Image configuration for overlayfs-etc
+EXTRA_IMAGE_FEATURES += "overlayfs-etc"
+IMAGE_FEATURES:remove = "package-management"
+"""
+configMountPoint = """
+OVERLAYFS_ETC_MOUNT_POINT = "/data"
+"""
+configDevice = """
+OVERLAYFS_ETC_DEVICE = "/dev/mmcblk0p1"
+"""
+
+self.write_config(configBase)
+res = bitbake('core-image-minimal', ignore_status=True)
+line = getline(res, "OVERLAYFS_ETC_MOUNT_POINT must be set in your
MACHINE configuration")
+self.assertTrue(line, msg=res.output)
+
+self.append_config(configMountPoint)
+res = bitbake('core-image-minimal', ignore_status=True)
+line = getline(res, "OVERLAYFS_ETC_DEVICE must be set in your MACHINE
configuration")
+self.assertTrue(line, msg=res.output)
+
+self.append_config(configDevice)
+res = bitbake('core-image-minimal', ignore_status=True)
+line = getline(res, "OVERLAYFS_ETC_FSTYPE should contain a valid file
system type on /dev/mmcblk0p1")
+self.assertTrue(line, msg=res.output)
+
+def test_image_feature_conflict(self):
+"""
+Summary: Overlayfs-etc is not allowed to be used with
package-management
+Expected: Feature conflict
+Author:Vyacheslav Yurkov
+"""
+
+config = """
+DISTRO_FEATURES += "systemd"
+
+# Use systemd as init manager
+VIRTUAL-RUNTIME_init_manager = "systemd"
+
+# enable overlayfs in the kernel
+KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
+EXTRA_IMAGE_FEATURES += "overlayfs-etc"
+EXTRA_IMAGE_FEATURES += "package-management"
+"""
+
+self.write_config(config)
+
+res = bitbake('core-image-minimal', ignore_status=True)
+line = getline(res, "contains conflicting IMAGE_FEATURES")
+self.assertTrue("overlayfs-etc" in res.output, msg=res.output)
+self.assertTrue("package-management" in res.output, msg=res.output)
+
+def test_image_feature_is_missing_class_included(self):
+configAppend = """
+INHERIT += "overlayfs-etc"
+"""
+self.run_check_image_feature(configAppend)
+
+def test_image_feature_is_missing(self):
+self.run_check_image_feature()
+
+def run_check_image_feature(self, appendToConfig=""):
+"""
+Summary: Overlayfs-etc class is not applied when image feature is
not set
+ even if we inherit it directly,
+Expected: Image is created successfully but /etc is not an overlay
+Author:Vyacheslav Yurkov
+"""
+
+config = f"""
+DISTRO_FEATURES += "systemd"
+
+# Use systemd as init manager
+VIRTUAL-RUNTIME_init_manager = "systemd"
+
+# enable overlayfs in the kernel
+KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
+
+IMAGE_FSTYPES += "wic"
+WKS_FILE = "overlayfs_etc.wks.in"
+
+EXTRA_IMAGE_FEATURES += "read-only-rootfs"
+# Image configuration for overlayfs-etc
+OVERLAYFS_ETC_MOUNT_POINT = "/data"
+OVERLAYFS_ETC_DEVICE = "/dev/sda3"
+{appendToConfig}
+"""
+
+self.write_config(config)
+
+bitbake('core-image-minimal')
+
+with runqemu('core-image-minimal', image_fstype='wic') as qemu:
+ status, output = qemu.run_serial("/bin/mount
[OE-core] [PATCH 8/8] overlayfs: move templates to files directory
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/overlayfs.bbclass | 52
meta/files/overlayfs-all-overlays.service.in | 12 +
meta/files/overlayfs-create-dirs.service.in | 14 ++
meta/files/overlayfs-unit.mount.in | 13 +
4 files changed, 49 insertions(+), 42 deletions(-)
create mode 100644 meta/files/overlayfs-all-overlays.service.in
create mode 100644 meta/files/overlayfs-create-dirs.service.in
create mode 100644 meta/files/overlayfs-unit.mount.in
diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass
index f1b8086ea8..4a860f7308 100644
--- a/meta/classes/overlayfs.bbclass
+++ b/meta/classes/overlayfs.bbclass
@@ -37,51 +37,19 @@ REQUIRED_DISTRO_FEATURES += "systemd overlayfs"
inherit systemd features_check
+OVERLAYFS_CREATE_DIRS_TEMPLATE ??=
"${COREBASE}/meta/files/overlayfs-create-dirs.service.in"
+OVERLAYFS_MOUNT_UNIT_TEMPLATE ??=
"${COREBASE}/meta/files/overlayfs-unit.mount.in"
+OVERLAYFS_ALL_OVERLAYS_TEMPLATE ??=
"${COREBASE}/meta/files/overlayfs-all-overlays.service.in"
+
python do_create_overlayfs_units() {
from oe.overlayfs import mountUnitName
-CreateDirsUnitTemplate = """[Unit]
-Description=Overlayfs directories setup
-Requires={DATA_MOUNT_UNIT}
-After={DATA_MOUNT_UNIT}
-DefaultDependencies=no
-
-[Service]
-Type=oneshot
-ExecStart=mkdir -p {DATA_MOUNT_POINT}/workdir{LOWERDIR} && mkdir -p
{DATA_MOUNT_POINT}/upper{LOWERDIR}
-RemainAfterExit=true
-StandardOutput=journal
-
-[Install]
-WantedBy=multi-user.target
-"""
-MountUnitTemplate = """[Unit]
-Description=Overlayfs mount unit
-Requires={CREATE_DIRS_SERVICE}
-After={CREATE_DIRS_SERVICE}
-
-[Mount]
-What=overlay
-Where={LOWERDIR}
-Type=overlay
-Options=lowerdir={LOWERDIR},upperdir={DATA_MOUNT_POINT}/upper{LOWERDIR},workdir={DATA_MOUNT_POINT}/workdir{LOWERDIR}
-
-[Install]
-WantedBy=multi-user.target
-"""
-AllOverlaysTemplate = """[Unit]
-Description=Groups all overlays required by {PN} in one unit
-After={ALL_OVERLAYFS_UNITS}
-Requires={ALL_OVERLAYFS_UNITS}
-
-[Service]
-Type=oneshot
-ExecStart=/bin/true
-RemainAfterExit=true
-
-[Install]
-WantedBy=local-fs.target
-"""
+with open(d.getVar("OVERLAYFS_CREATE_DIRS_TEMPLATE"), "r") as f:
+CreateDirsUnitTemplate = f.read()
+with open(d.getVar("OVERLAYFS_MOUNT_UNIT_TEMPLATE"), "r") as f:
+MountUnitTemplate = f.read()
+with open(d.getVar("OVERLAYFS_ALL_OVERLAYS_TEMPLATE"), "r") as f:
+AllOverlaysTemplate = f.read()
def prepareUnits(data, lower):
from oe.overlayfs import helperUnitName
diff --git a/meta/files/overlayfs-all-overlays.service.in
b/meta/files/overlayfs-all-overlays.service.in
new file mode 100644
index 00..74ee4e90ae
--- /dev/null
+++ b/meta/files/overlayfs-all-overlays.service.in
@@ -0,0 +1,12 @@
+[Unit]
+Description=Groups all overlays required by {PN} in one unit
+After={ALL_OVERLAYFS_UNITS}
+Requires={ALL_OVERLAYFS_UNITS}
+
+[Service]
+Type=oneshot
+ExecStart=/bin/true
+RemainAfterExit=true
+
+[Install]
+WantedBy=local-fs.target
diff --git a/meta/files/overlayfs-create-dirs.service.in
b/meta/files/overlayfs-create-dirs.service.in
new file mode 100644
index 00..17204145f2
--- /dev/null
+++ b/meta/files/overlayfs-create-dirs.service.in
@@ -0,0 +1,14 @@
+[Unit]
+Description=Overlayfs directories setup
+Requires={DATA_MOUNT_UNIT}
+After={DATA_MOUNT_UNIT}
+DefaultDependencies=no
+
+[Service]
+Type=oneshot
+ExecStart=mkdir -p {DATA_MOUNT_POINT}/workdir{LOWERDIR} && mkdir -p
{DATA_MOUNT_POINT}/upper{LOWERDIR}
+RemainAfterExit=true
+StandardOutput=journal
+
+[Install]
+WantedBy=multi-user.target
diff --git a/meta/files/overlayfs-unit.mount.in
b/meta/files/overlayfs-unit.mount.in
new file mode 100644
index 00..1d33b7e39c
--- /dev/null
+++ b/meta/files/overlayfs-unit.mount.in
@@ -0,0 +1,13 @@
+[Unit]
+Description=Overlayfs mount unit
+Requires={CREATE_DIRS_SERVICE}
+After={CREATE_DIRS_SERVICE}
+
+[Mount]
+What=overlay
+Where={LOWERDIR}
+Type=overlay
+Options=lowerdir={LOWERDIR},upperdir={DATA_MOUNT_POINT}/upper{LOWERDIR},workdir={DATA_MOUNT_POINT}/workdir{LOWERDIR}
+
+[Install]
+WantedBy=multi-user.target
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159506):
https://lists.openembedded.org/g/openembedded-core/message/159506
Mute This Topic: https://lists.openembedded.org/mt/87634124/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 4/8] image: add overlayfs-etc image feature
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/image.bbclass | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 8a46b4852c..2b0ce4a988 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -15,6 +15,7 @@ IMGCLASSES += "${@bb.utils.contains('IMAGE_FSTYPES',
'container', 'image-contain
IMGCLASSES += "image_types_wic"
IMGCLASSES += "rootfs-postcommands"
IMGCLASSES += "image-postinst-intercepts"
+IMGCLASSES += "overlayfs-etc"
inherit ${IMGCLASSES}
TOOLCHAIN_TARGET_TASK += "${PACKAGE_INSTALL}"
@@ -33,7 +34,7 @@ INHIBIT_DEFAULT_DEPS = "1"
# IMAGE_FEATURES may contain any available package group
IMAGE_FEATURES ?= ""
IMAGE_FEATURES[type] = "list"
-IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs
read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password
allow-empty-password allow-root-login post-install-logging"
+IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs
read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password
allow-empty-password allow-root-login post-install-logging overlayfs-etc"
# Generate companion debugfs?
IMAGE_GEN_DEBUGFS ?= "0"
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159502):
https://lists.openembedded.org/g/openembedded-core/message/159502
Mute This Topic: https://lists.openembedded.org/mt/87634119/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 5/8] oeqa/selftest: overlayfs helper function
Move helper functions out of class scope so they can be used in other tests
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/overlayfs.py | 32 +++
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index 84242a1605..43415778ce 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -5,16 +5,16 @@
from oeqa.selftest.case import OESelftestTestCase
from oeqa.utils.commands import runCmd, bitbake, get_bb_var, runqemu
-class OverlayFSTests(OESelftestTestCase):
-"""Overlayfs class usage tests"""
+def getline_qemu(out, line):
+for l in out.split('\n'):
+if line in l:
+return l
-def getline_qemu(self, out, line):
-for l in out.split('\n'):
-if line in l:
-return l
+def getline(res, line):
+return getline_qemu(res.output, line)
-def getline(self, res, line):
-return self.getline_qemu(res.output, line)
+class OverlayFSTests(OESelftestTestCase):
+"""Overlayfs class usage tests"""
def add_overlay_conf_to_machine(self):
machine_inc = """
@@ -40,7 +40,7 @@ inherit overlayfs
self.write_recipeinc('overlayfs-user', overlayfs_recipe_append)
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "overlayfs-user was skipped: missing required
distro features")
+line = getline(res, "overlayfs-user was skipped: missing required
distro features")
self.assertTrue("overlayfs" in res.output, msg=res.output)
self.assertTrue("systemd" in res.output, msg=res.output)
self.assertTrue("ERROR: Required build target 'core-image-minimal' has
no buildable providers." in res.output, msg=res.output)
@@ -61,9 +61,9 @@ DISTRO_FEATURES += "systemd overlayfs"
self.add_overlay_conf_to_machine()
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "Unit name mnt-overlay.mount not found in
systemd unit directories")
+line = getline(res, "Unit name mnt-overlay.mount not found in systemd
unit directories")
self.assertTrue(line and line.startswith("WARNING:"), msg=res.output)
-line = self.getline(res, "Not all mount units are installed by the
BSP")
+line = getline(res, "Not all mount units are installed by the BSP")
self.assertTrue(line and line.startswith("ERROR:"), msg=res.output)
def test_mount_unit_not_set(self):
@@ -81,7 +81,7 @@ DISTRO_FEATURES += "systemd overlayfs"
self.write_config(config)
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "A recipe uses overlayfs class but there is
no OVERLAYFS_MOUNT_POINT set in your MACHINE configuration")
+line = getline(res, "A recipe uses overlayfs class but there is no
OVERLAYFS_MOUNT_POINT set in your MACHINE configuration")
self.assertTrue(line and line.startswith("Parsing recipes...ERROR:"),
msg=res.output)
def test_wrong_mount_unit_set(self):
@@ -104,7 +104,7 @@ OVERLAYFS_MOUNT_POINT[usr-share-overlay] =
"/usr/share/overlay"
self.set_machine_config(wrong_machine_config)
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "Missing required mount point for
OVERLAYFS_MOUNT_POINT[mnt-overlay] in your MACHINE configuration")
+line = getline(res, "Missing required mount point for
OVERLAYFS_MOUNT_POINT[mnt-overlay] in your MACHINE configuration")
self.assertTrue(line and line.startswith("Parsing recipes...ERROR:"),
msg=res.output)
def test_correct_image(self):
@@ -201,11 +201,11 @@ EOT
# /usr/share/my-application as an overlay (see overlayfs-user
recipe)
status, output = qemu.run_serial("/bin/mount -t tmpfs,overlay")
-line = self.getline_qemu(output, "on /mnt/overlay")
+line = getline_qemu(output, "on /mnt/overlay")
self.assertTrue(line and line.startswith("tmpfs"), msg=output)
-line = self.getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/my-application")
+line = getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/my-application")
self.assertTrue(line and line.startswith("overlay"), msg=output)
-line = self.getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/another-overlay-mount")
+line = getline_qemu(output,
"upperdir=/mnt/o
[OE-core] [PATCH 3/8] wic: image for overlayfs-etc tests
Introduce wic image for overlayfs-etc tests with a dedicated /data
partition and configurable kernel parameters
Signed-off-by: Vyacheslav Yurkov
---
meta-selftest/wic/overlayfs_etc.wks.in | 4
1 file changed, 4 insertions(+)
create mode 100644 meta-selftest/wic/overlayfs_etc.wks.in
diff --git a/meta-selftest/wic/overlayfs_etc.wks.in
b/meta-selftest/wic/overlayfs_etc.wks.in
new file mode 100644
index 00..1e1e5836e7
--- /dev/null
+++ b/meta-selftest/wic/overlayfs_etc.wks.in
@@ -0,0 +1,4 @@
+part /boot --active --source bootimg-biosplusefi --ondisk sda
--sourceparams="loader=grub-efi" --align 1024
+part / --source rootfs --ondisk sda --fstype=ext4 --use-uuid --align 1024
+part --ondisk sda --fstype=ext4 --size=5 --align 1024
+bootloader --ptable gpt --timeout=1 --append="rootfstype=ext4
console=ttyS0,115200 console=tty0 ${OVERLAYFS_INIT_OPTION}"
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159501):
https://lists.openembedded.org/g/openembedded-core/message/159501
Mute This Topic: https://lists.openembedded.org/mt/87634118/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/8] files: add preinit.sh.in
A template init script for overlayfs-etc class
Signed-off-by: Vyacheslav Yurkov
---
meta/files/preinit.sh.in | 29 +
1 file changed, 29 insertions(+)
create mode 100644 meta/files/preinit.sh.in
diff --git a/meta/files/preinit.sh.in b/meta/files/preinit.sh.in
new file mode 100644
index 00..2ebb6c9224
--- /dev/null
+++ b/meta/files/preinit.sh.in
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+echo "PREINIT: Start"
+
+PATH=/sbin:/bin:/usr/sbin:/usr/bin
+mount -o remount,rw /
+
+mkdir -p /proc
+mkdir -p /sys
+mkdir -p /run
+mkdir -p /var/run
+
+mount -t proc proc /proc
+mount -t sysfs sysfs /sys
+
+[ -z "$CONSOLE" ] && CONSOLE="/dev/console"
+
+mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}
+if mount -n -t {OVERLAYFS_ETC_FSTYPE} -o {OVERLAYFS_ETC_MOUNT_OPTIONS}
{OVERLAYFS_ETC_DEVICE} {OVERLAYFS_ETC_MOUNT_POINT}
+then
+mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper
+mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/work
+mount -n -t overlay -o
upperdir={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper,lowerdir=/etc,workdir={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/work
{OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper /etc || echo "PREINIT: Mounting
etc-overlay failed!"
+else
+echo "PREINIT: Mounting failed!"
+fi
+
+echo "PREINIT: done; starting "
+exec {SBIN_INIT_NAME}
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159499):
https://lists.openembedded.org/g/openembedded-core/message/159499
Mute This Topic: https://lists.openembedded.org/mt/87634116/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/8] overlayfs-etc: mount etc as overlayfs
This class provides an image feature that mounts /etc as an overlayfs
file system. This is an extension for existing overlayfs class, which
doesn't support /etc
Signed-off-by: Alfred Schapansky
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/overlayfs-etc.bbclass | 76 ++
1 file changed, 76 insertions(+)
create mode 100644 meta/classes/overlayfs-etc.bbclass
diff --git a/meta/classes/overlayfs-etc.bbclass
b/meta/classes/overlayfs-etc.bbclass
new file mode 100644
index 00..5e89339269
--- /dev/null
+++ b/meta/classes/overlayfs-etc.bbclass
@@ -0,0 +1,76 @@
+# Class for setting up /etc in overlayfs
+#
+# In order to have /etc directory in overlayfs a special handling at early
boot stage is required
+# The idea is to supply a custom init script that mounts /etc before launching
actual init program,
+# because the latter already requires /etc to be mounted
+#
+# The configuration must be machine specific. You should at least set these
three variables:
+# OVERLAYFS_ETC_MOUNT_POINT ?= "/data"
+# OVERLAYFS_ETC_FSTYPE ?= "ext4"
+# OVERLAYFS_ETC_DEVICE ?= "/dev/mmcblk0p2"
+#
+# To control more mount options you should consider setting mount options:
+# OVERLAYFS_ETC_MOUNT_OPTIONS ?= "defaults"
+#
+# The class provides two options for /sbin/init generation
+# 1. Default option is to rename original /sbin/init to /sbin/init.orig and
place generated init under
+#original name, i.e. /sbin/init. It has an advantage that you won't need
to change any kernel
+#parameters in order to make it work, but it poses a restriction that
package-management can't
+#be used, becaause updating init manager would remove generated script
+# 2. If you are would like to keep original init as is, you can set
+#OVERLAYFS_ETC_USE_ORIG_INIT_NAME = "0"
+#Then generated init will be named /sbin/preinit and you would need to
extend you kernel parameters
+#manually in your bootloader configuration.
+#
+# Regardless which mode you choose, update and migration strategy of
configuration files under /etc
+# overlay is out of scope of this class
+
+ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("IMAGE_FEATURES",
"overlayfs-etc", "create_overlayfs_etc_preinit;", "", d)}'
+IMAGE_FEATURES_CONFLICTS_overlayfs-etc = "package-management"
+
+OVERLAYFS_ETC_MOUNT_POINT ??= ""
+OVERLAYFS_ETC_FSTYPE ??= ""
+OVERLAYFS_ETC_DEVICE ??= ""
+OVERLAYFS_ETC_USE_ORIG_INIT_NAME ??= "1"
+OVERLAYFS_ETC_MOUNT_OPTIONS ??= "defaults"
+OVERLAYFS_ETC_INIT_TEMPLATE ??= "${COREBASE}/meta/files/preinit.sh.in"
+
+python create_overlayfs_etc_preinit() {
+overlayEtcMountPoint = d.getVar("OVERLAYFS_ETC_MOUNT_POINT")
+overlayEtcFsType = d.getVar("OVERLAYFS_ETC_FSTYPE")
+overlayEtcDevice = d.getVar("OVERLAYFS_ETC_DEVICE")
+
+if not overlayEtcMountPoint:
+bb.fatal("OVERLAYFS_ETC_MOUNT_POINT must be set in your MACHINE
configuration")
+if not overlayEtcDevice:
+bb.fatal("OVERLAYFS_ETC_DEVICE must be set in your MACHINE
configuration")
+if not overlayEtcFsType:
+bb.fatal("OVERLAYFS_ETC_FSTYPE should contain a valid file system type
on {0}".format(overlayEtcDevice))
+
+with open(d.getVar("OVERLAYFS_ETC_INIT_TEMPLATE"), "r") as f:
+PreinitTemplate = f.read()
+
+useOrigInit =
oe.types.boolean(d.getVar('OVERLAYFS_ETC_USE_ORIG_INIT_NAME'))
+preinitPath = oe.path.join(d.getVar("IMAGE_ROOTFS"),
d.getVar("base_sbindir"), "preinit")
+initBaseName = oe.path.join(d.getVar("base_sbindir"), "init")
+origInitNameSuffix = ".orig"
+
+args = {
+'OVERLAYFS_ETC_MOUNT_POINT': overlayEtcMountPoint,
+'OVERLAYFS_ETC_MOUNT_OPTIONS': d.getVar('OVERLAYFS_ETC_MOUNT_OPTIONS'),
+'OVERLAYFS_ETC_FSTYPE': overlayEtcFsType,
+'OVERLAYFS_ETC_DEVICE': overlayEtcDevice,
+'SBIN_INIT_NAME': initBaseName + origInitNameSuffix if useOrigInit
else initBaseName
+}
+
+if useOrigInit:
+# rename original /sbin/init
+origInit = oe.path.join(d.getVar("IMAGE_ROOTFS"), initBaseName)
+bb.debug(1, "rootfs path %s, init path %s, test %s" %
(d.getVar('IMAGE_ROOTFS'), origInit, d.getVar("IMAGE_ROOTFS")))
+bb.utils.rename(origInit, origInit + origInitNameSuffix)
+preinitPath = origInit
+
+with open(preinitPath, 'w') as f:
+f.write(PreinitTemplate.format(**args))
+os.chmod(preinitPath, 0o755)
+}
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159500):
https://lists.openembedded.org/g/openembedded-core/message/159500
Mute This Topic: https://lists.openembedded.org/mt/87634117/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 0/8] Pull request (cover letter only)
This is a V2 of overlayfs-etc image feature implementation, that allows to setup the whole /etc under overlayfs. Please review and merge if you find it OK The following changes since commit 1a6c2a7345199d77ad5aeac8ad337ed80a8aa39b: build-appliance-image: Update to master head revision (2021-12-09 13:56:07 +) are available in the Git repository at: git://github.com/UVV-gh/openembedded-core feature/overlayfs-etc https://github.com/UVV-gh/openembedded-core/tree/feature/overlayfs-etc Vyacheslav Yurkov (8): files: add preinit.sh.in overlayfs-etc: mount etc as overlayfs wic: image for overlayfs-etc tests image: add overlayfs-etc image feature oeqa/selftest: overlayfs helper function oeqa/selftest: unit tests for overlayfs-etc overlayfs: update notes on /etc overlayfs: move templates to files directory meta-selftest/wic/overlayfs_etc.wks.in | 4 + meta/classes/image.bbclass | 3 +- meta/classes/overlayfs-etc.bbclass | 76 +++ meta/classes/overlayfs.bbclass | 53 + meta/files/overlayfs-all-overlays.service.in | 12 ++ meta/files/overlayfs-create-dirs.service.in | 14 ++ meta/files/overlayfs-unit.mount.in | 13 ++ meta/files/preinit.sh.in | 29 +++ meta/lib/oeqa/selftest/cases/overlayfs.py| 216 +-- 9 files changed, 361 insertions(+), 59 deletions(-) create mode 100644 meta-selftest/wic/overlayfs_etc.wks.in create mode 100644 meta/classes/overlayfs-etc.bbclass create mode 100644 meta/files/overlayfs-all-overlays.service.in create mode 100644 meta/files/overlayfs-create-dirs.service.in create mode 100644 meta/files/overlayfs-unit.mount.in create mode 100644 meta/files/preinit.sh.in -- 2.28.0 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#159498): https://lists.openembedded.org/g/openembedded-core/message/159498 Mute This Topic: https://lists.openembedded.org/mt/87634115/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH 1/6] overlayfs-etc: mount etc as overlayfs
On 09.12.2021 12:08, Richard Purdie wrote: The second thing which I don't really like is this mix of shell and python, it is very confusing to read. I'd much rather we put template files into meta/lib/files/ and then read them from there for the variable substitutions. We do this for other scripts. I appreciate the overlayfs class merged without that but I think we need to fix this there as well. Thanks for the feedback, Richard. I see only meta/files. I guess you meant this directory. Will do the same for overlayfs.bbclass too. Adding the tests in this series is really great through, thanks for that! Cheers, Richard Regards, Vyacheslav -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#159455): https://lists.openembedded.org/g/openembedded-core/message/159455 Mute This Topic: https://lists.openembedded.org/mt/87162487/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH 1/6] overlayfs-etc: mount etc as overlayfs
On 09.12.2021 11:59, Ross Burton wrote: On Fri, 19 Nov 2021 at 06:15, Vyacheslav Yurkov wrote: +# Class for setting up /etc in overlayfs +# +# In order to have /etc directory in overlayfs a special handling at early boot stage is required +# The idea is to supply a custom init script that mounts /etc before launching actual init program, +# because the latter already requires /etc to be mounted Can you elaborate on the exact scenario that is being supported here? Is it read-only / with /etc (and potentially more) in an overlayfs? Correct, that's a read-only / with only /etc in an overlayfs. For other directories (not under /etc) there's a systemd based solution in overlayfs.bbclass. +OVERLAYFS_ETC_MOUNT_POINT ?= "/data" +OVERLAYFS_ETC_FSTYPE ?= "ext4" +OVERLAYFS_ETC_DEVICE ?= "/dev/mmcblk0p2" I'm guessing these are not universal defaults, but the values that you use. If you can't provide default values which will actually work out of the box, just set them to "" and the class can abort if they're not set. Ross Thanks for the feedback. Will fix that. Vyacheslav -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#159454): https://lists.openembedded.org/g/openembedded-core/message/159454 Mute This Topic: https://lists.openembedded.org/mt/87162487/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] patch files in a directory aren't applied
Hi Max,
I think you are interpreting it not entirely correctly. You are reading
do_patch() description and the task would pick up all patches it finds.
But SRC_URI is another thing and it is looked at during parsing time.
Vyacheslav
On 01.12.2021 15:26, Max Krummenacher wrote:
Hello
I tried to add patches by specifing a directory in SRC_URI and have
bitbake appling all patch files in it without explicitely adding the
individual file names in SRC_URI.
According to the do_patch description [1] that should work, however
for my example recipe it does not. The files in the directory end up
in ${WORKDIR}/path_to_lots_of_patch_files/ but none of the *.patch
files are applied.
Looking at meta/lib/oe/patch.py src_patches() I believe that the
documented behaviour is not implemented.
Does anyone know why this is documented but not implemented?
Has it been removed for good reason and is now undesired in OE?
I'd be willing to work on the feature.
Max
[1]
https://www.yoctoproject.org/docs/current/mega-manual/mega-manual.html#ref-tasks-patch
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159034):
https://lists.openembedded.org/g/openembedded-core/message/159034
Mute This Topic: https://lists.openembedded.org/mt/87429103/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH 0/6] Pull request (cover letter only)
Just pinging to check if anyone had a chance to take a look at it Thanks, Vyacheslav On 19.11.2021 07:15, Vyacheslav Yurkov via lists.openembedded.org wrote: This is a V1 of overlayfs-etc image feature implementation, that allows to setup the whole /etc under overlayfs. Please review and merge if you find it OK The following changes since commit 0d15632f3db787d3f08eb260732567e62f52ffb3: libtasn1: upgrade 4.17.0 -> 4.18.0 (2021-11-16 22:19:47 +) are available in the Git repository at: git://github.com/UVV-gh/openembedded-core feature/overlayfs-etc https://github.com/UVV-gh/openembedded-core/tree/feature/overlayfs-etc Vyacheslav Yurkov (6): overlayfs-etc: mount etc as overlayfs wic: image for overlayfs-etc tests image: add overlayfs-etc image feature oeqa/selftest: overlayfs helper function oeqa/selftest: unit tests for overlayfs-etc overlayfs: update notes on /etc meta-selftest/wic/overlayfs_etc.wks.in| 4 + meta/classes/image.bbclass| 3 +- meta/classes/overlayfs-etc.bbclass| 93 meta/classes/overlayfs.bbclass| 1 + meta/lib/oeqa/selftest/cases/overlayfs.py | 173 -- 5 files changed, 257 insertions(+), 17 deletions(-) create mode 100644 meta-selftest/wic/overlayfs_etc.wks.in create mode 100644 meta/classes/overlayfs-etc.bbclass -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#158972): https://lists.openembedded.org/g/openembedded-core/message/158972 Mute This Topic: https://lists.openembedded.org/mt/87162486/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [docs] [PATCH] dev-manual: how to purge duplicate sstate cache files
Hi Michael On 23.11.2021 10:14, Michael Opdenacker wrote: Greetings, On 11/19/21 5:06 PM, Michael Opdenacker wrote: Signed-off-by: Michael Opdenacker --- documentation/dev-manual/common-tasks.rst | 25 ++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/documentation/dev-manual/common-tasks.rst b/documentation/dev-manual/common-tasks.rst index 3eead147a3..37612c84b7 100644 --- a/documentation/dev-manual/common-tasks.rst +++ b/documentation/dev-manual/common-tasks.rst @@ -6242,8 +6242,11 @@ Changing the listed common targets is as easy as editing your version of ``conf-notes.txt`` in your custom template configuration directory and making sure you have ``TEMPLATECONF`` set to your directory. +Conserving Disk Space += + Conserving Disk Space During Builds -=== +--- To help conserve disk space during builds, you can add the following statement to your project's ``local.conf`` configuration file found in @@ -6257,6 +6260,26 @@ building a recipe once the recipe is built. For more information on :ref:`rm_work ` class in the Yocto Project Reference Manual. +Purging Duplicate Shared State Cache Files +--- + +After multiple build iterations, the Shared State (sstate) cache can contain +duplicate cache files for a given package, while only the most recent one +is likely to be reusable. The following command purges all but the +newest sstate cache file for each package:: + + sstate-cache-management.sh --remove-duplicated --cache-dir=build/sstate-cache + +This command will ask you to confirm the deletions it identifies. + +Note:: + + The duplicated sstate cache files of one package must have the same + architecture, which means that sstate cache files with multiple + architectures are not considered as duplicate. + +Run ``sstate-cache-management.sh`` for more details about this script. + Working with Packages = Did anyone have time to have a look at what I wrote ? There may be better ways to purge the sstate cache files. Thanks in advance Michael. The description looks good to me. By other ways you mean other usage scenarios of sstate-cache-management.sh script or by means of something else? Vyacheslav -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#158827): https://lists.openembedded.org/g/openembedded-core/message/158827 Mute This Topic: https://lists.openembedded.org/mt/87255323/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] rootfs-postcommands: update systemd_create_users
Process all systemd-sysusers configuration files in order to create
users/groups at build time. systemd-sysusers would try to create them at
run-time, but for read-only rootfs that's not possible and results in
warnings from different services:
systemd-udevd[166]: /lib/udev/rules.d/50-udev-default.rules:39 Unknown group
'render', ignoring
systemd-udevd[166]: /lib/udev/rules.d/50-udev-default.rules:40 Unknown group
'render', ignoring
systemd-udevd[166]: /lib/udev/rules.d/50-udev-default.rules:42 Unknown group
'sgx', ignoring
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/rootfs-postcommands.bbclass | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/classes/rootfs-postcommands.bbclass
b/meta/classes/rootfs-postcommands.bbclass
index 7fe9e3d8c8..a3f96ef7ed 100644
--- a/meta/classes/rootfs-postcommands.bbclass
+++ b/meta/classes/rootfs-postcommands.bbclass
@@ -62,7 +62,7 @@ python () {
}
systemd_create_users () {
- for conffile in ${IMAGE_ROOTFS}/usr/lib/sysusers.d/systemd.conf
${IMAGE_ROOTFS}/usr/lib/sysusers.d/systemd-remote.conf; do
+ for conffile in ${IMAGE_ROOTFS}/usr/lib/sysusers.d/*.conf; do
[ -e $conffile ] || continue
grep -v "^#" $conffile | sed -e '/^$/d' | while read type name
id comment; do
if [ "$type" = "u" ]; then
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#158527):
https://lists.openembedded.org/g/openembedded-core/message/158527
Mute This Topic: https://lists.openembedded.org/mt/87178494/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 5/6] oeqa/selftest: unit tests for overlayfs-etc
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/overlayfs.py | 141 ++
1 file changed, 141 insertions(+)
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index 43415778ce..4623215a47 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -209,3 +209,144 @@ EOT
line = getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/another-overlay-mount")
self.assertTrue(line and line.startswith("overlay"), msg=output)
+
+class OverlayFSEtcRunTimeTests(OESelftestTestCase):
+"""overlayfs-etc class tests"""
+
+def test_image_feature_conflict(self):
+"""
+Summary: Overlayfs-etc is not allowed to be used with
package-management
+ Expected: Feature conflict
+Author:Vyacheslav Yurkov
+"""
+
+config = """
+DISTRO_FEATURES += "systemd"
+
+# Use systemd as init manager
+VIRTUAL-RUNTIME_init_manager = "systemd"
+
+# enable overlayfs in the kernel
+KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
+EXTRA_IMAGE_FEATURES += "overlayfs-etc"
+EXTRA_IMAGE_FEATURES += "package-management"
+"""
+
+self.write_config(config)
+
+res = bitbake('core-image-minimal', ignore_status=True)
+line = getline(res, "contains conflicting IMAGE_FEATURES")
+self.assertTrue("overlayfs-etc" in res.output, msg=res.output)
+self.assertTrue("package-management" in res.output, msg=res.output)
+
+def test_image_feature_is_missing_class_included(self):
+configAppend = """
+INHERIT += "overlayfs-etc"
+"""
+self.run_check_image_feature(configAppend)
+
+def test_image_feature_is_missing(self):
+self.run_check_image_feature()
+
+def run_check_image_feature(self, appendToConfig=""):
+ """
+Summary: Overlayfs-etc class is not applied when image feature is
not set
+ even if we inherit it directly,
+Expected: Image is created successfully but /etc is not an overlay
+Author:Vyacheslav Yurkov
+"""
+
+config = f"""
+DISTRO_FEATURES += "systemd"
+
+# Use systemd as init manager
+VIRTUAL-RUNTIME_init_manager = "systemd"
+
+# enable overlayfs in the kernel
+KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
+
+IMAGE_FSTYPES += "wic"
+WKS_FILE = "overlayfs_etc.wks.in"
+
+EXTRA_IMAGE_FEATURES += "read-only-rootfs"
+# Image configuration for overlayfs-etc
+OVERLAYFS_ETC_MOUNT_POINT = "/data"
+OVERLAYFS_ETC_DEVICE = "/dev/sda3"
+{appendToConfig}
+"""
+
+self.write_config(config)
+
+bitbake('core-image-minimal')
+
+with runqemu('core-image-minimal', image_fstype='wic') as qemu:
+status, output = qemu.run_serial("/bin/mount")
+
+line = getline_qemu(output, "upperdir=/data/overlay-etc/upper")
+self.assertFalse(line, msg=output)
+
+def test_sbin_init_preinit(self):
+self.run_sbin_init(False)
+
+def test_sbin_init_original(self):
+self.run_sbin_init(True)
+
+def run_sbin_init(self, origInit):
+"""
+Summary: Confirm we can replace original init and mount overlay on
top of /etc
+Expected: Image is created successfully and /etc is mounted as an
overlay
+Author:Vyacheslav Yurkov
+"""
+
+config = """
+DISTRO_FEATURES += "systemd"
+
+# Use systemd as init manager
+VIRTUAL-RUNTIME_init_manager = "systemd"
+
+# enable overlayfs in the kernel
+KERNEL_EXTRA_FEATURES:append = " features/overlayfs/overlayfs.scc"
+
+IMAGE_FSTYPES += "wic"
+OVERLAYFS_INIT_OPTION = "{OVERLAYFS_INIT_OPTION}"
+WKS_FILE = "overlayfs_etc.wks.in"
+
+EXTRA_IMAGE_FEATURES += "read-only-rootfs"
+# Image configuration for overlayfs-etc
+EXTRA_IMAGE_FEATURES += "overlayfs-etc"
+IMAGE_FEATURES:remove = "package-management"
+OVERLAYFS_ETC_MOUNT_POINT = "/data"
+OVERLAYFS_ETC_DEVICE = "/dev/sda3"
+OVERLAYFS_ETC_USE_ORIG_INIT_NAME = "{OVERLAYFS_ETC_USE_ORIG_INIT_NAME}"
+"""
+
+args = {
+'OVERLAYFS_INIT_OPTION': "" if origInit else "init=/sbin/preinit",
+'OVERLAYFS_ETC_USE_ORIG_INIT_NAME': int(origInit == True)
+}
+
+self.write_config(config.format(**args))
+
+bitbake('core-image-minimal')
+testFile = "/et
[OE-core] [PATCH 6/6] overlayfs: update notes on /etc
Signed-off-by: Vyacheslav Yurkov --- meta/classes/overlayfs.bbclass | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass index 3c0f4dc882..f1b8086ea8 100644 --- a/meta/classes/overlayfs.bbclass +++ b/meta/classes/overlayfs.bbclass @@ -31,6 +31,7 @@ # OVERLAYFS_WRITABLE_PATHS[mnt-overlay] = "/usr/share/another-application" # # Note: the class does not support /etc directory itself, because systemd depends on it +# For /etc directory use overlayfs-etc class REQUIRED_DISTRO_FEATURES += "systemd overlayfs" -- 2.28.0 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#158498): https://lists.openembedded.org/g/openembedded-core/message/158498 Mute This Topic: https://lists.openembedded.org/mt/87162492/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 4/6] oeqa/selftest: overlayfs helper function
Move helper functions out of class scope so they can be used in other tests
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/overlayfs.py | 32 +++
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index 84242a1605..43415778ce 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -5,16 +5,16 @@
from oeqa.selftest.case import OESelftestTestCase
from oeqa.utils.commands import runCmd, bitbake, get_bb_var, runqemu
-class OverlayFSTests(OESelftestTestCase):
-"""Overlayfs class usage tests"""
+def getline_qemu(out, line):
+for l in out.split('\n'):
+if line in l:
+return l
-def getline_qemu(self, out, line):
-for l in out.split('\n'):
-if line in l:
-return l
+def getline(res, line):
+return getline_qemu(res.output, line)
-def getline(self, res, line):
-return self.getline_qemu(res.output, line)
+class OverlayFSTests(OESelftestTestCase):
+"""Overlayfs class usage tests"""
def add_overlay_conf_to_machine(self):
machine_inc = """
@@ -40,7 +40,7 @@ inherit overlayfs
self.write_recipeinc('overlayfs-user', overlayfs_recipe_append)
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "overlayfs-user was skipped: missing required
distro features")
+line = getline(res, "overlayfs-user was skipped: missing required
distro features")
self.assertTrue("overlayfs" in res.output, msg=res.output)
self.assertTrue("systemd" in res.output, msg=res.output)
self.assertTrue("ERROR: Required build target 'core-image-minimal' has
no buildable providers." in res.output, msg=res.output)
@@ -61,9 +61,9 @@ DISTRO_FEATURES += "systemd overlayfs"
self.add_overlay_conf_to_machine()
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "Unit name mnt-overlay.mount not found in
systemd unit directories")
+line = getline(res, "Unit name mnt-overlay.mount not found in systemd
unit directories")
self.assertTrue(line and line.startswith("WARNING:"), msg=res.output)
-line = self.getline(res, "Not all mount units are installed by the
BSP")
+line = getline(res, "Not all mount units are installed by the BSP")
self.assertTrue(line and line.startswith("ERROR:"), msg=res.output)
def test_mount_unit_not_set(self):
@@ -81,7 +81,7 @@ DISTRO_FEATURES += "systemd overlayfs"
self.write_config(config)
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "A recipe uses overlayfs class but there is
no OVERLAYFS_MOUNT_POINT set in your MACHINE configuration")
+line = getline(res, "A recipe uses overlayfs class but there is no
OVERLAYFS_MOUNT_POINT set in your MACHINE configuration")
self.assertTrue(line and line.startswith("Parsing recipes...ERROR:"),
msg=res.output)
def test_wrong_mount_unit_set(self):
@@ -104,7 +104,7 @@ OVERLAYFS_MOUNT_POINT[usr-share-overlay] =
"/usr/share/overlay"
self.set_machine_config(wrong_machine_config)
res = bitbake('core-image-minimal', ignore_status=True)
-line = self.getline(res, "Missing required mount point for
OVERLAYFS_MOUNT_POINT[mnt-overlay] in your MACHINE configuration")
+line = getline(res, "Missing required mount point for
OVERLAYFS_MOUNT_POINT[mnt-overlay] in your MACHINE configuration")
self.assertTrue(line and line.startswith("Parsing recipes...ERROR:"),
msg=res.output)
def test_correct_image(self):
@@ -201,11 +201,11 @@ EOT
# /usr/share/my-application as an overlay (see overlayfs-user
recipe)
status, output = qemu.run_serial("/bin/mount -t tmpfs,overlay")
-line = self.getline_qemu(output, "on /mnt/overlay")
+line = getline_qemu(output, "on /mnt/overlay")
self.assertTrue(line and line.startswith("tmpfs"), msg=output)
-line = self.getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/my-application")
+line = getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/my-application")
self.assertTrue(line and line.startswith("overlay"), msg=output)
-line = self.getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/another-overlay-mount")
+line = getline_qemu(output,
"upperdir=/mnt/o
[OE-core] [PATCH 3/6] image: add overlayfs-etc image feature
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/image.bbclass | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index 6c759fdf70..e198f041b1 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -15,6 +15,7 @@ IMGCLASSES += "${@bb.utils.contains('IMAGE_FSTYPES',
'container', 'image-contain
IMGCLASSES += "image_types_wic"
IMGCLASSES += "rootfs-postcommands"
IMGCLASSES += "image-postinst-intercepts"
+IMGCLASSES += "overlayfs-etc"
inherit ${IMGCLASSES}
TOOLCHAIN_TARGET_TASK += "${PACKAGE_INSTALL}"
@@ -33,7 +34,7 @@ INHIBIT_DEFAULT_DEPS = "1"
# IMAGE_FEATURES may contain any available package group
IMAGE_FEATURES ?= ""
IMAGE_FEATURES[type] = "list"
-IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs
read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password
allow-empty-password allow-root-login post-install-logging"
+IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs
read-only-rootfs-delayed-postinsts stateless-rootfs empty-root-password
allow-empty-password allow-root-login post-install-logging overlayfs-etc"
# Generate companion debugfs?
IMAGE_GEN_DEBUGFS ?= "0"
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#158495):
https://lists.openembedded.org/g/openembedded-core/message/158495
Mute This Topic: https://lists.openembedded.org/mt/87162489/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/6] wic: image for overlayfs-etc tests
Introduce wic image for overlayfs-etc tests with a dedicated /data
partition and configurable kernel parameters
Signed-off-by: Vyacheslav Yurkov
---
meta-selftest/wic/overlayfs_etc.wks.in | 4
1 file changed, 4 insertions(+)
create mode 100644 meta-selftest/wic/overlayfs_etc.wks.in
diff --git a/meta-selftest/wic/overlayfs_etc.wks.in
b/meta-selftest/wic/overlayfs_etc.wks.in
new file mode 100644
index 00..1e1e5836e7
--- /dev/null
+++ b/meta-selftest/wic/overlayfs_etc.wks.in
@@ -0,0 +1,4 @@
+part /boot --active --source bootimg-biosplusefi --ondisk sda
--sourceparams="loader=grub-efi" --align 1024
+part / --source rootfs --ondisk sda --fstype=ext4 --use-uuid --align 1024
+part --ondisk sda --fstype=ext4 --size=5 --align 1024
+bootloader --ptable gpt --timeout=1 --append="rootfstype=ext4
console=ttyS0,115200 console=tty0 ${OVERLAYFS_INIT_OPTION}"
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#158494):
https://lists.openembedded.org/g/openembedded-core/message/158494
Mute This Topic: https://lists.openembedded.org/mt/87162488/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/6] overlayfs-etc: mount etc as overlayfs
This class provides an image feature that mounts /etc as an overlayfs
file system. This is an extension for existing overlayfs class, which
doesn't support /etc
Signed-off-by: Alfred Schapansky
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/overlayfs-etc.bbclass | 93 ++
1 file changed, 93 insertions(+)
create mode 100644 meta/classes/overlayfs-etc.bbclass
diff --git a/meta/classes/overlayfs-etc.bbclass
b/meta/classes/overlayfs-etc.bbclass
new file mode 100644
index 00..78caf0211b
--- /dev/null
+++ b/meta/classes/overlayfs-etc.bbclass
@@ -0,0 +1,93 @@
+# Class for setting up /etc in overlayfs
+#
+# In order to have /etc directory in overlayfs a special handling at early
boot stage is required
+# The idea is to supply a custom init script that mounts /etc before launching
actual init program,
+# because the latter already requires /etc to be mounted
+#
+# The configuration must be machine specific. You should at least set these
two variables if you
+# are not happy with default values:
+# OVERLAYFS_ETC_MOUNT_POINT ?= "/data"
+# OVERLAYFS_ETC_DEVICE ?= "/dev/mmcblk0p2"
+#
+# To control more mount options you should consider also setting file system
type and mount options:
+# OVERLAYFS_ETC_FSTYPE ?= "ext4"
+# OVERLAYFS_ETC_MOUNT_OPTIONS ?= "defaults"
+#
+# The class provides two options for /sbin/init generation
+# 1. Default option is to rename original /sbin/init to /sbin/init.orig and
place generated init under
+#original name, i.e. /sbin/init. It has an advantage that you won't need
to change any kernel
+#parameters in order to make it work, but it poses a restriction that
package-management can't
+#be used, becaause updating init manager would remove generated script
+# 2. If you are would like to keep original init as is, you can set
+#OVERLAYFS_ETC_USE_ORIG_INIT_NAME = "0"
+#Then generated init will be named /sbin/preinit and you would need to
extend you kernel parameters
+#manually in your bootloader configuration.
+#
+# Regardless which mode you choose, update and migration strategy of
configuration files under /etc
+# overlay is out of scope of this class
+
+ROOTFS_POSTPROCESS_COMMAND += '${@bb.utils.contains("IMAGE_FEATURES",
"overlayfs-etc", "create_overlayfs_etc_preinit;", "", d)}'
+IMAGE_FEATURES_CONFLICTS_overlayfs-etc = "package-management"
+
+OVERLAYFS_ETC_MOUNT_POINT ?= "/data"
+OVERLAYFS_ETC_FSTYPE ?= "ext4"
+OVERLAYFS_ETC_DEVICE ?= "/dev/mmcblk0p2"
+OVERLAYFS_ETC_USE_ORIG_INIT_NAME ?= "1"
+OVERLAYFS_ETC_MOUNT_OPTIONS ?= "defaults"
+
+python create_overlayfs_etc_preinit() {
+PreinitTemplate = """#!/bin/sh
+
+echo "PREINIT: Start"
+
+PATH=/sbin:/bin:/usr/sbin:/usr/bin
+mount -o remount,rw /
+
+mkdir -p /proc
+mkdir -p /sys
+mkdir -p /run
+mkdir -p /var/run
+
+mount -t proc proc /proc
+mount -t sysfs sysfs /sys
+
+[ -z "$CONSOLE" ] && CONSOLE="/dev/console"
+
+mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}
+if mount -n -t {OVERLAYFS_ETC_FSTYPE} -o {OVERLAYFS_ETC_MOUNT_OPTIONS}
{OVERLAYFS_ETC_DEVICE} {OVERLAYFS_ETC_MOUNT_POINT}
+then
+mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper
+mkdir -p {OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/work
+mount -n -t overlay -o
upperdir={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper,lowerdir=/etc,workdir={OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/work
{OVERLAYFS_ETC_MOUNT_POINT}/overlay-etc/upper /etc || echo "PREINIT: Mounting
etc-overlay failed!"
+else
+echo "PREINIT: Mounting failed!"
+fi
+
+echo "PREINIT: done; starting "
+exec {SBIN_INIT_NAME}
+"""
+
+useOrigInit =
oe.types.boolean(d.getVar('OVERLAYFS_ETC_USE_ORIG_INIT_NAME'))
+preinitPath = oe.path.join(d.getVar("IMAGE_ROOTFS"),
d.getVar("base_sbindir"), "preinit")
+initBaseName = oe.path.join(d.getVar("base_sbindir"), "init")
+origInitNameSuffix = ".orig"
+
+args = {
+'OVERLAYFS_ETC_MOUNT_POINT': d.getVar('OVERLAYFS_ETC_MOUNT_POINT'),
+'OVERLAYFS_ETC_MOUNT_OPTIONS': d.getVar('OVERLAYFS_ETC_MOUNT_OPTIONS'),
+'OVERLAYFS_ETC_FSTYPE': d.getVar('OVERLAYFS_ETC_FSTYPE'),
+'OVERLAYFS_ETC_DEVICE': d.getVar('OVERLAYFS_ETC_DEVICE'),
+'SBIN_INIT_NAME': initBaseName + origInitNameSuffix if useOrigInit
else initBaseName
+}
+
+if useOrigInit:
+# rename original /sbin/init
+origInit = oe.path.join(d.getVar("IMAGE_ROOTFS"), initBaseName)
+bb.debug(1, "rootfs path %s, init path %s, test %s" %
(d.getVar('IMAGE_ROOTFS'), origInit, d.getVar("IMAGE_ROOTFS")))
+bb.utils.rename(origInit, origInit + origInitNameSuffix)
+preinitPath = origInit
+
+with open(preinitPath, 'w') as f
[OE-core] [PATCH 0/6] Pull request (cover letter only)
This is a V1 of overlayfs-etc image feature implementation, that allows to setup the whole /etc under overlayfs. Please review and merge if you find it OK The following changes since commit 0d15632f3db787d3f08eb260732567e62f52ffb3: libtasn1: upgrade 4.17.0 -> 4.18.0 (2021-11-16 22:19:47 +) are available in the Git repository at: git://github.com/UVV-gh/openembedded-core feature/overlayfs-etc https://github.com/UVV-gh/openembedded-core/tree/feature/overlayfs-etc Vyacheslav Yurkov (6): overlayfs-etc: mount etc as overlayfs wic: image for overlayfs-etc tests image: add overlayfs-etc image feature oeqa/selftest: overlayfs helper function oeqa/selftest: unit tests for overlayfs-etc overlayfs: update notes on /etc meta-selftest/wic/overlayfs_etc.wks.in| 4 + meta/classes/image.bbclass| 3 +- meta/classes/overlayfs-etc.bbclass| 93 meta/classes/overlayfs.bbclass| 1 + meta/lib/oeqa/selftest/cases/overlayfs.py | 173 -- 5 files changed, 257 insertions(+), 17 deletions(-) create mode 100644 meta-selftest/wic/overlayfs_etc.wks.in create mode 100644 meta/classes/overlayfs-etc.bbclass -- 2.28.0 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#158492): https://lists.openembedded.org/g/openembedded-core/message/158492 Mute This Topic: https://lists.openembedded.org/mt/87162486/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] Suggestion to use python-parameterized for QA
Yes, technically it would be the same. Thanks for the hint. Vyacheslav On 15.11.2021 21:56, Alexander Kanavin wrote: Can't we achieve the same by abstracting the common part into a function and defining the two tests explicitly? Alex On Mon, 15 Nov 2021 at 21:26, Vyacheslav Yurkov wrote: It's substituted further down into def test_sbin_init(self, _, origInit): parameterized.expand generates two tests out of it. The string parameter is used as a suffix for the test, the rest is used as input parameters, i.e. the first test would be called with origInit=True, the second one with origInit=False. The sample output looks like this: 2021-11-15 20:21:15,740 - oe-selftest - INFO - RESULTS - overlayfs.OverlayFSEtcRunTimeTests.test_sbin_init_0_original: PASSED (295.73s) 2021-11-15 20:21:15,740 - oe-selftest - INFO - RESULTS - overlayfs.OverlayFSEtcRunTimeTests.test_sbin_init_1_preinit: PASSED (177.97s) Vyacheslav -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#158341): https://lists.openembedded.org/g/openembedded-core/message/158341 Mute This Topic: https://lists.openembedded.org/mt/87078063/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] Suggestion to use python-parameterized for QA
It's substituted further down into
def test_sbin_init(self, _, origInit):
parameterized.expand generates two tests out of it. The string parameter
is used as a suffix for the test, the rest is used as input parameters,
i.e. the first test would be called with origInit=True, the second one
with origInit=False.
The sample output looks like this:
2021-11-15 20:21:15,740 - oe-selftest - INFO - RESULTS -
overlayfs.OverlayFSEtcRunTimeTests.test_sbin_init_0_original: PASSED
(295.73s)
2021-11-15 20:21:15,740 - oe-selftest - INFO - RESULTS -
overlayfs.OverlayFSEtcRunTimeTests.test_sbin_init_1_preinit: PASSED
(177.97s)
Vyacheslav
On 15.11.2021 21:20, Alexander Kanavin wrote:
I can't understand what is supposed to happen, perhaps because it's
incomplete? There's
@parameterized.expand([
("original", True),
("preinit", False),
])
but where is that used?
Alex
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#158313):
https://lists.openembedded.org/g/openembedded-core/message/158313
Mute This Topic: https://lists.openembedded.org/mt/87078063/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] Suggestion to use python-parameterized for QA
This is still work-in-progress, but I'm working on overlayfs-etc class, which would have different behavior depending on an input variable. You can see the example here https://github.com/UVV-gh/openembedded-core/commit/ea424efb7780dc3c85e329b36f4a675b56e86877#diff-9980b9625344e11f15451ab7c97fb1f137e94b68fa27df737c576b75e313996aR215 Regardless of the input parameter the test would be the same and should work. What do you think? Vyacheslav On 15.11.2021 20:38, Alexander Kanavin wrote: It would be helpful to provide an example of improvements that can be enabled with it. Alex -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#158311): https://lists.openembedded.org/g/openembedded-core/message/158311 Mute This Topic: https://lists.openembedded.org/mt/87078063/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] Suggestion to use python-parameterized for QA
Hi guys, I have a suggestion to add https://pypi.org/project/parameterized/ as a required package for QA selftests. Together with python unittest module it can be useful to increase code coverage and reduce duplication in case you want to do the same test with different input values. Anything speaks against it? Cheers, Vyacheslav -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#158309): https://lists.openembedded.org/g/openembedded-core/message/158309 Mute This Topic: https://lists.openembedded.org/mt/87078063/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [meta-java] why openjdk-8-native is need to build openjdk-8 ?
I believe this should be sent to openembedded-de...@lists.openembedded.org Vyacheslav On 21.10.2021 12:11, Akira Shibakawa wrote: Hi. I have a question about openjdk-8 recipe. openjdk-8 seems to be built using openjdk-8-native as boot JDK. And the openjdk-8-native seems to be built using icedtea7 as boot JDK. Why openjdk-8-native is need to build openjdk-8 ? Isn't it simpler to build openjdk-8 using icedtea7 as boot JDK? regards -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#157288): https://lists.openembedded.org/g/openembedded-core/message/157288 Mute This Topic: https://lists.openembedded.org/mt/86486039/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 5/5] overlayfs: add debug information
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/overlayfs.bbclass | 6 ++
1 file changed, 6 insertions(+)
diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass
index 9397ab44f9..3c0f4dc882 100644
--- a/meta/classes/overlayfs.bbclass
+++ b/meta/classes/overlayfs.bbclass
@@ -92,9 +92,11 @@ WantedBy=local-fs.target
'LOWERDIR': lower,
}
+bb.debug(1, "Generate systemd unit %s" % mountUnitName(lower))
with open(os.path.join(d.getVar('WORKDIR'), mountUnitName(lower)),
'w') as f:
f.write(MountUnitTemplate.format(**args))
+bb.debug(1, "Generate helper systemd unit %s" % helperUnitName(lower))
with open(os.path.join(d.getVar('WORKDIR'), helperUnitName(lower)),
'w') as f:
f.write(CreateDirsUnitTemplate.format(**args))
@@ -105,13 +107,17 @@ WantedBy=local-fs.target
'PN': d.getVar('PN')
}
+bb.debug(1, "Generate systemd unit with all overlays %s" %
allOverlaysUnitName(d))
with open(os.path.join(d.getVar('WORKDIR'), allOverlaysUnitName(d)),
'w') as f:
f.write(AllOverlaysTemplate.format(**args))
mountUnitList = []
overlayMountPoints = d.getVarFlags("OVERLAYFS_MOUNT_POINT")
for mountPoint in overlayMountPoints:
+bb.debug(1, "Process variable flag %s" % mountPoint)
for lower in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS',
mountPoint).split():
+bb.debug(1, "Prepare mount unit for %s with data mount point %s" %
+ (lower, d.getVarFlag('OVERLAYFS_MOUNT_POINT',
mountPoint)))
prepareUnits(d.getVarFlag('OVERLAYFS_MOUNT_POINT', mountPoint),
lower)
mountUnitList.append(mountUnitName(lower))
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#157030):
https://lists.openembedded.org/g/openembedded-core/message/157030
Mute This Topic: https://lists.openembedded.org/mt/86386910/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 4/5] oeqa/selftest: extend overlayfs test
Test that overlayfs.bbclass generates one systemd unit, that
applications can set dependencies on, and that this unit mounts all
required overlays
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/overlayfs.py | 42 +++
1 file changed, 42 insertions(+)
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index f0c9860b48..84242a1605 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -149,15 +149,54 @@ WantedBy=multi-user.target
EOT
}
+"""
+
+overlayfs_recipe_append = """
+OVERLAYFS_WRITABLE_PATHS[mnt-overlay] += "/usr/share/another-overlay-mount"
+
+SYSTEMD_SERVICE:${PN} += " \
+my-application.service \
+"
+
+do_install:append() {
+install -d ${D}${systemd_system_unitdir}
+cat < ${D}${systemd_system_unitdir}/my-application.service
+[Unit]
+Description=Sample application start-up unit
+After=overlayfs-user-overlays.service
+Requires=overlayfs-user-overlays.service
+
+[Service]
+Type=oneshot
+ExecStart=/bin/true
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target
+EOT
+}
"""
self.write_config(config)
self.add_overlay_conf_to_machine()
self.write_recipeinc('systemd-machine-units',
systemd_machine_unit_append)
+self.write_recipeinc('overlayfs-user', overlayfs_recipe_append)
bitbake('core-image-minimal')
with runqemu('core-image-minimal') as qemu:
+# Check that application service started
+status, output = qemu.run_serial("systemctl status my-application")
+self.assertTrue("active (exited)" in output, msg=output)
+
+# Check that overlay mounts are dependencies of our application
unit
+status, output = qemu.run_serial("systemctl list-dependencies
my-application")
+self.assertTrue("overlayfs-user-overlays.service" in output,
msg=output)
+
+status, output = qemu.run_serial("systemctl list-dependencies
overlayfs-user-overlays")
+self.assertTrue("usr-share-another\\x2doverlay\\x2dmount.mount" in
output, msg=output)
+self.assertTrue("usr-share-my\\x2dapplication.mount" in output,
msg=output)
+
# Check that we have /mnt/overlay fs mounted as tmpfs and
# /usr/share/my-application as an overlay (see overlayfs-user
recipe)
status, output = qemu.run_serial("/bin/mount -t tmpfs,overlay")
@@ -167,3 +206,6 @@ EOT
line = self.getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/my-application")
self.assertTrue(line and line.startswith("overlay"), msg=output)
+
+line = self.getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/another-overlay-mount")
+self.assertTrue(line and line.startswith("overlay"), msg=output)
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#157029):
https://lists.openembedded.org/g/openembedded-core/message/157029
Mute This Topic: https://lists.openembedded.org/mt/86386909/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 3/5] overlayfs: meta-selftest recipe fix
Avoid strict assignment in the recipe to allow overrides in .inc file
Signed-off-by: Vyacheslav Yurkov
---
meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb
b/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb
index 60405067de..913a4d1fdb 100644
--- a/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb
+++ b/meta-selftest/recipes-test/overlayfs-user/overlayfs-user.bb
@@ -8,7 +8,7 @@ EXCLUDE_FROM_WORLD = "1"
inherit ${@bb.utils.contains("DISTRO_FEATURES", "overlayfs", "overlayfs", "",
d)}
include test_recipe.inc
-OVERLAYFS_WRITABLE_PATHS[mnt-overlay] = "/usr/share/my-application"
+OVERLAYFS_WRITABLE_PATHS[mnt-overlay] += "/usr/share/my-application"
do_install() {
install -d ${D}/usr/share/my-application
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#157028):
https://lists.openembedded.org/g/openembedded-core/message/157028
Mute This Topic: https://lists.openembedded.org/mt/86386908/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/5] overlayfs: all overlays unit
Application can depend on several overlayfs mount points. Provide a
systemd unit application can depend on to make sure all overlays are
mounted before it is started to avoid any race conditions
Signed-off-by: Bruno Knittel
Signed-off-by: Vyacheslav Yurkov
---
meta/classes/overlayfs.bbclass | 36 --
meta/lib/oe/overlayfs.py | 5 +
2 files changed, 39 insertions(+), 2 deletions(-)
diff --git a/meta/classes/overlayfs.bbclass b/meta/classes/overlayfs.bbclass
index 8d9b59c9bf..9397ab44f9 100644
--- a/meta/classes/overlayfs.bbclass
+++ b/meta/classes/overlayfs.bbclass
@@ -37,6 +37,8 @@ REQUIRED_DISTRO_FEATURES += "systemd overlayfs"
inherit systemd features_check
python do_create_overlayfs_units() {
+from oe.overlayfs import mountUnitName
+
CreateDirsUnitTemplate = """[Unit]
Description=Overlayfs directories setup
Requires={DATA_MOUNT_UNIT}
@@ -65,10 +67,23 @@
Options=lowerdir={LOWERDIR},upperdir={DATA_MOUNT_POINT}/upper{LOWERDIR},workdir=
[Install]
WantedBy=multi-user.target
+"""
+AllOverlaysTemplate = """[Unit]
+Description=Groups all overlays required by {PN} in one unit
+After={ALL_OVERLAYFS_UNITS}
+Requires={ALL_OVERLAYFS_UNITS}
+
+[Service]
+Type=oneshot
+ExecStart=/bin/true
+RemainAfterExit=true
+
+[Install]
+WantedBy=local-fs.target
"""
def prepareUnits(data, lower):
-from oe.overlayfs import mountUnitName, helperUnitName
+from oe.overlayfs import helperUnitName
args = {
'DATA_MOUNT_POINT': data,
@@ -83,10 +98,27 @@ WantedBy=multi-user.target
with open(os.path.join(d.getVar('WORKDIR'), helperUnitName(lower)),
'w') as f:
f.write(CreateDirsUnitTemplate.format(**args))
+def prepareGlobalUnit(dependentUnits):
+from oe.overlayfs import allOverlaysUnitName
+args = {
+'ALL_OVERLAYFS_UNITS': " ".join(dependentUnits),
+'PN': d.getVar('PN')
+}
+
+with open(os.path.join(d.getVar('WORKDIR'), allOverlaysUnitName(d)),
'w') as f:
+f.write(AllOverlaysTemplate.format(**args))
+
+mountUnitList = []
overlayMountPoints = d.getVarFlags("OVERLAYFS_MOUNT_POINT")
for mountPoint in overlayMountPoints:
for lower in d.getVarFlag('OVERLAYFS_WRITABLE_PATHS',
mountPoint).split():
prepareUnits(d.getVarFlag('OVERLAYFS_MOUNT_POINT', mountPoint),
lower)
+mountUnitList.append(mountUnitName(lower))
+
+# set up one unit, which depends on all mount units, so users can set
+# only one dependency in their units to make sure software starts
+# when all overlays are mounted
+prepareGlobalUnit(mountUnitList)
}
# we need to generate file names early during parsing stage
@@ -95,7 +127,7 @@ python () {
unitList = unitFileList(d)
for unit in unitList:
-d.appendVar('SYSTEMD_SERVICE:' + d.getVar('PN'), ' ' + unit);
+d.appendVar('SYSTEMD_SERVICE:' + d.getVar('PN'), ' ' + unit)
d.appendVar('FILES:' + d.getVar('PN'), ' ' + strForBash(unit))
d.setVar('OVERLAYFS_UNIT_LIST', ' '.join([strForBash(s) for s in
unitList]))
diff --git a/meta/lib/oe/overlayfs.py b/meta/lib/oe/overlayfs.py
index 21ef710509..b5d5e88e80 100644
--- a/meta/lib/oe/overlayfs.py
+++ b/meta/lib/oe/overlayfs.py
@@ -15,6 +15,9 @@ def escapeSystemdUnitName(path):
def strForBash(s):
return s.replace('\\', '')
+def allOverlaysUnitName(d):
+return d.getVar('PN') + '-overlays.service'
+
def mountUnitName(unit):
return escapeSystemdUnitName(unit) + '.mount'
@@ -39,5 +42,7 @@ def unitFileList(d):
fileList.append(mountUnitName(path))
fileList.append(helperUnitName(path))
+fileList.append(allOverlaysUnitName(d))
+
return fileList
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#157026):
https://lists.openembedded.org/g/openembedded-core/message/157026
Mute This Topic: https://lists.openembedded.org/mt/86386906/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/5] oeqa/selftest: refactor common functions
Signed-off-by: Vyacheslav Yurkov
---
meta/lib/oeqa/selftest/cases/overlayfs.py | 16 +++-
1 file changed, 7 insertions(+), 9 deletions(-)
diff --git a/meta/lib/oeqa/selftest/cases/overlayfs.py
b/meta/lib/oeqa/selftest/cases/overlayfs.py
index 0184d52494..f0c9860b48 100644
--- a/meta/lib/oeqa/selftest/cases/overlayfs.py
+++ b/meta/lib/oeqa/selftest/cases/overlayfs.py
@@ -8,11 +8,14 @@ from oeqa.utils.commands import runCmd, bitbake, get_bb_var,
runqemu
class OverlayFSTests(OESelftestTestCase):
"""Overlayfs class usage tests"""
-def getline(self, res, line):
-for l in res.output.split('\n'):
+def getline_qemu(self, out, line):
+for l in out.split('\n'):
if line in l:
return l
+def getline(self, res, line):
+return self.getline_qemu(res.output, line)
+
def add_overlay_conf_to_machine(self):
machine_inc = """
OVERLAYFS_MOUNT_POINT[mnt-overlay] = "/mnt/overlay"
@@ -154,18 +157,13 @@ EOT
bitbake('core-image-minimal')
-def getline_qemu(out, line):
-for l in out.split('\n'):
-if line in l:
-return l
-
with runqemu('core-image-minimal') as qemu:
# Check that we have /mnt/overlay fs mounted as tmpfs and
# /usr/share/my-application as an overlay (see overlayfs-user
recipe)
status, output = qemu.run_serial("/bin/mount -t tmpfs,overlay")
-line = getline_qemu(output, "on /mnt/overlay")
+line = self.getline_qemu(output, "on /mnt/overlay")
self.assertTrue(line and line.startswith("tmpfs"), msg=output)
-line = getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/my-application")
+line = self.getline_qemu(output,
"upperdir=/mnt/overlay/upper/usr/share/my-application")
self.assertTrue(line and line.startswith("overlay"), msg=output)
--
2.28.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#157027):
https://lists.openembedded.org/g/openembedded-core/message/157027
Mute This Topic: https://lists.openembedded.org/mt/86386907/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH v4 2/7] overlayfs.bbclass: generate overlayfs mount units
I've already updated the docs and will submit the patch as soon as this series is approved and merged. Vyacheslav On Fri, Aug 6, 2021, 16:06 Richard Purdie < richard.pur...@linuxfoundation.org> wrote: > On Fri, 2021-08-06 at 15:00 +0200, Michael Opdenacker wrote: > > Slava, Richard, > > > > On 8/6/21 2:06 PM, Vyacheslav Yurkov wrote: > > > It's often desired in Embedded System design to have a read-only > rootfs. > > > But a lot of different applications might want to have a read-write > access > > > to some parts of a filesystem. It can be especially useful when your > update > > > mechanism overwrites the whole rootfs, but you want your application > data > > > to be preserved between updates. This class provides a way to achieve > that > > > by means of overlayfs and at the same time keeping the base rootfs > read-only. > > > > > > Signed-off-by: Vyacheslav Yurkov > > > --- > > > meta/classes/overlayfs.bbclass | 111 + > > > 1 file changed, 111 insertions(+) > > > create mode 100644 meta/classes/overlayfs.bbclass > > > > > > diff --git a/meta/classes/overlayfs.bbclass > b/meta/classes/overlayfs.bbclass > > > new file mode 100644 > > > index 00..8d9b59c9bf > > > --- /dev/null > > > +++ b/meta/classes/overlayfs.bbclass > > > @@ -0,0 +1,111 @@ > > > +# Class for generation of overlayfs mount units > > > +# > > > +# It's often desired in Embedded System design to have a read-only > rootfs. > > > +# But a lot of different applications might want to have a read-write > access to > > > +# some parts of a filesystem. It can be especially useful when your > update mechanism > > > +# overwrites the whole rootfs, but you want your application data to > be preserved > > > +# between updates. This class provides a way to achieve that by means > > > +# of overlayfs and at the same time keeping the base rootfs read-only. > > > +# > > > +# Usage example. > > > +# > > > +# Set a mount point for a partition overlayfs is going to use as > upper layer > > > +# in your machine configuration. Underlying file system can be > anything that > > > +# is supported by overlayfs. This has to be done in your machine > configuration. > > > +# QA check fails to catch file existence if you redefine this > variable in your recipe! > > > +# > > > +# OVERLAYFS_MOUNT_POINT[data] ?= "/data" > > > +# > > > +# The class assumes you have a data.mount systemd unit defined in your > > > +# systemd-machine-units recipe and installed to the image. > > > +# > > > +# Then you can specify writable directories on a recipe base > > > +# > > > +# OVERLAYFS_WRITABLE_PATHS[data] = > "/usr/share/my-custom-application" > > > +# > > > +# To support several mount points you can use a different variable > flag. Assume we > > > +# want to have a writable location on the file system, but not > interested where the data > > > +# survive a reboot. Then we could have a mnt-overlay.mount unit for a > tmpfs file system: > > > +# > > > +# OVERLAYFS_MOUNT_POINT[mnt-overlay] = "/mnt/overlay" > > > +# OVERLAYFS_WRITABLE_PATHS[mnt-overlay] = > "/usr/share/another-application" > > > +# > > > +# Note: the class does not support /etc directory itself, because > systemd depends on it > > > > > > Many thanks for this new class and the corresponding documentation. > > Richard, I guess that's something we'll need to document in the manual, > > once it's merge, right? > > Ideally, yes, correct. > > Cheers, > > Richard > > > > > -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#154560): https://lists.openembedded.org/g/openembedded-core/message/154560 Mute This Topic: https://lists.openembedded.org/mt/84706478/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
