[OE-core] [PATCH v2] create-spdx: add support for SDKs

2022-01-26 Thread Andres Beltran 
Currently, SPDX SBOMs are only created for images. Add support for
SDKs. Fix json indent when outputting SBOMs for better readability.

Signed-off-by: Andres Beltran 
---
 meta/classes/create-spdx.bbclass | 88 ++--
 meta/lib/oe/sbom.py  |  4 ++
 2 files changed, 64 insertions(+), 28 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index eb9535069a4..8bae1359ac2 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -560,7 +560,7 @@ python do_create_spdx() {
 oe.sbom.write_doc(d, package_doc, "packages")
 }
 # NOTE: depending on do_unpack is a hack that is necessary to get it's 
dependencies for archive the source
-addtask do_create_spdx after do_package do_packagedata do_unpack before 
do_build do_rm_work
+addtask do_create_spdx after do_package do_packagedata do_unpack before 
do_populate_sdk do_build do_rm_work
 
 SSTATETASKS += "do_create_spdx"
 do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
@@ -792,28 +792,77 @@ def spdx_get_src(d):
 do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
 
 ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; "
+
+do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
+POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " 
sdk_host_combine_spdx; "
+POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " 
sdk_target_combine_spdx; "
+
 python image_combine_spdx() {
+import os
+import oe.sbom
+from pathlib import Path
+from oe.rootfs import image_list_installed_packages
+
+image_name = d.getVar("IMAGE_NAME")
+image_link_name = d.getVar("IMAGE_LINK_NAME")
+imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
+img_spdxid = oe.sbom.get_image_spdxid(image_name)
+packages = image_list_installed_packages(d)
+
+combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages)
+
+if image_link_name:
+image_spdx_path = imgdeploydir / (image_name + ".spdx.json")
+image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json")
+image_spdx_link.symlink_to(os.path.relpath(image_spdx_path, 
image_spdx_link.parent))
+
+def make_image_link(target_path, suffix):
+if image_link_name:
+link = imgdeploydir / (image_link_name + suffix)
+link.symlink_to(os.path.relpath(target_path, link.parent))
+
+spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst")
+make_image_link(spdx_tar_path, ".spdx.tar.zst")
+spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json")
+make_image_link(spdx_index_path, ".spdx.index.json")
+}
+
+python sdk_host_combine_spdx() {
+sdk_combine_spdx(d, "host")
+}
+
+python sdk_target_combine_spdx() {
+sdk_combine_spdx(d, "target")
+}
+
+def sdk_combine_spdx(d, sdk_type):
+import oe.sbom
+from pathlib import Path
+from oe.sdk import sdk_list_installed_packages
+
+sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type
+sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
+sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name)
+sdk_packages = sdk_list_installed_packages(d, sdk_type == "target")
+combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages)
+
+def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages):
 import os
 import oe.spdx
 import oe.sbom
 import io
 import json
-from oe.rootfs import image_list_installed_packages
 from datetime import timezone, datetime
 from pathlib import Path
 import tarfile
 import bb.compress.zstd
 
 creation_time = 
datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-image_name = d.getVar("IMAGE_NAME")
-image_link_name = d.getVar("IMAGE_LINK_NAME")
-
 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
-imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
 
 doc = oe.spdx.SPDXDocument()
-doc.name = image_name
+doc.name = rootfs_name
 doc.documentNamespace = get_doc_namespace(d, doc)
 doc.creationInfo.created = creation_time
 doc.creationInfo.comment = "This document was created by analyzing the 
source of the Yocto recipe during the build."
@@ -825,14 +874,12 @@ python image_combine_spdx() {
 image = oe.spdx.SPDXPackage()
 image.name = d.getVar("PN")
 image.versionInfo = d.getVar("PV")
-image.SPDXID = oe.sbom.get_image_spdxid(image_name)
+image.SPDXID = rootfs_spdxid
 
 doc.packages.append(image)
 
 spdx_package = oe.spdx.SPDXPackage()
 
-packages = image_list_installed_packages(d)
-
 for name in sorted

Re: [OE-core] [PATCH] create-spdx: add support for SDKs

2022-01-18 Thread Andres Beltran 

On 1/14/2022 2:16 PM, Saul Wold wrote:
Overall I think this is going in the right direction, I need to review 
it a little deeper and check the actual output.


I am not sure that you tested this against master as you use the old _ 
override syntax vs using a :.


See note below.

Sau!


Thanks Saul! Yes, I was certainly using an older version. I will
correct the override syntax. Let me know if I should make any other
changes once you review it.



-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160681): 
https://lists.openembedded.org/g/openembedded-core/message/160681
Mute This Topic: https://lists.openembedded.org/mt/88381128/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] create-spdx: add support for SDKs

2022-01-18 Thread Andres Beltran 

On 1/14/2022 3:49 PM, Joshua Watt wrote:


Most of us just pipe the output through `jq` to view it (colors are
really helpful). These files are already quite large, do we really
need to add more padding?


Ok thanks Joshua. I can certainly remove that change. I saw that the
json files didn't have any indentation out of a build, so added
some padding to make them more readable. But yes, we can certainly use
jq as well.


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160680): 
https://lists.openembedded.org/g/openembedded-core/message/160680
Mute This Topic: https://lists.openembedded.org/mt/88381128/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH] create-spdx: add support for SDKs

2022-01-12 Thread Andres Beltran 
Currently, SPDX SBOMs are only created for images. Add support for
SDKs. Fix json indent when outputting SBOMs for better readability.

Signed-off-by: Andres Beltran 
---
 meta/classes/create-spdx.bbclass | 95 +---
 meta/lib/oe/sbom.py  |  6 +-
 2 files changed, 68 insertions(+), 33 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index e44a204a8fc..d0f987315ee 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -556,7 +556,7 @@ python do_create_spdx() {
 oe.sbom.write_doc(d, package_doc, "packages")
 }
 # NOTE: depending on do_unpack is a hack that is necessary to get it's 
dependencies for archive the source
-addtask do_create_spdx after do_package do_packagedata do_unpack before 
do_build do_rm_work
+addtask do_create_spdx after do_package do_packagedata do_unpack before 
do_populate_sdk do_build do_rm_work
 
 SSTATETASKS += "do_create_spdx"
 do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
@@ -788,28 +788,77 @@ def spdx_get_src(d):
 do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
 
 ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; "
+
+do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
+POPULATE_SDK_POST_HOST_COMMAND_append_task-populate-sdk = " 
sdk_host_combine_spdx; "
+POPULATE_SDK_POST_TARGET_COMMAND_append_task-populate-sdk = " 
sdk_target_combine_spdx; "
+
 python image_combine_spdx() {
+import os
+import oe.sbom
+from pathlib import Path
+from oe.rootfs import image_list_installed_packages
+
+image_name = d.getVar("IMAGE_NAME")
+image_link_name = d.getVar("IMAGE_LINK_NAME")
+imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
+img_spdxid = oe.sbom.get_image_spdxid(image_name)
+packages = image_list_installed_packages(d)
+
+combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages)
+
+if image_link_name:
+image_spdx_path = imgdeploydir / (image_name + ".spdx.json")
+image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json")
+image_spdx_link.symlink_to(os.path.relpath(image_spdx_path, 
image_spdx_link.parent))
+
+def make_image_link(target_path, suffix):
+if image_link_name:
+link = imgdeploydir / (image_link_name + suffix)
+link.symlink_to(os.path.relpath(target_path, link.parent))
+
+spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst")
+make_image_link(spdx_tar_path, ".spdx.tar.zst")
+spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json")
+make_image_link(spdx_index_path, ".spdx.index.json")
+}
+
+python sdk_host_combine_spdx() {
+sdk_combine_spdx(d, "host")
+}
+
+python sdk_target_combine_spdx() {
+sdk_combine_spdx(d, "target")
+}
+
+def sdk_combine_spdx(d, sdk_type):
+import oe.sbom
+from pathlib import Path
+from oe.sdk import sdk_list_installed_packages
+
+sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type
+sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
+sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name)
+sdk_packages = sdk_list_installed_packages(d, sdk_type == "target")
+combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages)
+
+def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages):
 import os
 import oe.spdx
 import oe.sbom
 import io
 import json
-from oe.rootfs import image_list_installed_packages
 from datetime import timezone, datetime
 from pathlib import Path
 import tarfile
 import bb.compress.zstd
 
 creation_time = 
datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-image_name = d.getVar("IMAGE_NAME")
-image_link_name = d.getVar("IMAGE_LINK_NAME")
-
 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
-imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
 
 doc = oe.spdx.SPDXDocument()
-doc.name = image_name
+doc.name = rootfs_name
 doc.documentNamespace = get_doc_namespace(d, doc)
 doc.creationInfo.created = creation_time
 doc.creationInfo.comment = "This document was created by analyzing the 
source of the Yocto recipe during the build."
@@ -821,14 +870,12 @@ python image_combine_spdx() {
 image = oe.spdx.SPDXPackage()
 image.name = d.getVar("PN")
 image.versionInfo = d.getVar("PV")
-image.SPDXID = oe.sbom.get_image_spdxid(image_name)
+image.SPDXID = rootfs_spdxid
 
 doc.packages.append(image)
 
 spdx_package = oe.spdx.SPDXPackage()
 
-packages = image_list_installed_packages(d)
-
 fo

[OE-core] [PATCH] create-spdx: Fix key errors in do_create_runtime_spdx

2021-11-17 Thread Andres Beltran 
Currently, the do_create_runtime_spdx task fails with a Key Error if a
dependency is not contained in the package providers dictionary. Add a
check before using "dep" as a key in "providers".

Signed-off-by: Andres Beltran 
---
 meta/classes/create-spdx.bbclass | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index c85a11595f..601a928996 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -678,6 +678,9 @@ python do_create_runtime_spdx() {
 if dep in seen_deps:
 continue
 
+if dep not in providers:
+continue
+
 dep = providers[dep]
 
 if not oe.packagedata.packaged(dep, localdata):
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#158463): 
https://lists.openembedded.org/g/openembedded-core/message/158463
Mute This Topic: https://lists.openembedded.org/mt/87133369/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH] create-spdx: Set the Organization field via a variable

2021-11-04 Thread Andres Beltran 
Currently, the "Organization" field for SBOMs is hard-coded in
create-spdx. Create a new variable SPDX_ORG to make this field more
generic.

Signed-off-by: Andres Beltran 
---
 meta/classes/create-spdx.bbclass | 10 ++
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index c35dbe11849..eb1d446f3fb 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -28,6 +28,8 @@ SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc;
 
 SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
 
+SPDX_ORG ??= "OpenEmbedded ()"
+
 do_image_complete[depends] = "virtual/kernel:do_create_spdx"
 
 def get_doc_namespace(d, doc):
@@ -415,7 +417,7 @@ python do_create_spdx() {
 doc.creationInfo.comment = "This document was created by analyzing recipe 
files during the build."
 doc.creationInfo.licenseListVersion = 
d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
 doc.creationInfo.creators.append("Tool: OpenEmbedded Core 
create-spdx.bbclass")
-doc.creationInfo.creators.append("Organization: OpenEmbedded ()")
+doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
 doc.creationInfo.creators.append("Person: N/A ()")
 
 recipe = oe.spdx.SPDXPackage()
@@ -519,7 +521,7 @@ python do_create_spdx() {
 package_doc.creationInfo.comment = "This document was created by 
analyzing packages created during the build."
 package_doc.creationInfo.licenseListVersion = 
d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
 package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core 
create-spdx.bbclass")
-package_doc.creationInfo.creators.append("Organization: 
OpenEmbedded ()")
+package_doc.creationInfo.creators.append("Organization: %s" % 
d.getVar("SPDX_ORG"))
 package_doc.creationInfo.creators.append("Person: N/A ()")
 package_doc.externalDocumentRefs.append(recipe_ref)
 
@@ -653,7 +655,7 @@ python do_create_runtime_spdx() {
 runtime_doc.creationInfo.comment = "This document was created by 
analyzing package runtime dependencies."
 runtime_doc.creationInfo.licenseListVersion = 
d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
 runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core 
create-spdx.bbclass")
-runtime_doc.creationInfo.creators.append("Organization: 
OpenEmbedded ()")
+runtime_doc.creationInfo.creators.append("Organization: %s" % 
d.getVar("SPDX_ORG"))
 runtime_doc.creationInfo.creators.append("Person: N/A ()")
 
 package_ref = oe.spdx.SPDXExternalDocumentRef()
@@ -813,7 +815,7 @@ python image_combine_spdx() {
 doc.creationInfo.comment = "This document was created by analyzing the 
source of the Yocto recipe during the build."
 doc.creationInfo.licenseListVersion = 
d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
 doc.creationInfo.creators.append("Tool: OpenEmbedded Core 
create-spdx.bbclass")
-doc.creationInfo.creators.append("Organization: OpenEmbedded ()")
+doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
 doc.creationInfo.creators.append("Person: N/A ()")
 
 image = oe.spdx.SPDXPackage()
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#157854): 
https://lists.openembedded.org/g/openembedded-core/message/157854
Mute This Topic: https://lists.openembedded.org/mt/86820747/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

[OE-core] [PATCH v2] buildhistory: Fix package output files for SDKs

2021-10-04 Thread Andres Beltran 
Currently, installed packages are listed for images in image-info.txt, but
not for SDKs in sdk-info.txt. Add TOOLCHAIN_HOST_TASK and
TOOLCHAIN_TARGET_TASK to the output variables in sdk-info.txt.

Moreover, package output files for the SDK host are empty because
PKGDATA_DIR defaults to the target directory. Fix this bug and create a new
variable called PKGDATA_DIR_SDK which stores the correct path for the SDK
host package data.

Signed-off-by: Andres Beltran 
---
 meta/classes/buildhistory.bbclass   | 21 -
 meta/classes/cross-canadian.bbclass |  2 +-
 meta/classes/nativesdk.bbclass  |  2 +-
 meta/conf/bitbake.conf  |  1 +
 meta/lib/oe/package_manager/__init__.py |  2 +-
 5 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/meta/classes/buildhistory.bbclass 
b/meta/classes/buildhistory.bbclass
index a613306270..7c44fec2d1 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -442,11 +442,16 @@ def buildhistory_list_installed(d, rootfs_type="image"):
 else:
 pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target")
 
+if rootfs_type == "sdk_host":
+pkgdata_dir = d.getVar('PKGDATA_DIR_SDK')
+else:
+pkgdata_dir = d.getVar('PKGDATA_DIR')
+
 for output_type, output_file in process_list:
 output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
 
 with open(output_file_full, 'w') as output:
-output.write(format_pkg_list(pkgs, output_type, 
d.getVar('PKGDATA_DIR')))
+output.write(format_pkg_list(pkgs, output_type, pkgdata_dir))
 
 python buildhistory_list_installed_image() {
 buildhistory_list_installed(d)
@@ -496,13 +501,19 @@ buildhistory_get_installed() {
echo "}" >>  $1/depends.dot
rm $1/depends.tmp
 
+   # Set correct pkgdatadir
+   pkgdatadir=${PKGDATA_DIR}
+   if [ "$2" == "sdk" ] && [ "$3" == "host" ]; then
+   pkgdatadir="${PKGDATA_DIR_SDK}"
+   fi
+
# Produce installed package sizes list
-   oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache 
> $1/installed-package-sizes.tmp
+   oe-pkgdata-util -p $pkgdatadir read-value "PKGSIZE" -n -f $pkgcache > 
$1/installed-package-sizes.tmp
cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | 
sort -n -r > $1/installed-package-sizes.txt
rm $1/installed-package-sizes.tmp
 
# Produce package info: runtime_name, buildtime_name, recipe, version, 
size
-   oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PACKAGE,PN,PV,PKGSIZE" -n 
-f $pkgcache > $1/installed-package-info.tmp
+   oe-pkgdata-util -p $pkgdatadir read-value "PACKAGE,PN,PV,PKGSIZE" -n -f 
$pkgcache > $1/installed-package-info.tmp
cat $1/installed-package-info.tmp | sort -n -r -k 5 > 
$1/installed-package-info.txt
rm $1/installed-package-info.tmp
 
@@ -542,7 +553,7 @@ buildhistory_get_sdk_installed() {
return
fi
 
-   buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk
+   buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk $1
 }
 
 buildhistory_get_sdk_installed_host() {
@@ -773,7 +784,7 @@ def buildhistory_get_imagevars(d):
 def buildhistory_get_sdkvars(d):
 if d.getVar('BB_WORKERCONTEXT') != '1':
 return ""
-sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE 
SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE"
+sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE 
SDKIMAGE_FEATURES TOOLCHAIN_HOST_TASK TOOLCHAIN_TARGET_TASK BAD_RECOMMENDATIONS 
NO_RECOMMENDATIONS PACKAGE_EXCLUDE"
 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext':
 # Extensible SDK uses some additional variables
 sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST 
SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS 
SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN"
diff --git a/meta/classes/cross-canadian.bbclass 
b/meta/classes/cross-canadian.bbclass
index ffbc2167e3..ac82e86356 100644
--- a/meta/classes/cross-canadian.bbclass
+++ b/meta/classes/cross-canadian.bbclass
@@ -169,7 +169,7 @@ USE_NLS = "${SDKUSE_NLS}"
 # and not any particular tune that is enabled.
 TARGET_ARCH[vardepsexclude] = "TUNE_ARCH"
 
-PKGDATA_DIR = "${TMPDIR}/pkgdata/${SDK_SYS}"
+PKGDATA_DIR = "${PKGDATA_DIR_SDK}"
 # If MLPREFIX is set by multilib code, shlibs
 # points to the wrong place so force it
 SHLIBSDIRS = "${PKGDATA_DIR}/nativesdk-shlibs2"
diff --git a/meta/classes/nativesdk.bbclass b/meta/classes/nativesdk.bbclass
index c66de8c787..14e210562f 100644
--- a/meta/classes/nativesdk.bbclass
+++ b/meta/classes/nativesd

[OE-core] [PATCH] buildhistory: Fix package output files for SDKs

2021-10-04 Thread Andres Beltran 
Currently, installed packages are listed for images in image-info.txt, but
not for SDKs in sdk-info.txt. Add TOOLCHAIN_HOST_TASK and
TOOLCHAIN_TARGET_TASK to the output variables in sdk-info.txt.

Moreover, package output files for the SDK host are empty because
PKGDATA_DIR defaults to the target directory. Fix this bug and create a new
variable called PKGDATA_DIR_SDK which stores the correct path for the SDK
host package data.

Signed-off-by: Andres Beltran 
---
 meta/classes/buildhistory.bbclass   | 21 -
 meta/classes/cross-canadian.bbclass |  2 +-
 meta/classes/nativesdk.bbclass  |  2 +-
 meta/conf/bitbake.conf  |  1 +
 meta/lib/oe/package_manager.py  |  2 +-
 5 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/meta/classes/buildhistory.bbclass 
b/meta/classes/buildhistory.bbclass
index 20609c435b..ab9d882752 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -434,11 +434,16 @@ def buildhistory_list_installed(d, rootfs_type="image"):
 else:
 pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target")
 
+if rootfs_type == "sdk_host":
+pkgdata_dir = d.getVar('PKGDATA_DIR_SDK')
+else:
+pkgdata_dir = d.getVar('PKGDATA_DIR')
+
 for output_type, output_file in process_list:
 output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
 
 with open(output_file_full, 'w') as output:
-output.write(format_pkg_list(pkgs, output_type, 
d.getVar('PKGDATA_DIR')))
+output.write(format_pkg_list(pkgs, output_type, pkgdata_dir))
 
 python buildhistory_list_installed_image() {
 buildhistory_list_installed(d)
@@ -488,13 +493,19 @@ buildhistory_get_installed() {
echo "}" >>  $1/depends.dot
rm $1/depends.tmp
 
+   # Set correct pkgdatadir
+   pkgdatadir=${PKGDATA_DIR}
+   if [ "$2" == "sdk" ] && [ "$3" == "host" ]; then
+   pkgdatadir="${PKGDATA_DIR_SDK}"
+   fi
+
# Produce installed package sizes list
-   oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PKGSIZE" -n -f $pkgcache 
> $1/installed-package-sizes.tmp
+   oe-pkgdata-util -p $pkgdatadir read-value "PKGSIZE" -n -f $pkgcache > 
$1/installed-package-sizes.tmp
cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | 
sort -n -r > $1/installed-package-sizes.txt
rm $1/installed-package-sizes.tmp
 
# Produce package info: runtime_name, buildtime_name, recipe, version, 
size
-   oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PACKAGE,PN,PV,PKGSIZE" -n 
-f $pkgcache > $1/installed-package-info.tmp
+   oe-pkgdata-util -p $pkgdatadir read-value "PACKAGE,PN,PV,PKGSIZE" -n -f 
$pkgcache > $1/installed-package-info.tmp
cat $1/installed-package-info.tmp | sort -n -r -k 5 > 
$1/installed-package-info.txt
rm $1/installed-package-info.tmp
 
@@ -534,7 +545,7 @@ buildhistory_get_sdk_installed() {
return
fi
 
-   buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk
+   buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk $1
 }
 
 buildhistory_get_sdk_installed_host() {
@@ -762,7 +773,7 @@ def buildhistory_get_imagevars(d):
 def buildhistory_get_sdkvars(d):
 if d.getVar('BB_WORKERCONTEXT') != '1':
 return ""
-sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE 
SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE"
+sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE 
SDKIMAGE_FEATURES TOOLCHAIN_HOST_TASK TOOLCHAIN_TARGET_TASK BAD_RECOMMENDATIONS 
NO_RECOMMENDATIONS PACKAGE_EXCLUDE"
 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext':
 # Extensible SDK uses some additional variables
 sdkvars += " SDK_LOCAL_CONF_WHITELIST SDK_LOCAL_CONF_BLACKLIST 
SDK_INHERIT_BLACKLIST SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS 
SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN"
diff --git a/meta/classes/cross-canadian.bbclass 
b/meta/classes/cross-canadian.bbclass
index 1e54035084..1974b33943 100644
--- a/meta/classes/cross-canadian.bbclass
+++ b/meta/classes/cross-canadian.bbclass
@@ -167,7 +167,7 @@ USE_NLS = "${SDKUSE_NLS}"
 # and not any particular tune that is enabled.
 TARGET_ARCH[vardepsexclude] = "TUNE_ARCH"
 
-PKGDATA_DIR = "${TMPDIR}/pkgdata/${SDK_SYS}"
+PKGDATA_DIR = "${PKGDATA_DIR_SDK}"
 # If MLPREFIX is set by multilib code, shlibs
 # points to the wrong place so force it
 SHLIBSDIRS = "${PKGDATA_DIR}/nativesdk-shlibs2"
diff --git a/meta/classes/nativesdk.bbclass b/meta/classes/nativesdk.bbclass
index 7f2692c51a..d2a3f0bec9 100644
--- a/meta/classes/nativesdk.bbclass
+++ b/meta/classes/nativesdk.bbclass
@@ -3

[OE-core] [PATCH v2] buildhistory: Label packages providing per-file dependencies in depends.dot

2021-08-30 Thread Andres Beltran 
Currently, depends.dot includes per-file dependencies but not the packages
providing those files. This makes it hard to obtain all package
dependencies by just looking at depends.dot.

Parse the RPROVIDES and FILERPROVIDES fields from pkgdata to map each of
their values to the package providing the component. Include runtime
packages as dependencies in depends.dot, together with the component
provided by the package as a label.

Signed-off-by: Andres Beltran 
---
 meta/classes/buildhistory.bbclass |  4 +++-
 meta/lib/oe/utils.py  | 32 +--
 2 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/meta/classes/buildhistory.bbclass 
b/meta/classes/buildhistory.bbclass
index 134b56ab71..20609c435b 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -438,7 +438,7 @@ def buildhistory_list_installed(d, rootfs_type="image"):
 output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
 
 with open(output_file_full, 'w') as output:
-output.write(format_pkg_list(pkgs, output_type))
+output.write(format_pkg_list(pkgs, output_type, 
d.getVar('PKGDATA_DIR')))
 
 python buildhistory_list_installed_image() {
 buildhistory_list_installed(d)
@@ -479,6 +479,8 @@ buildhistory_get_installed() {
   -e 's:|: -> :' \
   -e 's:"\[REC\]":[style=dotted]:' \
   -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \
+  -e 's:"\([*]\+\)" "\([^"]*\)":[label="\2"]:' \
+  -e 's:"\[RPROVIDES\]":[style=dashed]:' \
$1/depends.tmp
# Add header, sorted and de-duped contents and footer and then delete 
the temp file
printf "digraph depends {\nnode [shape=plaintext]\n" > 
$1/depends.dot
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index 83d298906b..75269a9feb 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -345,7 +345,29 @@ def squashspaces(string):
 import re
 return re.sub(r"\s+", " ", string).strip()
 
-def format_pkg_list(pkg_dict, ret_format=None):
+def rprovides_map(pkgdata_dir, pkg_dict):
+# Map file -> pkg provider
+rprov_map = {}
+
+for pkg in pkg_dict:
+path_to_pkgfile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg)
+if not os.path.isfile(path_to_pkgfile):
+continue
+with open(path_to_pkgfile) as f:
+for line in f:
+if line.startswith('RPROVIDES') or 
line.startswith('FILERPROVIDES'):
+# List all components provided by pkg.
+# Exclude version strings, i.e. those starting with (
+provides = [x for x in line.split()[1:] if not 
x.startswith('(')]
+for prov in provides:
+if prov in rprov_map:
+rprov_map[prov].append(pkg)
+else:
+rprov_map[prov] = [pkg]
+
+return rprov_map
+
+def format_pkg_list(pkg_dict, ret_format=None, pkgdata_dir=None):
 output = []
 
 if ret_format == "arch":
@@ -358,9 +380,15 @@ def format_pkg_list(pkg_dict, ret_format=None):
 for pkg in sorted(pkg_dict):
 output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], 
pkg_dict[pkg]["ver"]))
 elif ret_format == "deps":
+rprov_map = rprovides_map(pkgdata_dir, pkg_dict)
 for pkg in sorted(pkg_dict):
 for dep in pkg_dict[pkg]["deps"]:
-output.append("%s|%s" % (pkg, dep))
+if dep in rprov_map:
+# There could be multiple providers within the image
+for pkg_provider in rprov_map[dep]:
+output.append("%s|%s * %s [RPROVIDES]" % (pkg, 
pkg_provider, dep))
+else:
+output.append("%s|%s" % (pkg, dep))
 else:
 for pkg in sorted(pkg_dict):
 output.append(pkg)
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#155511): 
https://lists.openembedded.org/g/openembedded-core/message/155511
Mute This Topic: https://lists.openembedded.org/mt/85264210/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] buildhistory: Add output file listing package information

2021-08-23 Thread Andres Beltran 

Hello,

This is a reminder request for review.

Best,
Andres Beltran


On 8/12/2021 9:58 AM, Andres Beltran wrote:

Currently, buildhistory does not produce a single file combining relevant
information of installed packages. Produce an output file
"installed-package-info.txt" listing a package's runtime name, buildtime name,
its recipe, version, and size to avoid having to look up each package 
externally.
Leave the existing package list files as-is for backwards compatibility.

In order to support this efficiently, extend oe-pkgdata-util to accept multiple 
keys
for its read-value argument.

Signed-off-by: Andres Beltran 
---
  meta/classes/buildhistory.bbclass |  5 +
  scripts/oe-pkgdata-util   | 37 +++
  2 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/meta/classes/buildhistory.bbclass 
b/meta/classes/buildhistory.bbclass
index 8a1359acbe..134b56ab71 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -491,6 +491,11 @@ buildhistory_get_installed() {
cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort 
-n -r > $1/installed-package-sizes.txt
rm $1/installed-package-sizes.tmp
  
+	# Produce package info: runtime_name, buildtime_name, recipe, version, size

+   oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PACKAGE,PN,PV,PKGSIZE" -n -f 
$pkgcache > $1/installed-package-info.tmp
+   cat $1/installed-package-info.tmp | sort -n -r -k 5 > 
$1/installed-package-info.txt
+   rm $1/installed-package-info.tmp
+
# We're now done with the cache, delete it
rm $pkgcache
  
diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util

index 75dd23efa3..c30baaa580 100755
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -171,7 +171,7 @@ def read_value(args):
  val = line.split(': ', 1)[1].rstrip()
  return val
  
-logger.debug("read-value('%s', '%s' '%s')" % (args.pkgdata_dir, args.valuename, packages))

+logger.debug("read-value('%s', '%s' '%s')" % (args.pkgdata_dir, 
args.valuenames, packages))
  for package in packages:
  pkg_split = package.split('_')
  pkg_name = pkg_split[0]
@@ -180,20 +180,29 @@ def read_value(args):
  logger.debug(revlink)
  if os.path.exists(revlink):
  mappedpkg = os.path.basename(os.readlink(revlink))
-qvar = args.valuename
-value = readvar(revlink, qvar, mappedpkg)
-if qvar == "PKGSIZE":
-# PKGSIZE is now in bytes, but we we want it in KB
-pkgsize = (int(value) + 1024 // 2) // 1024
-value = "%d" % pkgsize
-if args.unescape:
-import codecs
-# escape_decode() unescapes backslash encodings in byte streams
-value = codecs.escape_decode(bytes(value, 
"utf-8"))[0].decode("utf-8")
+qvars = args.valuenames
+val_names = qvars.split(',')
+values = []
+for qvar in val_names:
+if qvar == "PACKAGE":
+value = mappedpkg
+else:
+value = readvar(revlink, qvar, mappedpkg)
+if qvar == "PKGSIZE":
+# PKGSIZE is now in bytes, but we we want it in KB
+pkgsize = (int(value) + 1024 // 2) // 1024
+value = "%d" % pkgsize
+if args.unescape:
+import codecs
+# escape_decode() unescapes backslash encodings in byte 
streams
+value = codecs.escape_decode(bytes(value, 
"utf-8"))[0].decode("utf-8")
+values.append(value)
+
+values_str = ' '.join(values)
  if args.prefix_name:
-print('%s %s' % (pkg_name, value))
+print('%s %s' % (pkg_name, values_str))
  else:
-print(value)
+print(values_str)
  else:
  logger.debug("revlink %s does not exist", revlink)
  
@@ -570,7 +579,7 @@ def main():

  parser_read_value = subparsers.add_parser('read-value',
help='Read any pkgdata value for 
one or more packages',
description='Reads the named value 
from the pkgdata files for the specified packages')
-parser_read_value.add_argument('valuename', help='Name of the value to 
look up')
+parser_read_value.add_argument('valuenames', help='Name of the value/s to 
look up (separated by commas, no spaces)')
  parser_read_value.add_argument('pkg', nargs='*', help='Runtime package 
name to look up')
  parser_read_value.add_argument('-f', '--file', help='Read package names 
from the specified

[OE-core] [PATCH] buildhistory: Label packages providing per-file dependencies in depends.dot

2021-08-18 Thread Andres Beltran 
Currently, depends.dot includes per-file dependencies but not the packages
providing those files. This makes it hard to obtain all package
dependencies by just looking at depends.dot.

Parse the RPROVIDES and FILERPROVIDES fields from pkgdata to map each of
their values to the package providing the component. Include runtime
packages as dependencies in depends.dot, together with the component
provided by the package as a label.

Signed-off-by: Andres Beltran 
---
 meta/classes/buildhistory.bbclass |  4 +++-
 meta/lib/oe/utils.py  | 29 +++--
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/meta/classes/buildhistory.bbclass 
b/meta/classes/buildhistory.bbclass
index 8a1359acbe..9ec48a879a 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -438,7 +438,7 @@ def buildhistory_list_installed(d, rootfs_type="image"):
 output_file_full = os.path.join(d.getVar('WORKDIR'), output_file)
 
 with open(output_file_full, 'w') as output:
-output.write(format_pkg_list(pkgs, output_type))
+output.write(format_pkg_list(pkgs, output_type, 
d.getVar('PKGDATA_DIR')))
 
 python buildhistory_list_installed_image() {
 buildhistory_list_installed(d)
@@ -479,6 +479,8 @@ buildhistory_get_installed() {
   -e 's:|: -> :' \
   -e 's:"\[REC\]":[style=dotted]:' \
   -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \
+  -e 's:"\([*]\+\)" "\([^"]*\)":[label="\2"]:' \
+  -e 's:"\[RPROVIDES\]":[style=dashed]:' \
$1/depends.tmp
# Add header, sorted and de-duped contents and footer and then delete 
the temp file
printf "digraph depends {\nnode [shape=plaintext]\n" > 
$1/depends.dot
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index 83d298906b..0f3dfd644d 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -345,7 +345,26 @@ def squashspaces(string):
 import re
 return re.sub(r"\s+", " ", string).strip()
 
-def format_pkg_list(pkg_dict, ret_format=None):
+def rprovides_map(pkgdata_dir, pkg_dict):
+# Map file -> pkg provider
+rprov_map = {}
+
+for pkg in pkg_dict:
+with open(os.path.join(pkgdata_dir, 'runtime-reverse', pkg)) as f:
+for line in f:
+if line.startswith('RPROVIDES') or 
line.startswith('FILERPROVIDES'):
+# List all components provided by pkg.
+# Exclude version strings, i.e. those starting with (
+provides = [x for x in line.split()[1:] if not 
x.startswith('(')]
+for prov in provides:
+if prov in rprov_map:
+rprov_map[prov].append(pkg)
+else:
+rprov_map[prov] = [pkg]
+
+return rprov_map
+
+def format_pkg_list(pkg_dict, ret_format=None, pkgdata_dir=None):
 output = []
 
 if ret_format == "arch":
@@ -358,9 +377,15 @@ def format_pkg_list(pkg_dict, ret_format=None):
 for pkg in sorted(pkg_dict):
 output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], 
pkg_dict[pkg]["ver"]))
 elif ret_format == "deps":
+rprov_map = rprovides_map(pkgdata_dir, pkg_dict)
 for pkg in sorted(pkg_dict):
 for dep in pkg_dict[pkg]["deps"]:
-output.append("%s|%s" % (pkg, dep))
+if dep in rprov_map:
+# There could be multiple providers within the image
+for pkg_provider in rprov_map[dep]:
+output.append("%s|%s * %s [RPROVIDES]" % (pkg, 
pkg_provider, dep))
+else:
+output.append("%s|%s" % (pkg, dep))
 else:
 for pkg in sorted(pkg_dict):
 output.append(pkg)
-- 
2.17.1


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#154937): 
https://lists.openembedded.org/g/openembedded-core/message/154937
Mute This Topic: https://lists.openembedded.org/mt/84977439/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-

Re: [OE-core] [PATCH] buildhistory: Add output file listing package information

2021-08-16 Thread Andres Beltran 

Hi Mikko,

This patch outputs information that is readily available in the context 
of the
image, i.e. information that can be read directly from pkgdata. The 
information

for the recipe's layer is not readily available, so it would have
to be a separate task to add that information (or make it extensible to 
allow

it).

Best,
-Andres


On 8/13/2021 12:38 AM, Mikko Rapeli wrote:

Hi,

On Thu, Aug 12, 2021 at 04:58:56PM +, Andres Beltran wrote:

Currently, buildhistory does not produce a single file combining relevant
information of installed packages. Produce an output file
"installed-package-info.txt" listing a package's runtime name, buildtime name,
its recipe, version, and size to avoid having to look up each package 
externally.
Leave the existing package list files as-is for backwards compatibility.

Thanks for this patch, I find it useful!

I have had similar needs and have created scripts which parse the same info
from buildhistory. What I'd add here is the layer of the recipe.

In large projects I have the need to also map binary packages into
their maintainers and subprojects, and produce reports, e.g. rootfs usage
reports per maintainer/subproject to be checked against an agreed budget.
Thus I've used layer and MAINTAINER fields to map binary package to the
subprojects and maintainers inside the projects.

Would be nice to be able to extend the rootfs binary package reports with
this info but a bit of scripting on the buildhistory output is fine too.

Cheers,

-Mikko


In order to support this efficiently, extend oe-pkgdata-util to accept multiple 
keys
for its read-value argument.

Signed-off-by: Andres Beltran 
---
  meta/classes/buildhistory.bbclass |  5 +
  scripts/oe-pkgdata-util   | 37 +++
  2 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/meta/classes/buildhistory.bbclass 
b/meta/classes/buildhistory.bbclass
index 8a1359acbe..134b56ab71 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -491,6 +491,11 @@ buildhistory_get_installed() {
cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort 
-n -r > $1/installed-package-sizes.txt
rm $1/installed-package-sizes.tmp
  
+	# Produce package info: runtime_name, buildtime_name, recipe, version, size

+   oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PACKAGE,PN,PV,PKGSIZE" -n -f 
$pkgcache > $1/installed-package-info.tmp
+   cat $1/installed-package-info.tmp | sort -n -r -k 5 > 
$1/installed-package-info.txt
+   rm $1/installed-package-info.tmp
+
# We're now done with the cache, delete it
rm $pkgcache
  
diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util

index 75dd23efa3..c30baaa580 100755
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -171,7 +171,7 @@ def read_value(args):
  val = line.split(': ', 1)[1].rstrip()
  return val
  
-logger.debug("read-value('%s', '%s' '%s')" % (args.pkgdata_dir, args.valuename, packages))

+logger.debug("read-value('%s', '%s' '%s')" % (args.pkgdata_dir, 
args.valuenames, packages))
  for package in packages:
  pkg_split = package.split('_')
  pkg_name = pkg_split[0]
@@ -180,20 +180,29 @@ def read_value(args):
  logger.debug(revlink)
  if os.path.exists(revlink):
  mappedpkg = os.path.basename(os.readlink(revlink))
-qvar = args.valuename
-value = readvar(revlink, qvar, mappedpkg)
-if qvar == "PKGSIZE":
-# PKGSIZE is now in bytes, but we we want it in KB
-pkgsize = (int(value) + 1024 // 2) // 1024
-value = "%d" % pkgsize
-if args.unescape:
-import codecs
-# escape_decode() unescapes backslash encodings in byte streams
-value = codecs.escape_decode(bytes(value, 
"utf-8"))[0].decode("utf-8")
+qvars = args.valuenames
+val_names = qvars.split(',')
+values = []
+for qvar in val_names:
+if qvar == "PACKAGE":
+value = mappedpkg
+else:
+value = readvar(revlink, qvar, mappedpkg)
+if qvar == "PKGSIZE":
+# PKGSIZE is now in bytes, but we we want it in KB
+pkgsize = (int(value) + 1024 // 2) // 1024
+value = "%d" % pkgsize
+if args.unescape:
+import codecs
+# escape_decode() unescapes backslash encodings in byte 
streams
+value = codecs.escape_decode(bytes(value, 
"utf-8"))[0].decode("utf-8")
+values.append(value)
+
+values_str = ' '.join(values)

[OE-core] [PATCH] buildhistory: Add output file listing package information

2021-08-12 Thread Andres Beltran 
Currently, buildhistory does not produce a single file combining relevant
information of installed packages. Produce an output file
"installed-package-info.txt" listing a package's runtime name, buildtime name,
its recipe, version, and size to avoid having to look up each package 
externally.
Leave the existing package list files as-is for backwards compatibility.

In order to support this efficiently, extend oe-pkgdata-util to accept multiple 
keys
for its read-value argument.

Signed-off-by: Andres Beltran 
---
 meta/classes/buildhistory.bbclass |  5 +
 scripts/oe-pkgdata-util   | 37 +++
 2 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/meta/classes/buildhistory.bbclass 
b/meta/classes/buildhistory.bbclass
index 8a1359acbe..134b56ab71 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -491,6 +491,11 @@ buildhistory_get_installed() {
cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | 
sort -n -r > $1/installed-package-sizes.txt
rm $1/installed-package-sizes.tmp
 
+   # Produce package info: runtime_name, buildtime_name, recipe, version, 
size
+   oe-pkgdata-util -p ${PKGDATA_DIR} read-value "PACKAGE,PN,PV,PKGSIZE" -n 
-f $pkgcache > $1/installed-package-info.tmp
+   cat $1/installed-package-info.tmp | sort -n -r -k 5 > 
$1/installed-package-info.txt
+   rm $1/installed-package-info.tmp
+
# We're now done with the cache, delete it
rm $pkgcache
 
diff --git a/scripts/oe-pkgdata-util b/scripts/oe-pkgdata-util
index 75dd23efa3..c30baaa580 100755
--- a/scripts/oe-pkgdata-util
+++ b/scripts/oe-pkgdata-util
@@ -171,7 +171,7 @@ def read_value(args):
 val = line.split(': ', 1)[1].rstrip()
 return val
 
-logger.debug("read-value('%s', '%s' '%s')" % (args.pkgdata_dir, 
args.valuename, packages))
+logger.debug("read-value('%s', '%s' '%s')" % (args.pkgdata_dir, 
args.valuenames, packages))
 for package in packages:
 pkg_split = package.split('_')
 pkg_name = pkg_split[0]
@@ -180,20 +180,29 @@ def read_value(args):
 logger.debug(revlink)
 if os.path.exists(revlink):
 mappedpkg = os.path.basename(os.readlink(revlink))
-qvar = args.valuename
-value = readvar(revlink, qvar, mappedpkg)
-if qvar == "PKGSIZE":
-# PKGSIZE is now in bytes, but we we want it in KB
-pkgsize = (int(value) + 1024 // 2) // 1024
-value = "%d" % pkgsize
-if args.unescape:
-import codecs
-# escape_decode() unescapes backslash encodings in byte streams
-value = codecs.escape_decode(bytes(value, 
"utf-8"))[0].decode("utf-8")
+qvars = args.valuenames
+val_names = qvars.split(',')
+values = []
+for qvar in val_names:
+if qvar == "PACKAGE":
+value = mappedpkg
+else:
+value = readvar(revlink, qvar, mappedpkg)
+if qvar == "PKGSIZE":
+# PKGSIZE is now in bytes, but we we want it in KB
+pkgsize = (int(value) + 1024 // 2) // 1024
+value = "%d" % pkgsize
+if args.unescape:
+import codecs
+# escape_decode() unescapes backslash encodings in byte 
streams
+value = codecs.escape_decode(bytes(value, 
"utf-8"))[0].decode("utf-8")
+values.append(value)
+
+values_str = ' '.join(values)
 if args.prefix_name:
-print('%s %s' % (pkg_name, value))
+print('%s %s' % (pkg_name, values_str))
 else:
-print(value)
+print(values_str)
 else:
 logger.debug("revlink %s does not exist", revlink)
 
@@ -570,7 +579,7 @@ def main():
 parser_read_value = subparsers.add_parser('read-value',
   help='Read any pkgdata value for one 
or more packages',
   description='Reads the named value 
from the pkgdata files for the specified packages')
-parser_read_value.add_argument('valuename', help='Name of the value to 
look up')
+parser_read_value.add_argument('valuenames', help='Name of the value/s to 
look up (separated by commas, no spaces)')
 parser_read_value.add_argument('pkg', nargs='*', help='Runtime package 
name to look up')
 parser_read_value.add_argument('-f', '--file', help='Read package names 
from the specified file (one per line, first field only)')
 parser_read_value.add_argument('-n', '--prefix-name', help='Prefix output 
with p