Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package diffoscope for openSUSE:Factory checked in at 2021-09-11 22:24:32 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/diffoscope (Old) and /work/SRC/openSUSE:Factory/.diffoscope.new.1899 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "diffoscope" Sat Sep 11 22:24:32 2021 rev:17 rq:918464 version:183 Changes: -------- --- /work/SRC/openSUSE:Factory/diffoscope/diffoscope.changes 2021-08-28 22:29:22.241990430 +0200 +++ /work/SRC/openSUSE:Factory/.diffoscope.new.1899/diffoscope.changes 2021-09-11 22:25:17.459431586 +0200 @@ -1,0 +2,30 @@ +Sat Sep 4 17:53:01 UTC 2021 - Sebastian Wagner <sebix+novell....@sebix.at> + +- update to version 183: + - Add support for extracting Android signing blocks. + (Closes: reproducible-builds/diffoscope#246) + - Format debug messages for elf sections using our + diffoscope.utils.format_class utility. + - Clarify a comment about the HUGE_TOOLS dict in diffoscope.external_tools. + - Clarify output around APK Signing Blocks and remove an accidental duplicate + "0x" prefix. + +------------------------------------------------------------------- +Sun Aug 29 09:41:54 UTC 2021 - Sebastian Wagner <sebix+novell....@sebix.at> + +- update to version 182: + - Also ignore, for example, spurious "fwGCC: (Debian ... )" lines in output + from strings(1). + - Only use "java -jar /path/to/apksigner.jar" if we have a .jar as newer + versions of apksigner use a shell wrapper script which will obviously be + rejected by the JVM. Also mention in the diff if apksigner is missing. + - Pass "-f" to apktool to avoid creating a strangely-named subdirectory and + to simplify code. + - If we specify a suffix for temporary file or directory, ensure it starts + with a "_" to make the generated filenames more human-readable. + - Drop an unused File import. + - Update the minimum version of the Black source code formatter. + - Support parsing the return value of squashfs versions which discriminate + between fatal and non-fatal errors. + +------------------------------------------------------------------- Old: ---- diffoscope-181.tar.bz2 diffoscope-181.tar.bz2.asc New: ---- diffoscope-183.tar.bz2 diffoscope-183.tar.bz2.asc ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ diffoscope.spec ++++++ --- /var/tmp/diff_new_pack.BEVcHW/_old 2021-09-11 22:25:17.855431976 +0200 +++ /var/tmp/diff_new_pack.BEVcHW/_new 2021-09-11 22:25:17.859431980 +0200 @@ -17,7 +17,7 @@ Name: diffoscope -Version: 181 +Version: 183 Release: 0 Summary: In-depth comparison of files, archives, and directories License: GPL-3.0-or-later ++++++ diffoscope-181.tar.bz2 -> diffoscope-183.tar.bz2 ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/debian/changelog new/diffoscope-183/debian/changelog --- old/diffoscope-181/debian/changelog 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/debian/changelog 2021-09-03 12:14:57.000000000 +0200 @@ -1,3 +1,39 @@ +diffoscope (183) unstable; urgency=medium + + [ Chris Lamb ] + * Add support for extracting Android signing blocks. + (Closes: reproducible-builds/diffoscope#246) + * Format debug messages for elf sections using our + diffoscope.utils.format_class utility. + * Clarify a comment about the HUGE_TOOLS dict in diffoscope.external_tools. + + [ Felix C. Stegerman ] + * Clarify output around APK Signing Blocks and remove an accidental duplicate + "0x" prefix. + + -- Chris Lamb <la...@debian.org> Fri, 03 Sep 2021 11:14:32 +0100 + +diffoscope (182) unstable; urgency=medium + + [ Chris Lamb ] + * Also ignore, for example, spurious "fwGCC: (Debian ... )" lines in output + from strings(1). + * Only use "java -jar /path/to/apksigner.jar" if we have a .jar as newer + versions of apksigner use a shell wrapper script which will obviously be + rejected by the JVM. Also mention in the diff if apksigner is missing. + * Pass "-f" to apktool to avoid creating a strangely-named subdirectory and + to simplify code. + * If we specify a suffix for temporary file or directory, ensure it starts + with a "_" to make the generated filenames more human-readable. + * Drop an unused File import. + * Update the minimum version of the Black source code formatter. + + [ Santiago Torres Arias ] + * Support parsing the return value of squashfs versions which discriminate + between fatal and non-fatal errors. + + -- Chris Lamb <la...@debian.org> Fri, 27 Aug 2021 09:11:44 +0100 + diffoscope (181) unstable; urgency=medium [ Chris Lamb ] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/debian/control new/diffoscope-183/debian/control --- old/diffoscope-181/debian/control 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/debian/control 2021-09-03 12:14:57.000000000 +0200 @@ -8,6 +8,7 @@ Mattia Rizzolo <mat...@debian.org>, Build-Depends: abootimg <!nocheck>, + androguard <!nocheck>, apksigner <!nocheck>, apktool [!ppc64el !s390x] <!nocheck>, bash-completion, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/debian/tests/control new/diffoscope-183/debian/tests/control --- old/diffoscope-181/debian/tests/control 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/debian/tests/control 2021-09-03 12:14:57.000000000 +0200 @@ -7,7 +7,7 @@ # $ mv debian/tests/control.tmp debian/tests/control Tests: pytest-with-recommends -Depends: python3-all, diffoscope, black, python3-pytest, python3-h5py, file, linux-image-amd64 [amd64] | linux-image-generic [amd64], abootimg, acl, apksigner, apktool [!ppc64el !s390x], binutils-multiarch, bzip2, caca-utils, colord, db-util, default-jdk-headless | default-jdk | java-sdk, device-tree-compiler, docx2txt, e2fsprogs, enjarify, ffmpeg, fontforge-extras, fp-utils [!ppc64el !s390x], genisoimage, gettext, ghc, ghostscript, giflib-tools, gnumeric, gnupg, gnupg-utils, hdf5-tools, imagemagick, jsbeautifier, libarchive-tools, llvm, lz4 | liblz4-tool, mono-utils, ocaml-nox, odt2txt, oggvideotools [!s390x], openssh-client, openssl, pgpdump, poppler-utils, procyon-decompiler, python3-pdfminer, r-base-core, rpm2cpio, sng, sqlite3, squashfs-tools, tcpdump, u-boot-tools, unzip, wabt, xmlbeans, xxd | vim-common, xz-utils, zip, zstd, python3-argcomplete, python3-binwalk, python3-defusedxml, python3-distro, python3-guestfs, python3-jsondiff, python3-progressbar, python3-pypdf2, python3 -debian, python3-pyxattr, python3-rpm, python3-tlsh +Depends: python3-all, diffoscope, black, python3-pytest, python3-h5py, file, linux-image-amd64 [amd64] | linux-image-generic [amd64], abootimg, acl, apksigner, apktool [!ppc64el !s390x], binutils-multiarch, bzip2, caca-utils, colord, db-util, default-jdk-headless | default-jdk | java-sdk, device-tree-compiler, docx2txt, e2fsprogs, enjarify, ffmpeg, fontforge-extras, fp-utils [!ppc64el !s390x], genisoimage, gettext, ghc, ghostscript, giflib-tools, gnumeric, gnupg, gnupg-utils, hdf5-tools, imagemagick, jsbeautifier, libarchive-tools, llvm, lz4 | liblz4-tool, mono-utils, ocaml-nox, odt2txt, oggvideotools [!s390x], openssh-client, openssl, pgpdump, poppler-utils, procyon-decompiler, python3-pdfminer, r-base-core, rpm2cpio, sng, sqlite3, squashfs-tools, tcpdump, u-boot-tools, unzip, wabt, xmlbeans, xxd | vim-common, xz-utils, zip, zstd, python3-androguard, python3-argcomplete, python3-binwalk, python3-defusedxml, python3-distro, python3-guestfs, python3-jsondiff, python3-progressbar, pyt hon3-pypdf2, python3-debian, python3-pyxattr, python3-rpm, python3-tlsh Tests: pytest Depends: python3-all, diffoscope, python3-pytest, python3-h5py, file, python3-tlsh diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/diffoscope/__init__.py new/diffoscope-183/diffoscope/__init__.py --- old/diffoscope-181/diffoscope/__init__.py 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/diffoscope/__init__.py 2021-09-03 12:14:57.000000000 +0200 @@ -17,4 +17,4 @@ # You should have received a copy of the GNU General Public License # along with diffoscope. If not, see <https://www.gnu.org/licenses/>. -VERSION = "181" +VERSION = "183" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/diffoscope/comparators/apk.py new/diffoscope-183/diffoscope/comparators/apk.py --- old/diffoscope-181/diffoscope/comparators/apk.py 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/diffoscope/comparators/apk.py 2021-09-03 12:14:57.000000000 +0200 @@ -18,6 +18,8 @@ # along with diffoscope. If not, see <https://www.gnu.org/licenses/>. import re +import binascii +import textwrap import os.path import logging import itertools @@ -25,10 +27,13 @@ from diffoscope.difference import Difference from diffoscope.exc import RequiredToolNotFound -from diffoscope.tools import tool_required, find_executable +from diffoscope.tools import ( + tool_required, + find_executable, + python_module_missing, +) from diffoscope.tempfiles import get_temporary_directory -from .utils.file import File from .utils.archive import Archive from .utils.command import Command from .utils.compare import compare_files @@ -37,6 +42,12 @@ logger = logging.getLogger(__name__) +try: + import androguard +except ImportError: + python_module_missing("androguard") + androguard = None + class ApkContainer(Archive): @property @@ -48,22 +59,22 @@ def open_archive(self): self._members = [] self._tmpdir = get_temporary_directory(suffix="apk") - self._unpacked = os.path.join( - self._tmpdir.name, os.path.basename(self.source.name) - ) self._andmanifest = None self._andmanifest_orig = None - logger.debug("Extracting %s to %s", self.source.name, self._unpacked) + logger.debug( + "Extracting %s to %s", self.source.name, self._tmpdir.name + ) subprocess.check_call( ( "apktool", "d", + "-f", "-k", "-m", "-o", - self._unpacked, + self._tmpdir.name, self.source.path, ), stderr=None, @@ -73,7 +84,7 @@ # Optionally extract a few files that apktool does not for x in ("classes.dex", "resources.arsc"): subprocess.call( - ("unzip", "-d", self._unpacked, self.source.path, x), + ("unzip", "-d", self._tmpdir.name, self.source.path, x), stderr=subprocess.PIPE, stdout=subprocess.PIPE, ) @@ -85,7 +96,7 @@ ( "unzip", "-d", - self._unpacked, + self._tmpdir.name, self.source.path, f"classes{x}.dex", ), @@ -95,28 +106,28 @@ except subprocess.CalledProcessError: break - for root, _, files in os.walk(self._unpacked): + for root, _, files in os.walk(self._tmpdir.name): current_dir = [] for filename in files: abspath = os.path.join(root, filename) - # apktool.yml is a file created by apktool and containing - # metadata information. Rename it to clarify and always make it - # appear at the beginning of the directory listing for + # apktool.yml is a file created by apktool which contains + # metadata information. We eename it for clarity and always + # make it appear at the beginning of the directory listing for # reproducibility. if filename == "apktool.yml": abspath = filter_apk_metadata( abspath, os.path.basename(self.source.path) ) - relpath = abspath[len(self._unpacked) + 1 :] + relpath = abspath[len(self._tmpdir.name) + 1 :] current_dir.insert(0, relpath) continue - relpath = abspath[len(self._unpacked) + 1 :] + relpath = abspath[len(self._tmpdir.name) + 1 :] if filename == "AndroidManifest.xml": - containing_dir = root[len(self._unpacked) + 1 :] + containing_dir = root[len(self._tmpdir.name) + 1 :] if containing_dir == "original": self._andmanifest_orig = relpath if containing_dir == "": @@ -146,7 +157,7 @@ return self._members def extract(self, member_name, dest_dir): - return os.path.join(self._unpacked, member_name) + return os.path.join(self._tmpdir.name, member_name) def compare_manifests(self, other): my_android_manifest = self.get_android_manifest() @@ -193,15 +204,18 @@ @tool_required("apksigner") def cmdline(self): - # In Debian, the `apksigner` binary is a symbolic link to the .jar file - # itself, requiring binfmt_misc support to execute directly. We - # therefore resolve its location and pass that to `java -jar`. - apksigner_jar = find_executable("apksigner") - - return [ - "java", - "-jar", - apksigner_jar, + # Older versions of the `apksigner` binary under /usr/bin (or similar) + # are a symbolic link to the apksigner .jar file. If we detect a .jar + # we resolve its 'real' location and pass that to `java -jar`, so we + # don't need kernel binfmt_misc to execute. We can't do this in all + # situations as later versions of apksigner use a wrapper script and + # will therefore fail to run at all if we use `java -jar`. + apksigner = os.path.realpath(find_executable("apksigner")) + + prefix = ["java", "-jar"] if apksigner.endswith(".jar") else [] + + return prefix + [ + apksigner, "verify", "--verbose", "--print-certs", @@ -223,13 +237,47 @@ x = Difference.from_operation(Apksigner, self.path, other.path) if x is not None: differences.insert(0, x) - except RequiredToolNotFound: # noqa + except RequiredToolNotFound as exc: # noqa # Don't require apksigner - pass + self.add_comment(exc.get_comment()) + + if androguard is None: + self.add_comment( + "'androguard' Python package not installed; cannot extract V2 signing keys." + ) + else: + x = Difference.from_text_readers( + get_v2_signing_keys(self.path), + get_v2_signing_keys(other.path), + self.path, + other.path, + source="APK Signing Block", + ) + if x is not None: + differences.insert(0, x) return differences +def get_v2_signing_keys(path): + from androguard.core.bytecodes import apk + + try: + instance = apk.APK(path) + instance.parse_v2_signing_block() + except Exception: + return "" + + def format_key(x): + return "\n".join(textwrap.wrap(binascii.hexlify(x).decode("utf-8"))) + + output = [] + for k, v in sorted(instance._v2_blocks.items()): + output.append("Key {}:\n{}\n".format(hex(k), format_key(v))) + + return "\n".join(output) + + def filter_apk_metadata(filepath, archive_name): new_filename = os.path.join(os.path.dirname(filepath), "APK metadata") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/diffoscope/comparators/elf.py new/diffoscope-183/diffoscope/comparators/elf.py --- old/diffoscope-181/diffoscope/comparators/elf.py 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/diffoscope/comparators/elf.py 2021-09-03 12:14:57.000000000 +0200 @@ -28,6 +28,7 @@ from diffoscope.config import Config from diffoscope.tempfiles import get_named_temporary_file from diffoscope.difference import Difference +from diffoscope.utils import format_class from .deb import DebFile, get_build_id_map from .decompile import DecompilableContainer @@ -491,7 +492,12 @@ ][0] logger.debug( - "Adding section %s (%s) as %s", name, type, elf_class + "Adding section %s (%s) as %s", + name, + type, + format_class( + elf_class, strip="diffoscope.comparators.elf." + ), ) self._sections[name] = elf_class(self, name) @@ -641,7 +647,7 @@ class Strings(Command): re_debug_line = re.compile(r"^\s?\w{38,40}\.debug\n$") - re_gcc_line = re.compile(r"^.?GCC: \([^\)]+\)") + re_gcc_line = re.compile(r"^.{1,4}GCC: \([^\)]+\)") @tool_required("strings") def cmdline(self): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/diffoscope/comparators/macho.py new/diffoscope-183/diffoscope/comparators/macho.py --- old/diffoscope-181/diffoscope/comparators/macho.py 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/diffoscope/comparators/macho.py 2021-09-03 12:14:57.000000000 +0200 @@ -224,42 +224,42 @@ class LlvmFileHeaders(LlvmReadobj): - """ Display file headers """ + """Display file headers""" def readobj_options(self): return super().readobj_options() + ["--file-headers"] class LlvmNeededLibs(LlvmReadobj): - """ Display the needed libraries """ + """Display the needed libraries""" def readobj_options(self): return super().readobj_options() + ["--needed-libs"] class LlvmSymbols(LlvmReadobj): - """ Display the symbol table """ + """Display the symbol table""" def readobj_options(self): return super().readobj_options() + ["--symbols"] class LlvmDynSymbols(LlvmReadobj): - """ Display the dynamic symbol table """ + """Display the dynamic symbol table""" def readobj_options(self): return super().readobj_options() + ["--dyn-symbols"] class LlvmRelocations(LlvmReadobj): - """ Display the relocation entries in the fil """ + """Display the relocation entries in the fil""" def readobj_options(self): return super().readobj_options() + ["--relocations"] class LlvmDynRelocations(LlvmReadobj): - """ Display the dynamic relocation entries in the file """ + """Display the dynamic relocation entries in the file""" def readobj_options(self): return super().readobj_options() + ["--dyn-relocations"] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/diffoscope/comparators/squashfs.py new/diffoscope-183/diffoscope/comparators/squashfs.py --- old/diffoscope-181/diffoscope/comparators/squashfs.py 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/diffoscope/comparators/squashfs.py 2021-09-03 12:14:57.000000000 +0200 @@ -280,13 +280,16 @@ output, stderr = p.communicate() if p.returncode != 0: - # unsquashfs(1) exits with 1 (with a suitable logging messages that - # we can check for) if it could not extract, for example, character - # devices that require superuser privileges. In this case, don't - # treat this as a failure that requires reverting to xxd(1), but do - # let the user know via a comment. + # unsquashfs(1) exits with 1 (with a suitable logging messages + # that we can check for) if it could not extract, for example, + # character devices that require superuser privileges. In this + # case, don't treat this as a failure that requires reverting to + # xxd(1), but do let the user know via a comment. After 4.5, this + # return code can also be 2, to distinguish between fatal and + # non-fatal errors (see upstream commit + # 06fc061ef426672576075f2989f52acd873c0141) if ( - p.returncode == 1 + p.returncode in {1, 2} and b"because you're not superuser" in stderr and b"\n\ncreated " in output ): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/diffoscope/external_tools.py new/diffoscope-183/diffoscope/external_tools.py --- old/diffoscope-181/diffoscope/external_tools.py 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/diffoscope/external_tools.py 2021-09-03 12:14:57.000000000 +0200 @@ -1,7 +1,7 @@ # # diffoscope: in-depth comparison of files, archives, and directories # -# Copyright ?? 2017-2020 Chris Lamb <la...@debian.org> +# Copyright ?? 2017-2021 Chris Lamb <la...@debian.org> # Copyright ?? 2021 Jean-Romain Garnier <sa...@jean-romain.com> # # diffoscope is free software: you can redistribute it and/or modify @@ -24,6 +24,7 @@ EXTERNAL_TOOLS = { "abootimg": {"debian": "abootimg", "guix": "abootimg"}, + "androguard": {"debian": "androguard"}, "apktool": {"debian": "apktool"}, "apksigner": {"debian": "apksigner"}, "db_dump": {"debian": "db-util", "guix": "bdb"}, @@ -249,10 +250,10 @@ # version (so that our tests don't break) then it's unnecessary to add it here. GNU_TOOL_NAMES = {"diff", "readelf", "objcopy", "objdump"} -# Set of tools considered "large" their installation size, or too niche in -# their target users. To be easily excluded from installation if not -# specifically required. -# These are the names of the tools, not package names. +# Set of tools considered "large" in their installation size or too niche in +# their target users. This is so they can easily be excluded from installation +# if not required in most cases. Note that these are the names of the tools +# (ie. the keys in the EXTERNAL_TOOLS dict), not the package names. HUGE_TOOLS = { "ghc", "ocamlobjinfo", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/diffoscope/tempfiles.py new/diffoscope-183/diffoscope/tempfiles.py --- old/diffoscope-181/diffoscope/tempfiles.py 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/diffoscope/tempfiles.py 2021-09-03 12:14:57.000000000 +0200 @@ -35,6 +35,9 @@ def get_named_temporary_file(*args, **kwargs): kwargs["dir"] = kwargs.pop("dir", _get_base_temporary_directory()) + if not kwargs.get("suffix", "_").startswith("_"): + kwargs["suffix"] = "_{}".format(kwargs["suffix"]) + f = tempfile.NamedTemporaryFile(*args, **kwargs) _FILES.append(f.name) @@ -61,6 +64,9 @@ """ kwargs["dir"] = kwargs.pop("dir", _get_base_temporary_directory()) + if not kwargs.get("suffix", "_").startswith("_"): + kwargs["suffix"] = "_{}".format(kwargs["suffix"]) + d = tempfile.TemporaryDirectory(*args, **kwargs) return d diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/setup.py new/diffoscope-183/setup.py --- old/diffoscope-181/setup.py 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/setup.py 2021-09-03 12:14:57.000000000 +0200 @@ -58,6 +58,7 @@ "distro_detection": ["distro"], "cmdline": ["argcomplete", "progressbar"], "comparators": [ + "androguard", "binwalk", "defusedxml", "guestfs", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/diffoscope-181/tests/test_source.py new/diffoscope-183/tests/test_source.py --- old/diffoscope-181/tests/test_source.py 2021-08-20 11:03:38.000000000 +0200 +++ new/diffoscope-183/tests/test_source.py 2021-09-03 12:14:57.000000000 +0200 @@ -245,7 +245,7 @@ return out.strip().decode("utf-8").rsplit(" ", 1)[-1] -@skip_unless_tool_is_at_least("black", black_version, "20.8b1") +@skip_unless_tool_is_at_least("black", black_version, "21.4b2") def test_code_is_black_clean(): output = subprocess.check_output( ("black", "--diff", "."), stderr=subprocess.PIPE
