Hello community, here is the log from the commit of package qemu for openSUSE:Factory checked in at 2019-10-30 14:41:22 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/qemu (Old) and /work/SRC/openSUSE:Factory/.qemu.new.2990 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "qemu" Wed Oct 30 14:41:22 2019 rev:161 rq:743387 version:4.1.0 Changes: -------- --- /work/SRC/openSUSE:Factory/qemu/qemu.changes 2019-10-05 16:20:48.989357343 +0200 +++ /work/SRC/openSUSE:Factory/.qemu.new.2990/qemu.changes 2019-10-30 14:41:38.321787916 +0100 @@ -1,0 +2,39 @@ +Sat Oct 26 03:07:00 UTC 2019 - Bruce Rogers <brog...@suse.com> + +- Address potential corruption when using qcow2 images + coroutine-Add-qemu_co_mutex_assert_locke.patch + qcow2-Fix-corruption-bug-in-qcow2_detect.patch +- Include more tweaks to our packaging workflow scripts - this will + continue as we refine the scripts +- Patch queue updated from git://github.com/openSUSE/qemu.git opensuse-4.1 + +------------------------------------------------------------------- +Thu Oct 17 09:15:07 UTC 2019 - Ludwig Nussel <lnus...@suse.de> + +- use %gcc_version for cross compilers (boo#1153703) + +------------------------------------------------------------------- +Tue Oct 15 20:13:27 UTC 2019 - Bruce Rogers <brog...@suse.com> + +- Add upstream edk2 submodule fix for creating tarball +- Switch to upstream patch for avoiding git ref in edk2 makefile +- Fix failing block tests which aren't compatible with the configure + option --enable-membarrier +* Patches dropped: + roms-Makefile.edk2-don-t-invoke-git-sinc.patch + tests-block-io-test-130-needs-some-delay.patch +* Patches added: + make-release-pull-in-edk2-submodules-so-.patch + roms-Makefile.edk2-don-t-pull-in-submodu.patch + tests-Fix-block-tests-to-be-compatible-w.patch +- Patch queue updated from git://github.com/openSUSE/qemu.git opensuse-4.1 + +------------------------------------------------------------------- +Thu Oct 10 01:42:04 UTC 2019 - Bruce Rogers <brog...@suse.com> + +- Reduce the cross compiler versions we rely on +- Fix some qemu-testsuite issues, reducing known error cases + test-add-mapping-from-arch-of-i686-to-qe.patch +- Patch queue updated from git://github.com/openSUSE/qemu.git opensuse-4.1 + +------------------------------------------------------------------- Old: ---- roms-Makefile.edk2-don-t-invoke-git-sinc.patch tests-block-io-test-130-needs-some-delay.patch New: ---- coroutine-Add-qemu_co_mutex_assert_locke.patch make-release-pull-in-edk2-submodules-so-.patch qcow2-Fix-corruption-bug-in-qcow2_detect.patch roms-Makefile.edk2-don-t-pull-in-submodu.patch test-add-mapping-from-arch-of-i686-to-qe.patch tests-Fix-block-tests-to-be-compatible-w.patch ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ qemu.spec ++++++ --- /var/tmp/diff_new_pack.tdJYMB/_old 2019-10-30 14:41:45.913796000 +0100 +++ /var/tmp/diff_new_pack.tdJYMB/_new 2019-10-30 14:41:45.913796000 +0100 @@ -145,46 +145,50 @@ Patch00020: hw-arm-boot.c-Set-NSACR.-CP11-CP10-for-N.patch Patch00021: s390-PCI-fix-IOMMU-region-init.patch Patch00022: hw-core-loader-Fix-possible-crash-in-rom.patch -Patch00023: XXX-dont-dump-core-on-sigabort.patch -Patch00024: qemu-binfmt-conf-Modify-default-path.patch -Patch00025: qemu-cvs-gettimeofday.patch -Patch00026: qemu-cvs-ioctl_debug.patch -Patch00027: qemu-cvs-ioctl_nodirection.patch -Patch00028: linux-user-add-binfmt-wrapper-for-argv-0.patch -Patch00029: PPC-KVM-Disable-mmu-notifier-check.patch -Patch00030: linux-user-binfmt-support-host-binaries.patch -Patch00031: linux-user-Fake-proc-cpuinfo.patch -Patch00032: linux-user-use-target_ulong.patch -Patch00033: Make-char-muxer-more-robust-wrt-small-FI.patch -Patch00034: linux-user-lseek-explicitly-cast-non-set.patch -Patch00035: AIO-Reduce-number-of-threads-for-32bit-h.patch -Patch00036: xen_disk-Add-suse-specific-flush-disable.patch -Patch00037: qemu-bridge-helper-reduce-security-profi.patch -Patch00038: qemu-binfmt-conf-use-qemu-ARCH-binfmt.patch -Patch00039: linux-user-properly-test-for-infinite-ti.patch -Patch00040: roms-Makefile-pass-a-packaging-timestamp.patch -Patch00041: Raise-soft-address-space-limit-to-hard-l.patch -Patch00042: increase-x86_64-physical-bits-to-42.patch -Patch00043: vga-Raise-VRAM-to-16-MiB-for-pc-0.15-and.patch -Patch00044: i8254-Fix-migration-from-SLE11-SP2.patch -Patch00045: acpi_piix4-Fix-migration-from-SLE11-SP2.patch -Patch00046: Switch-order-of-libraries-for-mpath-supp.patch -Patch00047: Make-installed-scripts-explicitly-python.patch -Patch00048: hw-smbios-handle-both-file-formats-regar.patch -Patch00049: xen-add-block-resize-support-for-xen-dis.patch -Patch00050: tests-qemu-iotests-Triple-timeout-of-i-o.patch -Patch00051: tests-block-io-test-130-needs-some-delay.patch -Patch00052: xen-ignore-live-parameter-from-xen-save-.patch -Patch00053: Conditionalize-ui-bitmap-installation-be.patch -Patch00054: tests-change-error-message-in-test-162.patch -Patch00055: hw-usb-hcd-xhci-Fix-GCC-9-build-warning.patch -Patch00056: hw-usb-dev-mtp-Fix-GCC-9-build-warning.patch -Patch00057: hw-intc-exynos4210_gic-provide-more-room.patch -Patch00058: configure-only-populate-roms-if-softmmu.patch -Patch00059: pc-bios-s390-ccw-net-avoid-warning-about.patch -Patch00060: roms-change-cross-compiler-naming-to-be-.patch -Patch00061: roms-Makefile.edk2-don-t-invoke-git-sinc.patch -Patch00062: tests-Disable-some-block-tests-for-now.patch +Patch00023: make-release-pull-in-edk2-submodules-so-.patch +Patch00024: roms-Makefile.edk2-don-t-pull-in-submodu.patch +Patch00025: coroutine-Add-qemu_co_mutex_assert_locke.patch +Patch00026: qcow2-Fix-corruption-bug-in-qcow2_detect.patch +Patch00027: XXX-dont-dump-core-on-sigabort.patch +Patch00028: qemu-binfmt-conf-Modify-default-path.patch +Patch00029: qemu-cvs-gettimeofday.patch +Patch00030: qemu-cvs-ioctl_debug.patch +Patch00031: qemu-cvs-ioctl_nodirection.patch +Patch00032: linux-user-add-binfmt-wrapper-for-argv-0.patch +Patch00033: PPC-KVM-Disable-mmu-notifier-check.patch +Patch00034: linux-user-binfmt-support-host-binaries.patch +Patch00035: linux-user-Fake-proc-cpuinfo.patch +Patch00036: linux-user-use-target_ulong.patch +Patch00037: Make-char-muxer-more-robust-wrt-small-FI.patch +Patch00038: linux-user-lseek-explicitly-cast-non-set.patch +Patch00039: AIO-Reduce-number-of-threads-for-32bit-h.patch +Patch00040: xen_disk-Add-suse-specific-flush-disable.patch +Patch00041: qemu-bridge-helper-reduce-security-profi.patch +Patch00042: qemu-binfmt-conf-use-qemu-ARCH-binfmt.patch +Patch00043: linux-user-properly-test-for-infinite-ti.patch +Patch00044: roms-Makefile-pass-a-packaging-timestamp.patch +Patch00045: Raise-soft-address-space-limit-to-hard-l.patch +Patch00046: increase-x86_64-physical-bits-to-42.patch +Patch00047: vga-Raise-VRAM-to-16-MiB-for-pc-0.15-and.patch +Patch00048: i8254-Fix-migration-from-SLE11-SP2.patch +Patch00049: acpi_piix4-Fix-migration-from-SLE11-SP2.patch +Patch00050: Switch-order-of-libraries-for-mpath-supp.patch +Patch00051: Make-installed-scripts-explicitly-python.patch +Patch00052: hw-smbios-handle-both-file-formats-regar.patch +Patch00053: xen-add-block-resize-support-for-xen-dis.patch +Patch00054: tests-qemu-iotests-Triple-timeout-of-i-o.patch +Patch00055: tests-Fix-block-tests-to-be-compatible-w.patch +Patch00056: xen-ignore-live-parameter-from-xen-save-.patch +Patch00057: Conditionalize-ui-bitmap-installation-be.patch +Patch00058: tests-change-error-message-in-test-162.patch +Patch00059: hw-usb-hcd-xhci-Fix-GCC-9-build-warning.patch +Patch00060: hw-usb-dev-mtp-Fix-GCC-9-build-warning.patch +Patch00061: hw-intc-exynos4210_gic-provide-more-room.patch +Patch00062: configure-only-populate-roms-if-softmmu.patch +Patch00063: pc-bios-s390-ccw-net-avoid-warning-about.patch +Patch00064: roms-change-cross-compiler-naming-to-be-.patch +Patch00065: tests-Disable-some-block-tests-for-now.patch +Patch00066: test-add-mapping-from-arch-of-i686-to-qe.patch # Patches applied in roms/seabios/: Patch01000: seabios-use-python2-explicitly-as-needed.patch Patch01001: seabios-switch-to-python3-as-needed.patch @@ -255,43 +259,19 @@ BuildRequires: brlapi-devel %ifnarch %{ix86} aarch64 BuildRequires: cross-aarch64-binutils -%if 0%{?is_opensuse} -%ifarch riscv64 -BuildRequires: cross-aarch64-gcc8 -%else -BuildRequires: cross-aarch64-gcc9 -%endif -%else -BuildRequires: cross-aarch64-gcc7 -%endif +BuildRequires: cross-aarch64-gcc%gcc_version %endif %ifnarch %{ix86} armv7hl BuildRequires: cross-arm-binutils -%if 0%{?is_opensuse} -%ifarch riscv64 -BuildRequires: cross-arm-gcc8 -%else -BuildRequires: cross-arm-gcc9 -%endif -%else -BuildRequires: cross-arm-gcc7 -%endif +BuildRequires: cross-arm-gcc%gcc_version %endif %if %{build_x86_firmware_from_source} %ifnarch %{ix86} x86_64 # We must cross-compile on non-x86* BuildRequires: cross-i386-binutils -%if 0%{?is_opensuse} -BuildRequires: cross-i386-gcc9 -%else -BuildRequires: cross-i386-gcc7 -%endif +BuildRequires: cross-i386-gcc%gcc_version BuildRequires: cross-x86_64-binutils -%if 0%{?is_opensuse} -BuildRequires: cross-x86_64-gcc9 -%else -BuildRequires: cross-x86_64-gcc7 -%endif +BuildRequires: cross-x86_64-gcc%gcc_version %endif %endif BuildRequires: curl-devel @@ -961,6 +941,10 @@ %patch00060 -p1 %patch00061 -p1 %patch00062 -p1 +%patch00063 -p1 +%patch00064 -p1 +%patch00065 -p1 +%patch00066 -p1 %patch01000 -p1 %patch01001 -p1 %patch01002 -p1 @@ -1428,10 +1412,18 @@ # ------------------------------------------------------------------------ %if "%{name}" == "qemu-testsuite" +# TODO: Some of these are actually overwritten during the following make's ln -s %_bindir/qemu-img qemu-img ln -s %_bindir/qemu-ga qemu-ga ln -s %_bindir/qemu-io qemu-io -# And what about these binaries: qemu-nbd, elf2dmp, ivshmem-client, ivshmem-server, qemu-edid, qemu-keymap, qemu-pr-helper, virtfs-proxy-helper +ln -s %_bindir/elf2map elf2map +ln -s %_bindir/qemu-nbd qemu-nbd +ln -s %_bindir/qemu-edid qemu-edid +ln -s %_bindir/qemu-keymap qemu-keymap +ln -s %_bindir/ivshmem-client ivshmem-client +ln -s %_bindir/ivshmem-server ivshmem-server +ln -s %_bindir/qemu-pr-helper scsi/qemu-pr-helper +ln -s %_bindir/virtfs-proxy-helper fsdev/virtfs-proxy-helper for i in %firmware do @@ -1453,7 +1445,7 @@ # ... make comes in fresh and has lots of address space (needed for 32bit, bsc#957379) make %{?_smp_mflags} check-report.tap V=1 -%endif # qemu +%endif # qemu-testsuite # ------------------------------------------------------------------------ %if "%{name}" == "qemu-linux-user" ++++++ README.PACKAGING ++++++ --- /var/tmp/diff_new_pack.tdJYMB/_old 2019-10-30 14:41:45.993796085 +0100 +++ /var/tmp/diff_new_pack.tdJYMB/_new 2019-10-30 14:41:45.993796085 +0100 @@ -3,32 +3,31 @@ you make be acceptable to the package maintainers. The qemu.spec file is generated from a qemu.spec.in template, so to make changes -to the spec file (besides patch name or order changes), you'll need to edit the -template. The patches included are inserted into the spec file by scripts -described below. +to the spec file (besides the script generated patch references), you'll need to +edit the template, not the spec file. The spec file patch references are inserted +by scripts, which are described below. If you are not modifying any patches or their order, but just need to update the spec file from a changed template, run 'bash ./update_git.sh refresh'. If the set of patches is being modified, including their order, you will want to -first run 'bash ./update_git.sh pkg2git', in order to get the current patch -queue made available in a local git branch named frombundle. (qemu packaging now -includes a "bundle of git bundles" stored as a package source file, which -contains the "git version" of the package patches.) You will create an altered -patch queue in the $GIT_BRANCH branch (see config.sh for actual name), and use -the frombundle branch as a starting point (eg perhaps start by doing git reset ---hard frombundle, then cherry-pick upstream patches from there). Once you have -the patch queue ready to go run 'bash ./update_git.sh git2pkg' to update the -bundle of bundles (bundles.tsr.xz file) as well as the spec and patch files. - -Additional Notes: +first run 'bash ./update_git.sh pkg2git', which makes the current package patch +queue available in a local git branch named frombundle (see confi.sh for the +locations). This set of patches comes from a "bundle of git bundles", the +bundles.tar.xz file, which is included as a package source file. You will then +create an altered patch queue in the $GIT_BRANCH branch ( again,see config.sh), +and use the frombundle branch as the starting point for your changes (eg perhaps +start by doing git reset --hard frombundle, then cherry-pick upstream patches +from there). Once you have the patch queue ready to go run 'bash ./update_git.sh +git2pkg' which updates the bunndles.tar.xz file, as well as the spec and patch +files. ** SPECIAL NOTE ABOUT CURRENT STATUS *** -The current status as of 12 Sep 2019 is that for this incarnation of qemu -packaging, the old update_git.sh workflow is no longer supported, being -suplanted with the workflow described above. THIS IS STILL A WORK IN PROGRESS. +The current status as of 03 Oct 2019 is that for this incarnation of qemu +packaging, the previous workflow which relied on the patches being in a shared +git repo on github or gitlab is no longer supported. This new bundle based +workflos is still a work in progress. If it isn't working right, please contact Bruce Rogers <brog...@suse.com>. -It isn't foolproof by a long shot! Also note that qemu now takes advantage of the MultiBuild feature of the Build Service. No longer do the qemu-linux-user and qemu-testsuite packages have build @@ -38,6 +37,8 @@ - Bruce Rogers *** END SPECIAL NOTE *** +Additional Notes: + Patches which are from an upstream git repo should have the commit id recorded just below the Subject line (after a blank line) as follows: ++++++ bundles.tar.xz ++++++ Binary files old/9e06029aea3b2eca1d5261352e695edc1e7d7b8b.bundle and new/9e06029aea3b2eca1d5261352e695edc1e7d7b8b.bundle differ ++++++ config.sh ++++++ --- /var/tmp/diff_new_pack.tdJYMB/_old 2019-10-30 14:41:46.101796200 +0100 +++ /var/tmp/diff_new_pack.tdJYMB/_new 2019-10-30 14:41:46.101796200 +0100 @@ -4,19 +4,18 @@ # The next few VARIABLES may be edited (or uncommented) as required: -# The commit upon which our patchqueue gets rebased. The special value LATEST -# may be used to "automatically" track the upstream development tree in the -# master branch +# The following specifies the upstream tag or commit upon which our patchqueue +# gets rebased. The special value LATEST may be used to "automatically" track +# the upstream development tree in the master branch GIT_UPSTREAM_COMMIT_ISH=v4.1.0 -if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then - echo "Using LATEST upstream commit as base for tarball and patch queue" - GIT_BRANCH=master +if [[ "$GIT_UPSTREAM_COMMIT_ISH" != "LATEST" ]]; then + # This is the git branch used (otherwise it is computed) + GIT_BRANCH=opensuse-4.1 fi -# otherwise we specify the branch to use, eg: # WARNING: If transitioning from using LATEST to not, MANUALLY re-set the -# tarball present -GIT_BRANCH=opensuse-4.1 -# This is used for the automated development branch tracking +# tarball present. If transitioning TO LATEST, make sure that +# NEXT_RELEASE_IS_MAJOR is set correctly +# This is used to choose the version number when LATEST processing is active NEXT_RELEASE_IS_MAJOR=0 # The shared openSUSE specific git repo, on which $GIT_LOCAL_TREE is based ++++++ coroutine-Add-qemu_co_mutex_assert_locke.patch ++++++ From: Kevin Wolf <kw...@redhat.com> Date: Thu, 24 Oct 2019 16:26:57 +0200 Subject: coroutine: Add qemu_co_mutex_assert_locked() Git-commit: 944f3d5dd216fcd8cb007eddd4f82dced0a15b3d Some functions require that the caller holds a certain CoMutex for them to operate correctly. Add a function so that they can assert the lock is really held. Cc: qemu-sta...@nongnu.org Signed-off-by: Kevin Wolf <kw...@redhat.com> Tested-by: Michael Weiser <michael.wei...@gmx.de> Reviewed-by: Michael Weiser <michael.wei...@gmx.de> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsement...@virtuozzo.com> Reviewed-by: Denis V. Lunev <d...@openvz.org> Reviewed-by: Max Reitz <mre...@redhat.com> Signed-off-by: Bruce Rogers <brog...@suse.com> --- include/qemu/coroutine.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h index 9801e7f5a4978bf4a5a74d33aeb6..f4843b5f595eb0beb1ce2f3e4e39 100644 --- a/include/qemu/coroutine.h +++ b/include/qemu/coroutine.h @@ -167,6 +167,21 @@ void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex); */ void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex); +/** + * Assert that the current coroutine holds @mutex. + */ +static inline coroutine_fn void qemu_co_mutex_assert_locked(CoMutex *mutex) +{ + /* + * mutex->holder doesn't need any synchronisation if the assertion holds + * true because the mutex protects it. If it doesn't hold true, we still + * don't mind if another thread takes or releases mutex behind our back, + * because the condition will be false no matter whether we read NULL or + * the pointer for any other coroutine. + */ + assert(atomic_read(&mutex->locked) && + mutex->holder == qemu_coroutine_self()); +} /** * CoQueues are a mechanism to queue coroutines in order to continue executing ++++++ make-release-pull-in-edk2-submodules-so-.patch ++++++ From: Michael Roth <mdr...@linux.vnet.ibm.com> Date: Thu, 12 Sep 2019 18:12:01 -0500 Subject: make-release: pull in edk2 submodules so we can build it from tarballs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git-commit: 45c61c6c23918e3b05ed9ecac5b2328ebae5f774 The `make efi` target added by 536d2173 is built from the roms/edk2 submodule, which in turn relies on additional submodules nested under roms/edk2. The make-release script currently only pulls in top-level submodules, so these nested submodules are missing in the resulting tarball. We could try to address this situation more generally by recursively pulling in all submodules, but this doesn't necessarily ensure the end-result will build properly (this case also required other changes). Additionally, due to the nature of submodules, we may not always have control over how these sorts of things are dealt with, so for now we continue to handle it on a case-by-case in the make-release script. Cc: Laszlo Ersek <ler...@redhat.com> Cc: Bruce Rogers <brog...@suse.com> Cc: qemu-sta...@nongnu.org # v4.1.0 Reported-by: Bruce Rogers <brog...@suse.com> Reviewed-by: Philippe Mathieu-Daudé <phi...@redhat.com> Tested-by: Philippe Mathieu-Daudé <phi...@redhat.com> Signed-off-by: Michael Roth <mdr...@linux.vnet.ibm.com> Message-Id: <20190912231202.12327-2-mdr...@linux.vnet.ibm.com> Signed-off-by: Philippe Mathieu-Daudé <phi...@redhat.com> Signed-off-by: Bruce Rogers <brog...@suse.com> --- scripts/make-release | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/make-release b/scripts/make-release index b4af9c9e5202cb5c4f092d67ea51..a2a8cda33c0e0da3caa1f8641515 100755 --- a/scripts/make-release +++ b/scripts/make-release @@ -20,6 +20,14 @@ git checkout "v${version}" git submodule update --init (cd roms/seabios && git describe --tags --long --dirty > .version) (cd roms/skiboot && ./make_version.sh > .version) +# Fetch edk2 submodule's submodules, since it won't have access to them via +# the tarball later. +# +# A more uniform way to handle this sort of situation would be nice, but we +# don't necessarily have much control over how a submodule handles its +# submodule dependencies, so we continue to handle these on a case-by-case +# basis for now. +(cd roms/edk2 && git submodule update --init) popd tar --exclude=.git -cjf ${destination}.tar.bz2 ${destination} rm -rf ${destination} ++++++ qcow2-Fix-corruption-bug-in-qcow2_detect.patch ++++++ From: Kevin Wolf <kw...@redhat.com> Date: Thu, 24 Oct 2019 16:26:58 +0200 Subject: qcow2: Fix corruption bug in qcow2_detect_metadata_preallocation() Git-commit: 5e9785505210e2477e590e61b1ab100d0ec22b01 qcow2_detect_metadata_preallocation() calls qcow2_get_refcount() which requires s->lock to be taken to protect its accesses to the refcount table and refcount blocks. However, nothing in this code path actually took the lock. This could cause the same cache entry to be used by two requests at the same time, for different tables at different offsets, resulting in image corruption. As it would be preferable to base the detection on consistent data (even though it's just heuristics), let's take the lock not only around the qcow2_get_refcount() calls, but around the whole function. This patch takes the lock in qcow2_co_block_status() earlier and asserts in qcow2_detect_metadata_preallocation() that we hold the lock. Fixes: 69f47505ee66afaa513305de0c1895a224e52c45 Cc: qemu-sta...@nongnu.org Reported-by: Michael Weiser <michael.wei...@gmx.de> Signed-off-by: Kevin Wolf <kw...@redhat.com> Tested-by: Michael Weiser <michael.wei...@gmx.de> Reviewed-by: Michael Weiser <michael.wei...@gmx.de> Reviewed-by: Vladimir Sementsov-Ogievskiy <vsement...@virtuozzo.com> Reviewed-by: Max Reitz <mre...@redhat.com> Signed-off-by: Bruce Rogers <brog...@suse.com> --- block/qcow2-refcount.c | 2 ++ block/qcow2.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c index ef965d78952637ed92d777db4e9a..0d64bf5a5e9c0359e5391185f6c5 100644 --- a/block/qcow2-refcount.c +++ b/block/qcow2-refcount.c @@ -3455,6 +3455,8 @@ int qcow2_detect_metadata_preallocation(BlockDriverState *bs) int64_t i, end_cluster, cluster_count = 0, threshold; int64_t file_length, real_allocation, real_clusters; + qemu_co_mutex_assert_locked(&s->lock); + file_length = bdrv_getlength(bs->file->bs); if (file_length < 0) { return file_length; diff --git a/block/qcow2.c b/block/qcow2.c index 865839682cd639d1b7aba0cc328f..c0f5439dc8f10d7f920d9b4a29b1 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -1899,6 +1899,8 @@ static int coroutine_fn qcow2_co_block_status(BlockDriverState *bs, unsigned int bytes; int status = 0; + qemu_co_mutex_lock(&s->lock); + if (!s->metadata_preallocation_checked) { ret = qcow2_detect_metadata_preallocation(bs); s->metadata_preallocation = (ret == 1); @@ -1906,7 +1908,6 @@ static int coroutine_fn qcow2_co_block_status(BlockDriverState *bs, } bytes = MIN(INT_MAX, count); - qemu_co_mutex_lock(&s->lock); ret = qcow2_get_cluster_offset(bs, offset, &bytes, &cluster_offset); qemu_co_mutex_unlock(&s->lock); if (ret < 0) { ++++++ qemu.spec.in ++++++ --- /var/tmp/diff_new_pack.tdJYMB/_old 2019-10-30 14:41:46.241796349 +0100 +++ /var/tmp/diff_new_pack.tdJYMB/_new 2019-10-30 14:41:46.241796349 +0100 @@ -160,43 +160,19 @@ BuildRequires: brlapi-devel %ifnarch %{ix86} aarch64 BuildRequires: cross-aarch64-binutils -%if 0%{?is_opensuse} -%ifarch riscv64 -BuildRequires: cross-aarch64-gcc8 -%else -BuildRequires: cross-aarch64-gcc9 -%endif -%else -BuildRequires: cross-aarch64-gcc7 -%endif +BuildRequires: cross-aarch64-gcc%gcc_version %endif %ifnarch %{ix86} armv7hl BuildRequires: cross-arm-binutils -%if 0%{?is_opensuse} -%ifarch riscv64 -BuildRequires: cross-arm-gcc8 -%else -BuildRequires: cross-arm-gcc9 -%endif -%else -BuildRequires: cross-arm-gcc7 -%endif +BuildRequires: cross-arm-gcc%gcc_version %endif %if %{build_x86_firmware_from_source} %ifnarch %{ix86} x86_64 # We must cross-compile on non-x86* BuildRequires: cross-i386-binutils -%if 0%{?is_opensuse} -BuildRequires: cross-i386-gcc9 -%else -BuildRequires: cross-i386-gcc7 -%endif +BuildRequires: cross-i386-gcc%gcc_version BuildRequires: cross-x86_64-binutils -%if 0%{?is_opensuse} -BuildRequires: cross-x86_64-gcc9 -%else -BuildRequires: cross-x86_64-gcc7 -%endif +BuildRequires: cross-x86_64-gcc%gcc_version %endif %endif BuildRequires: curl-devel @@ -1243,10 +1219,18 @@ # ------------------------------------------------------------------------ %if "%{name}" == "qemu-testsuite" +# TODO: Some of these are actually overwritten during the following make's ln -s %_bindir/qemu-img qemu-img ln -s %_bindir/qemu-ga qemu-ga ln -s %_bindir/qemu-io qemu-io -# And what about these binaries: qemu-nbd, elf2dmp, ivshmem-client, ivshmem-server, qemu-edid, qemu-keymap, qemu-pr-helper, virtfs-proxy-helper +ln -s %_bindir/elf2map elf2map +ln -s %_bindir/qemu-nbd qemu-nbd +ln -s %_bindir/qemu-edid qemu-edid +ln -s %_bindir/qemu-keymap qemu-keymap +ln -s %_bindir/ivshmem-client ivshmem-client +ln -s %_bindir/ivshmem-server ivshmem-server +ln -s %_bindir/qemu-pr-helper scsi/qemu-pr-helper +ln -s %_bindir/virtfs-proxy-helper fsdev/virtfs-proxy-helper for i in %firmware do @@ -1268,7 +1252,7 @@ # ... make comes in fresh and has lots of address space (needed for 32bit, bsc#957379) make %{?_smp_mflags} check-report.tap V=1 -%endif # qemu +%endif # qemu-testsuite # ------------------------------------------------------------------------ %if "%{name}" == "qemu-linux-user" ++++++ roms-Makefile.edk2-don-t-pull-in-submodu.patch ++++++ From: Michael Roth <mdr...@linux.vnet.ibm.com> Date: Thu, 12 Sep 2019 18:12:02 -0500 Subject: roms/Makefile.edk2: don't pull in submodules when building from tarball MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git-commit: f3e330e3c319160ac04954399b5a10afc965098c Currently the `make efi` target pulls submodules nested under the roms/edk2 submodule as dependencies. However, when we attempt to build from a tarball this fails since we are no longer in a git tree. A preceding patch will pre-populate these submodules in the tarball, so assume this build dependency is only needed when building from a git tree. Cc: Laszlo Ersek <ler...@redhat.com> Cc: Bruce Rogers <brog...@suse.com> Cc: qemu-sta...@nongnu.org # v4.1.0 Reported-by: Bruce Rogers <brog...@suse.com> Reviewed-by: Laszlo Ersek <ler...@redhat.com> Reviewed-by: Philippe Mathieu-Daudé <phi...@redhat.com> Tested-by: Philippe Mathieu-Daudé <phi...@redhat.com> Signed-off-by: Michael Roth <mdr...@linux.vnet.ibm.com> Message-Id: <20190912231202.12327-3-mdr...@linux.vnet.ibm.com> Signed-off-by: Philippe Mathieu-Daudé <phi...@redhat.com> Signed-off-by: Bruce Rogers <brog...@suse.com> --- roms/Makefile.edk2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roms/Makefile.edk2 b/roms/Makefile.edk2 index c2f2ff59d5b1d8b3fa394ed42c8d..33a074d3a4c21b5460d98bbc7d4a 100644 --- a/roms/Makefile.edk2 +++ b/roms/Makefile.edk2 @@ -46,8 +46,13 @@ all: $(foreach flashdev,$(flashdevs),../pc-bios/edk2-$(flashdev).fd.bz2) \ # files. .INTERMEDIATE: $(foreach flashdev,$(flashdevs),../pc-bios/edk2-$(flashdev).fd) +# Fetch edk2 submodule's submodules. If it is not in a git tree, assume +# we're building from a tarball and that they've already been fetched by +# make-release/tarball scripts. submodules: - cd edk2 && git submodule update --init --force + if test -d edk2/.git; then \ + cd edk2 && git submodule update --init --force; \ + fi # See notes on the ".NOTPARALLEL" target and the "+" indicator in # "tests/uefi-test-tools/Makefile". ++++++ test-add-mapping-from-arch-of-i686-to-qe.patch ++++++ From: Bruce Rogers <brog...@suse.com> Date: Sat, 5 Oct 2019 09:09:42 -0600 Subject: test: add mapping from arch of i686 to qemu_arch=i386 While we don't specifically set QEMU_PROG, the code which detects the host architecture needs a little help mapping the output of uname -m to what the qemu project uses to reference that architecture. Signed-off-by: Bruce Rogers <brog...@suse.com> --- tests/qemu-iotests/common.config | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/qemu-iotests/common.config b/tests/qemu-iotests/common.config index 9bd1a5a6fc8367c336e9f51fe22f..e1c6ffa0cca3a8f14feeb38d6da8 100644 --- a/tests/qemu-iotests/common.config +++ b/tests/qemu-iotests/common.config @@ -24,6 +24,7 @@ PATH=".:$PATH" HOSTOS=$(uname -s) arch=$(uname -m) [[ "$arch" =~ "ppc64" ]] && qemu_arch=ppc64 || qemu_arch="$arch" +[[ "$arch" = "i686" ]] && qemu_arch=i386 # make sure we have a standard umask umask 022 ++++++ tests-Disable-some-block-tests-for-now.patch ++++++ --- /var/tmp/diff_new_pack.tdJYMB/_old 2019-10-30 14:41:46.349796464 +0100 +++ /var/tmp/diff_new_pack.tdJYMB/_new 2019-10-30 14:41:46.349796464 +0100 @@ -7,14 +7,11 @@ now build qemu using multibuild, the qemu-testsuite failure is now more problematic to just ignore. Until we can address these failures, which seem to be tied to the build environment in some way, disable the tests -which are randomly failing. +which are either randomly or reliably failing. -Currently, this patch is based on these failures seen on 02OCT2019 on +Currently, this patch is based on these failures seen on 09OCT2019 on v4.1.0 qemu: -on x86_64: 130, 177, 204, 205, 246, 250 -on i586: 161, 250 -on ppc64le: 161, 182, 250, 250 -on aarch64: 235 +disable 065, 129, 169, 182, 205, 218, 242, 248, 250 (other architectures not evaluated at this time) Signed-off-by: Bruce Rogers <brog...@suse.com> @@ -23,33 +20,37 @@ 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group -index f13e5f2e23417b94bd84d820bcf6..d0da6ec0c8daba541b39ae920f68 100644 +index f13e5f2e23417b94bd84d820bcf6..ed532134bb8af0a1f78a1a2abcf3 100644 --- a/tests/qemu-iotests/group +++ b/tests/qemu-iotests/group -@@ -151,7 +151,7 @@ +@@ -86,7 +86,7 @@ + 062 rw auto quick + 063 rw auto quick + 064 rw quick +-065 rw quick ++#DISABLE FOR NOW 065 rw quick + 066 rw auto quick + 067 rw quick + 068 rw quick +@@ -150,7 +150,7 @@ + 126 rw auto backing 127 rw backing quick 128 rw quick - 129 rw quick --130 rw auto quick -+#DISABLE FOR NOW 130 rw auto quick +-129 rw quick ++#DISABLE FOR NOW 129 rw quick + 130 rw auto quick 131 rw quick 132 rw quick - 133 auto quick -@@ -182,7 +182,7 @@ - 158 rw auto quick - 159 rw auto quick - 160 rw quick --161 rw auto quick -+#DISABLE FOR NOW 161 rw auto quick +@@ -186,7 +186,7 @@ 162 quick 163 rw 165 rw quick -@@ -194,11 +194,11 @@ - 174 auto - 175 quick - 176 rw auto backing --177 rw auto quick -+#DISABLE FOR NOW 177 rw auto quick +-169 rw quick migration ++#DISABLE FOR NOW 169 rw quick migration + 170 rw auto quick + 171 rw quick + 172 auto +@@ -198,7 +198,7 @@ 178 img 179 rw auto quick 181 rw migration @@ -58,34 +59,37 @@ 183 rw migration 184 rw auto quick 185 rw -@@ -219,8 +219,8 @@ - 201 rw migration +@@ -220,7 +220,7 @@ 202 rw quick 203 rw migration --204 rw quick + 204 rw quick -205 rw quick -+#DISABLE FOR NOW 204 rw quick +#DISABLE FOR NOW 205 rw quick 206 rw 207 rw 208 rw quick -@@ -249,7 +249,7 @@ - 232 quick - 233 quick - 234 quick migration --235 quick -+#DISABLE FOR NOW 235 quick - 236 quick - 237 rw quick - 238 quick -@@ -260,11 +260,11 @@ +@@ -233,7 +233,7 @@ + 215 rw quick + 216 rw quick + 217 rw auto quick +-218 rw quick ++#DISABLE FOR NOW 218 rw quick + 219 rw + 220 rw auto + 221 rw quick +@@ -256,15 +256,15 @@ + 239 rw quick + 240 quick + 241 rw quick +-242 rw quick ++#DISABLE FOR NOW 242 rw quick 243 rw quick 244 rw auto quick 245 rw --246 rw quick -+#DISABLE FOR NOW 246 rw quick + 246 rw quick 247 rw quick - 248 rw quick +-248 rw quick ++#DISABLE FOR NOW 248 rw quick 249 rw auto quick -250 rw quick +#DISABLE FOR NOW 250 rw quick ++++++ tests-Fix-block-tests-to-be-compatible-w.patch ++++++ From: Bruce Rogers <brog...@suse.com> Date: Tue, 15 Oct 2019 11:16:14 -0600 Subject: tests: Fix block tests to be compatible with membarrier configuration The use of membarriers collides with the block test's practice of SIGKILLing test vm's. Have them quit politely. Tests: 130, 153, 161 Signed-off-by: Bruce Rogers <brog...@suse.com> --- tests/qemu-iotests/130 | 6 ++++-- tests/qemu-iotests/130.out | 2 ++ tests/qemu-iotests/153 | 6 ++++-- tests/qemu-iotests/153.out | 4 ++++ tests/qemu-iotests/161 | 3 +++ 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/qemu-iotests/130 b/tests/qemu-iotests/130 index 77ad2aa13a06094f26d2c8991e48..fd84a4c77d192e15ee961b07994b 100755 --- a/tests/qemu-iotests/130 +++ b/tests/qemu-iotests/130 @@ -64,7 +64,8 @@ echo _launch_qemu -drive id=testdisk,file="$TEST_IMG",backing.file.filename="$TEST_IMG.base" _send_qemu_cmd $QEMU_HANDLE "commit testdisk" "(qemu)" _send_qemu_cmd $QEMU_HANDLE '' '(qemu)' -_cleanup_qemu +_send_qemu_cmd $QEMU_HANDLE 'quit' '' +wait=1 _cleanup_qemu _img_info | _filter_img_info # Make sure that if there was a backing file that was just overridden on the @@ -73,7 +74,8 @@ _make_test_img -F raw -b "$TEST_IMG.orig" 64M _launch_qemu -drive id=testdisk,file="$TEST_IMG",backing.file.filename="$TEST_IMG.base",backing.driver=$IMGFMT _send_qemu_cmd $QEMU_HANDLE "commit testdisk" "(qemu)" _send_qemu_cmd $QEMU_HANDLE '' '(qemu)' -_cleanup_qemu +_send_qemu_cmd $QEMU_HANDLE 'quit' '' +wait=1 _cleanup_qemu _img_info | _filter_img_info echo diff --git a/tests/qemu-iotests/130.out b/tests/qemu-iotests/130.out index e45285ccc311522481ac1b27ba99..7168bdf70c3eb32d4de0d28bb947 100644 --- a/tests/qemu-iotests/130.out +++ b/tests/qemu-iotests/130.out @@ -11,6 +11,7 @@ virtual size: 64 MiB (67108864 bytes) QEMU X.Y.Z monitor - type 'help' for more information (qemu) commit testdisk (qemu) +(qemu) quit image: TEST_DIR/t.IMGFMT file format: IMGFMT virtual size: 64 MiB (67108864 bytes) @@ -18,6 +19,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t QEMU X.Y.Z monitor - type 'help' for more information (qemu) commit testdisk (qemu) +(qemu) quit image: TEST_DIR/t.IMGFMT file format: IMGFMT virtual size: 64 MiB (67108864 bytes) diff --git a/tests/qemu-iotests/153 b/tests/qemu-iotests/153 index c969a1a16ff8382b9bb69252f6de..39d6da725bff3932a7cb88acff8e 100755 --- a/tests/qemu-iotests/153 +++ b/tests/qemu-iotests/153 @@ -206,7 +206,8 @@ _send_qemu_cmd $QEMU_HANDLE \ 'return' _run_cmd $QEMU_IMG commit -b "${TEST_IMG}.b" "${TEST_IMG}.c" -_cleanup_qemu +_send_qemu_cmd $QEMU_HANDLE "{ 'execute': 'quit' }" '' +wait=1 _cleanup_qemu _launch_qemu @@ -258,7 +259,8 @@ _send_qemu_cmd $QEMU_HANDLE \ _run_cmd $QEMU_IO "${TEST_IMG}" -c 'write 0 512' -_cleanup_qemu +_send_qemu_cmd $QEMU_HANDLE "{ 'execute': 'quit' }" '' +wait=1 _cleanup_qemu echo echo "== Detecting -U and force-share conflicts ==" diff --git a/tests/qemu-iotests/153.out b/tests/qemu-iotests/153.out index e9694200066280fa0637292de3ef..177b08b7e3e6606db552d7e54c8e 100644 --- a/tests/qemu-iotests/153.out +++ b/tests/qemu-iotests/153.out @@ -416,6 +416,8 @@ Is another process using the image [TEST_DIR/t.qcow2]? _qemu_img_wrapper commit -b TEST_DIR/t.qcow2.b TEST_DIR/t.qcow2.c {"return": {}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} +{"return": {}} Adding drive {"return": "OKrn"} @@ -443,6 +445,8 @@ Closing the other {"return": ""} _qemu_io_wrapper TEST_DIR/t.qcow2 -c write 0 512 +{"return": {}} +{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false, "reason": "host-qmp-quit"}} == Detecting -U and force-share conflicts == diff --git a/tests/qemu-iotests/161 b/tests/qemu-iotests/161 index 456a4bd8c4cb54c9c0d295f83bed..990af8a412d72fd522068490638b 100755 --- a/tests/qemu-iotests/161 +++ b/tests/qemu-iotests/161 @@ -68,6 +68,7 @@ _send_qemu_cmd $QEMU_HANDLE \ 'qemu-io none0 \"reopen -o backing.detect-zeroes=on\"' } }" \ "return" +_send_qemu_cmd $QEMU_HANDLE "{ 'execute': 'quit' }" '' _cleanup_qemu # Second test: stream $TEST_IMG.base into $TEST_IMG.int and then @@ -96,6 +97,7 @@ _send_qemu_cmd $QEMU_HANDLE \ 'qemu-io none0 \"reopen -o backing.detect-zeroes=on\"' } }" \ "return" +_send_qemu_cmd $QEMU_HANDLE "{ 'execute': 'quit' }" '' _cleanup_qemu # Third test: commit $TEST_IMG.int into $TEST_IMG.base and then reopen @@ -129,6 +131,7 @@ _send_qemu_cmd $QEMU_HANDLE \ 'qemu-io none0 \"reopen -o backing.detect-zeroes=on\"' } }" \ "return" +_send_qemu_cmd $QEMU_HANDLE "{ 'execute': 'quit' }" '' _cleanup_qemu # success, all done ++++++ update_git.sh ++++++ --- /var/tmp/diff_new_pack.tdJYMB/_old 2019-10-30 14:41:46.365796482 +0100 +++ /var/tmp/diff_new_pack.tdJYMB/_new 2019-10-30 14:41:46.365796482 +0100 @@ -12,6 +12,8 @@ source ./config.sh +declare -A COMMIT_IDS_BY_SUBMODULE_PATH + TEMP_CHECK() { # TEMPORARY! FOR NOW WE REQUIRE THESE LOCALLY TO DO WORK ON PACKAGE REQUIRED_LOCAL_REPO_MAP=( @@ -27,12 +29,12 @@ # TEMPORARY REQUIREMENT! for entry in ${REQUIRED_LOCAL_REPO_MAP[@]}; do if [[ -e $(readlink -f ${entry}) ]]; then - if $(git -C $entry branch| grep -F "$GIT_BRANCH" >/dev/null); then - : - else - echo "Didn't find the $GIT_BRANCH branch in repo at $entry" - exit - fi + if $(git -C $entry branch| grep -F "$GIT_BRANCH" >/dev/null); then + : + else + echo "Didn't find the $GIT_BRANCH branch in repo at $entry" + exit + fi else echo "ERROR! For now, you need to have these local git repos available:" echo ${REQUIRED_LOCAL_REPO_MAP[@]} @@ -43,12 +45,12 @@ #============================================================================== initbundle() { -# What is needed to "start"? -# it all begins with an upstream repo, which may have submodules, incl. recursively (each represents another repo) -# To facilitate speedy work on this upstream repo, we want to have local clones of these repos -# Next we have a tarball, either that we created from the repo, or that upstream provided -# To alter the content of this tarball, lets use git to track these changes, and produce patches which can be included -# in the package spec file +# The bundle tarball has git bundles stored in a directory structure which mimics the +# submodule locations in the containing git repo. Also at that same dir level +# is a file named repo which contains the one line git repo url (with git:// or +# http(s) prefix). The bundles are named as follows: +# "{path/}{git_sha}.{bundle}", where {path/} isn't present for +# the top (qemu) bundle (ie it's for submodules). SUBMODULE_COMMIT_IDS=($(git -C ${LOCAL_REPO_MAP[0]} submodule status --recursive|awk '{print $1}')) SUBMODULE_DIRS=($(git -C ${LOCAL_REPO_MAP[0]} submodule status --recursive|awk '{print $2}')) @@ -58,6 +60,7 @@ echo "ERROR: submodule count doesn't match the REPO_COUNT variable in config.sh file!" exit fi +rm -rf $GIT_DIR rm -rf $BUNDLE_DIR mkdir -p $BUNDLE_DIR for (( i=0; i <$SUBMODULE_COUNT; i++ )); do @@ -65,29 +68,35 @@ # what should this file be? for now use an extension of id touch $BUNDLE_DIR/${SUBMODULE_DIRS[$i]}/${SUBMODULE_COMMIT_IDS[$i]}.id done -# also handle the superproject (I need to make this smarter, or change something - works for tag, but not normal commit: - -GIT_UPSTREAM_COMMIT=$(git -C ${LOCAL_REPO_MAP[0]} show-ref -d $GIT_UPSTREAM_COMMIT_ISH|grep -F "^{}"|awk '{print $1}') +if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then + GIT_UPSTREAM_COMMIT=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse upstream/master) +else +# (I need to make this smarter, or change something - works for tag, but not normal commit?): + GIT_UPSTREAM_COMMIT=$(git -C ${LOCAL_REPO_MAP[0]} show-ref -d $GIT_UPSTREAM_COMMIT_ISH|grep -F "^{}"|awk '{print $1}') +fi touch $BUNDLE_DIR/$GIT_UPSTREAM_COMMIT.id # Now go through all the submodule local repos that are present and create a bundle file for the patches found there -rm -rf $GIT_DIR for (( i=0; i <$REPO_COUNT; i++ )); do if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then - SUBDIR=${PATCH_PATH_MAP[$i]} - GITREPO_COMMIT_ISH=($BUNDLE_DIR/$SUBDIR*.id) - if [[ $GITREPO_COMMIT_ISH =~ .*(.{40})[.]id ]]; then - GITREPO_COMMIT_ISH=${BASH_REMATCH[1]} - fi - echo "Using $GITREPO_COMMIT_ISH" - PATCH_RANGE_INDEX=$i + SUBDIR=${PATCH_PATH_MAP[$i]} + GITREPO_COMMIT_ISH=($BUNDLE_DIR/$SUBDIR*.id) + if [[ $GITREPO_COMMIT_ISH =~ .*(.{40})[.]id ]]; then + GITREPO_COMMIT_ISH=${BASH_REMATCH[1]} + echo "Using $GITREPO_COMMIT_ISH" + PATCH_RANGE_INDEX=$i mkdir -p $GIT_DIR/$SUBDIR git -C $GIT_DIR/$SUBDIR init git -C $GIT_DIR/$SUBDIR remote add origin file://$(readlink -f \ ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) - git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH - git -C $GIT_DIR/$SUBDIR bundle create $BUNDLE_DIR/$SUBDIR$GITREPO_COMMIT_ISH.bundle $GITREPO_COMMIT_ISH..FETCH_HEAD || true - git -C $(readlink -f ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) remote get-url origin >$BUNDLE_DIR/$SUBDIR/repo + git -C $(readlink -f ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) remote get-url origin >$BUNDLE_DIR/$SUBDIR/repo + if [[ $(git -C $GIT_DIR/$SUBDIR ls-remote --heads origin $GIT_BRANCH) ]]; then + git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH + if [[ $(git -C $GIT_DIR/$SUBDIR rev-list $GITREPO_COMMIT_ISH..FETCH_HEAD) ]]; then + git -C $GIT_DIR/$SUBDIR bundle create $BUNDLE_DIR/$SUBDIR$GITREPO_COMMIT_ISH.bundle $GITREPO_COMMIT_ISH..FETCH_HEAD || true + fi + fi + fi fi done tar cJvf bundles.tar.xz -C $BUNDLE_DIR . @@ -113,30 +122,29 @@ fi for (( i=0; i <$REPO_COUNT; i++ )); do if [[ "$SUBDIR" = "${PATCH_PATH_MAP[$i]}" ]]; then - PATCH_RANGE_INDEX=$i - break + PATCH_RANGE_INDEX=$i + break fi done LOCAL_REPO=$(readlink -f ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) if [ -e $LOCAL_REPO ]; then - echo "Found local repo $LOCAL_REPO corresponding to archived git bundle" - else - echo "No local repo $LOCAL_REPO corresponding to archived git bundle" - fi - git -C $LOCAL_REPO remote remove bundlerepo || true +# TODO: Detect if it's there before trying to remove! + git -C $LOCAL_REPO remote remove bundlerepo || true # git won't let you delete this branch if it's the current branch (returns 1) HOW TO HANDLE? # detect this case, and ask user to switch to another branch? or do it for them - switch to master killing any "state" for this branch - git -C $LOCAL_REPO branch -D frombundle || true - git -C $LOCAL_REPO remote add bundlerepo $BUNDLE_DIR/$entry - # in next, the head may be FETCH_HEAD or HEAD depending on how we created: - git -C $LOCAL_REPO fetch bundlerepo FETCH_HEAD - git -C $LOCAL_REPO branch frombundle FETCH_HEAD - git -C $LOCAL_REPO remote remove bundlerepo + git -C $LOCAL_REPO checkout master -f + git -C $LOCAL_REPO branch -D frombundle || true + git -C $LOCAL_REPO remote add bundlerepo $BUNDLE_DIR/$entry +# in next, the head may be FETCH_HEAD or HEAD depending on how we created: + git -C $LOCAL_REPO fetch bundlerepo FETCH_HEAD + git -C $LOCAL_REPO branch frombundle FETCH_HEAD + git -C $LOCAL_REPO remote remove bundlerepo + else + echo "No local repo $LOCAL_REPO corresponding to archived git bundle!" + exit + fi done -echo "For each local repo found, a branch named frombundle is created containing the" -echo "patches from the bundle. Use this as the starting point for making changes to" -echo "the $GIT_BRANCH, which is used when updating the bundle stored with the package." rm -rf $BUNDLE_DIR } @@ -148,14 +156,26 @@ rm -rf $BUNDLE_DIR rm -f checkpatch.log rm -f checkthese +# there's probably a better place for the next: (only needed due to development failures?) +rm -rf checkdir if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then - # This is just a safety valve in case the above gets edited wrong: - if ! [ "$GIT_BRANCH" = "master" ]; then - echo "LATEST implies master branch, please fix configuration" - exit + for (( i=0; i <$REPO_COUNT; i++ )); do + if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then + git -C ${LOCAL_REPO_MAP[$i]} remote update upstream &> /dev/null + fi + done +#TODO: do we really want to checkout here? the code which gets the latest submodule commits doesnt rely on this !!! IN FACT master here isn't for latest upstream - that is the upstream branch! +# git -C ${LOCAL_REPO_MAP[0]} checkout master --recurse-submodules -f +# TODO: THE FOLLOWING NEEDS HELP + QEMU_VERSION=$(git -C ${LOCAL_REPO_MAP[0]} show origin:VERSION) + MAJOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $1}') + MINOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $2}') + if [ "$NEXT_RELEASE_IS_MAJOR" = "0" ]; then + GIT_BRANCH=opensuse-$MAJOR_VERSION.$[$MINOR_VERSION+1] + else + GIT_BRANCH=opensuse-$[$MAJOR_VERSION+1].0 fi - (cd ${LOCAL_REPO_MAP[0]} && git remote update upstream) fi BASE_RE="qemu-[[:digit:]]+(\.[[:digit:]]+){2}(-rc[[:digit:]])?" @@ -184,19 +204,11 @@ echo "Warning: No tarball found" fi -mkdir -p $BUNDLE_DIR # TODO: (repo file not yet done) -# This tarball has git bundles stored in a directory structure which mimics the -# submodule locations in the containing git repo. Also at that same dir level -# is a file named repo which contains the one line git repo url (with git:// or -# http(s) prefix). The bundles are named as follows: -# "{path/}{git_sha}.{bundle}", where {path/} isn't present for -# the top (qemu) bundle (ie it's for submodules). - -tar xJf bundles.tar.xz -C $BUNDLE_DIR -BUNDLE_FILES=$(find $BUNDLE_DIR -printf "%P\n"|grep "bundle$") - if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# DO TARBALL, GETTING ALL FROM UPSTREAM DIRECTLY +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if [[ $QEMU_TARBALL =~ $BASE_RE$EXTRA_RE$SUFFIX_RE ]]; then OLD_COMMIT_ISH=${BASH_REMATCH[3]} else @@ -208,18 +220,26 @@ echo "INFO: Ignoring signature file: $QEMU_TARBALL_SIG" QEMU_TARBALL_SIG= fi +# TODO: HERE WE REFERENCE MASTER - NEEDS FIXING + NEW_COMMIT_ISH_FULL=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse upstream/master) NEW_COMMIT_ISH=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse --short=9 \ - upstream/$GIT_BRANCH) + upstream/master) NOW_SECONDS=$(date +%s) - git clone -ls ${LOCAL_REPO_MAP[0]} $GIT_DIR -b $GIT_BRANCH --single-branch &>/dev/null +# TODO: HERE WE REFERENCE MASTER - NEEDS FIXING + git clone -ls ${LOCAL_REPO_MAP[0]} $GIT_DIR -b master --single-branch &>/dev/null if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then echo "Please wait..." (cd $GIT_DIR && git remote add upstream \ git://git.qemu-project.org/qemu.git &>/dev/null) (cd $GIT_DIR && git remote update upstream &>/dev/null) (cd $GIT_DIR && git checkout $NEW_COMMIT_ISH &>/dev/null) - (cd $GIT_DIR && git submodule update --init --recursive &>/dev/null) +# As an alternative, we could add a --recurse-submodules to the checkout instead here as well, right? +#UPSTREAM DOESNT DO THIS (time takes 17 minutes!): +# (cd $GIT_DIR && git submodule update --init --recursive &>/dev/null) +#INSTEAD THESE NEXT TWO LINES ARE WHAT IS DONE (these take 9 minutes and 3 minutes respectively): + (cd $GIT_DIR && git submodule update --init &>/dev/null) + (cd $GIT_DIR/roms/edk2 && git submodule update --init &>/dev/null) VERSION_EXTRA=+git.$NOW_SECONDS.$NEW_COMMIT_ISH fi QEMU_VERSION=$(cat $GIT_DIR/VERSION) @@ -236,7 +256,7 @@ else SOURCE_VERSION=$MAJOR_VERSION.$MINOR_VERSION.$X fi - if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then + if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then if (cd ${LOCAL_REPO_MAP[0]} && git describe --exact-match $NEW_COMMIT_ISH \ &>/dev/null); then if [ "$X" = "50" ]; then @@ -252,25 +272,6 @@ echo "Almost there..." tar --exclude=.git --transform "s,$GIT_DIR,qemu-$SOURCE_VERSION," \ -Pcf qemu-$SOURCE_VERSION$VERSION_EXTRA.tar $GIT_DIR - echo "New tarball created. Attempting rebase..." - if ! (cd $GIT_DIR && git rebase upstream/$GIT_BRANCH $GIT_BRANCH); then - echo "rebasing master on upstream/master needs human assistance." \ - "Exiting" - (cd $GIT_DIR && git rebase --abort) - rm qemu-$SOURCE_VERSION$VERSION_EXTRA.tar - exit - fi - echo "WARNING: To rebase, master is being checked out" - if ! (cd ${LOCAL_REPO_MAP[0]} && git rebase upstream/$GIT_BRANCH \ - $GIT_BRANCH); then - echo "WARNING: Script error? rebasing master on upstream/master" \ - "succeeded in temp" - echo "dir but failed in local tree! Please investigate" - (cd ${LOCAL_REPO_MAP[0]} && git rebase --abort) - rm qemu-$SOURCE_VERSION$VERSION_EXTRA.tar - exit - fi - echo "Rebase successful" osc rm --force qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz &>/dev/null ||\ true osc rm --force qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz.sig \ @@ -278,10 +279,72 @@ unset QEMU_TARBALL_SIG xz -T 0 qemu-$SOURCE_VERSION$VERSION_EXTRA.tar osc add qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# OK GET THE SUBMODULE COMMIT ID'S FROM THIS NEWLY MINTED QEMU CHECKOUT! WE'LL USE THAT WHEN WE REBASE OUR PATCHES +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# !! We (perhaps temporarily) do MORE recursive submodules, since we are tracking ALL in these scripts, while upstream doesn't include all in tarball currently + (cd $GIT_DIR && git submodule update --init --recursive &>/dev/null) + SUBMODULE_COMMIT_IDS=($(git -C $GIT_DIR submodule status --recursive|awk '{print $1}')) + SUBMODULE_DIRS=($(git -C $GIT_DIR submodule status --recursive|awk '{print $2}')) + SUBMODULE_COUNT=${#SUBMODULE_COMMIT_IDS[@]} +# TODO: do this with simply math - ie: use (( ... )) + if [[ "$REPO_COUNT" != "$(expr $SUBMODULE_COUNT + 1)" ]]; then + echo "ERROR: submodule count doesn't match the REPO_COUNT variable in config.sh file!" + exit + fi +# We have the submodule commits, but not in the PATCH ORDER which our config.sh has (see $PATCH_PATH_MAP) + for (( i=0; i <$REPO_COUNT-1; i++ )); do + COMMIT_IDS_BY_SUBMODULE_PATH[${SUBMODULE_DIRS[$i]}/]=${SUBMODULE_COMMIT_IDS[$i]} + done + COMMIT_IDS_BY_SUBMODULE_PATH[SUPERPROJECT]=$NEW_COMMIT_ISH_FULL +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# MOVE BUNDLE COMMITS OVER TO LOCAL frombundle BRANCH +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + bundle2local + mkdir -p $BUNDLE_DIR + tar xJf bundles.tar.xz -C $BUNDLE_DIR +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# REBASE frombundle patches USING COMMIT_IDS_BY_SUBMODULE, ALSO USING OLD ID'S STORED IN OLD BUNDLE +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# Now go through all the submodule local repos that are present and create a bundle file for the patches found there + for (( i=0; i <$REPO_COUNT; i++ )); do + if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then + if $(git -C ${LOCAL_REPO_MAP[$i]} branch | grep -F "frombundle" >/dev/null); then + SUBDIR=${PATCH_PATH_MAP[$i]} + GITREPO_COMMIT_ISH=($BUNDLE_DIR/$SUBDIR*.id) + if [[ $GITREPO_COMMIT_ISH =~ .*(.{40})[.]id ]]; then + GITREPO_COMMIT_ISH=${BASH_REMATCH[1]} + fi + git -C ${LOCAL_REPO_MAP[$i]} checkout frombundle -f + git -C ${LOCAL_REPO_MAP[$i]} branch -D $GIT_BRANCH + git -C ${LOCAL_REPO_MAP[$i]} checkout -b $GIT_BRANCH + if [[ "$SUBDIR" = "" ]]; then + SUBDIR=SUPERPROJECT + fi + if ! $(git -C ${LOCAL_REPO_MAP[$i]} rebase --onto ${COMMIT_IDS_BY_SUBMODULE_PATH[$SUBDIR]} $GITREPO_COMMIT_ISH >/dev/null); then +# TODO: record that this one needs manual help! + echo "Rebase of ${LOCAL_REPO_MAP[$i]}, branch $GIT_BRANCH needs manual help" + fi + fi + fi + done +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# CREATE BUNDLE FROM $GIT_BRANCH branch +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + initbundle +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# GET BUNDLE PATCHES FROM BUNDLE_DIR +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! fi + rm -rf $GIT_DIR # We're done with GIT_UPSTREAM_COMMIT_ISH carrying the special value LATEST GIT_UPSTREAM_COMMIT_ISH=$NEW_COMMIT_ISH WRITE_LOG=0 +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# DONE WITH LATEST WORK +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! else # not based on LATEST upstream master, rather any upstream commitish if [ "$OLD_SOURCE_VERSION_AND_EXTRA" = "" ]; then echo "Failure: tarball required which corresponds to commitish:" \ @@ -289,60 +352,62 @@ exit fi if [ -d "${LOCAL_REPO_MAP[0]}" ]; then - echo "Processing local git tree branch: $GIT_BRANCH, using commitish:"\ + echo "Processing local git tree branch: master, using commitish:"\ "$GIT_UPSTREAM_COMMIT_ISH" - if ! (cd ${LOCAL_REPO_MAP[0]} && git show-branch $GIT_BRANCH &>/dev/null) + if ! (cd ${LOCAL_REPO_MAP[0]} && git show-branch master &>/dev/null) then - echo "Error: Branch $GIT_BRANCH not found - please create a remote"\ - "tracking branch of origin/$GIT_BRANCH" + echo "Error: Branch master not found - please create a remote"\ + "tracking branch of origin/master" exit fi - for entry in ${BUNDLE_FILES[@]}; do - if [[ $entry =~ ^(.*)[/]*([a-f0-9]{40})[.]bundle$ ]]; then - SUBDIR=${BASH_REMATCH[1]} - GITREPO_COMMIT_ISH=${BASH_REMATCH[2]} - else - echo "ERROR! BAD BUNDLE CONTENT!" - exit - fi - for (( i=0; i <$REPO_COUNT; i++ )); do - if [[ "$SUBDIR" = "${PATCH_PATH_MAP[$i]}" ]]; then - PATCH_RANGE_INDEX=$i - break - fi - done - -# !!!!! REVIEW WHERE THIS mkdir SHOULD BE HAPPENING (kind of replaces the clone operation) - mkdir -p $GIT_DIR/$SUBDIR - git -C $GIT_DIR/$SUBDIR init - git -C $GIT_DIR/$SUBDIR remote add origin file://$(readlink -f \ - ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) - git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH - git -C $GIT_DIR/$SUBDIR reset --hard $GITREPO_COMMIT_ISH - git -C $GIT_DIR/$SUBDIR remote add bundle $BUNDLE_DIR/$entry - # depending on how created, the bundle's head is called HEAD or FETCH_HEAD - #git -C $GIT_DIR/$SUBDIR fetch bundle HEAD - git -C $GIT_DIR/$SUBDIR fetch bundle FETCH_HEAD - git -C $GIT_DIR/$SUBDIR format-patch -N --suffix= --no-renames -o $CMP_DIR -k --stat=72 \ - --indent-heuristic --zero-commit --no-signature --full-index \ - --src-prefix=a/$SUBDIR --dst-prefix=b/$SUBDIR \ - --start-number=$(expr $PATCH_RANGE_INDEX \* $PATCH_RANGE) \ - $GITREPO_COMMIT_ISH..FETCH_HEAD > /dev/null - done - # ( THIS ISNT WORKING - IS OLD HISTORY:) else echo "Processing $GIT_BRANCH branch of remote git tree, using"\ "commitish: $GIT_UPSTREAM_COMMIT_ISH" echo "(For fast processing, consider establishing a local git tree"\ "at ${LOCAL_REPO_MAP[0]})" -# NYI - should be able to combine with local case for the most part fi - QEMU_VERSION=$(cat $GIT_DIR/VERSION) SOURCE_VERSION=$OLD_SOURCE_VERSION_AND_EXTRA + QEMU_VERSION=$(tar JxfO qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz qemu-$SOURCE_VERSION/VERSION) NEW_COMMIT_ISH= WRITE_LOG=1 fi +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# NOW PROCESS BUNDLES INTO COMMITS AND FILL SPEC FILE +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +mkdir -p $BUNDLE_DIR +tar xJf bundles.tar.xz -C $BUNDLE_DIR +BUNDLE_FILES=$(find $BUNDLE_DIR -printf "%P\n"|grep "bundle$") + +for entry in ${BUNDLE_FILES[@]}; do + if [[ $entry =~ ^(.*)[/]*([a-f0-9]{40})[.]bundle$ ]]; then + SUBDIR=${BASH_REMATCH[1]} + GITREPO_COMMIT_ISH=${BASH_REMATCH[2]} + else + echo "ERROR! BAD BUNDLE CONTENT!" + exit + fi + for (( i=0; i <$REPO_COUNT; i++ )); do + if [[ "$SUBDIR" = "${PATCH_PATH_MAP[$i]}" ]]; then + PATCH_RANGE_INDEX=$i + break + fi + done + + mkdir -p $GIT_DIR/$SUBDIR + git -C $GIT_DIR/$SUBDIR init + git -C $GIT_DIR/$SUBDIR remote add origin file://$(readlink -f \ + ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) + git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH + git -C $GIT_DIR/$SUBDIR reset --hard $GITREPO_COMMIT_ISH + git -C $GIT_DIR/$SUBDIR remote add bundle $BUNDLE_DIR/$entry + git -C $GIT_DIR/$SUBDIR fetch bundle FETCH_HEAD + git -C $GIT_DIR/$SUBDIR format-patch -N --suffix= --no-renames -o $CMP_DIR -k --stat=72 \ + --indent-heuristic --zero-commit --no-signature --full-index \ + --src-prefix=a/$SUBDIR --dst-prefix=b/$SUBDIR \ + --start-number=$(expr $PATCH_RANGE_INDEX \* $PATCH_RANGE) \ + $GITREPO_COMMIT_ISH..FETCH_HEAD > /dev/null +done rm -rf $GIT_DIR rm -rf $BUNDLE_DIR @@ -591,8 +656,8 @@ usage() { echo "Usage:" - echo "bash ./git_update.sh <command>: script to manage package maintenance" - echo "using a git-based workflow. Commands are as follows:" + echo "bash ./git_update.sh <command> + echo description: package maintenance using a git-based workflow. Commands:" echo " git2pkg (update package spec file and patches from git. Is default)" echo " pkg2git (update git (frombundle branch) from the package "bundleofbundles")" echo " refresh (refresh spec file from spec file template and "bundlofbundles")" @@ -600,6 +665,17 @@ #============================================================================== +# LATEST processing currently doesn't expect cmdline params, so do it here, up front +if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then + echo "Processing latest upstream changes" + echo "(If SUCCESS is not printed upon completion, see /tmp/latest.log for issues)" + TEMP_CHECK + bundle2spec &> /tmp/latest.log + echo "SUCCESS" + tail -9 /tmp/latest.log + exit +fi + if [ "$1" = "" ]; then set -- git2pkg fi