Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package os-autoinst for openSUSE:Factory checked in at 2026-09-28 13:24:38 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/os-autoinst (Old) and /work/SRC/openSUSE:Factory/.os-autoinst.new.383539 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "os-autoinst" Mon Sep 28 13:24:38 2026 rev:624 rq:1381143 version:5.1790583840.ff0d715 Changes: -------- --- /work/SRC/openSUSE:Factory/os-autoinst/os-autoinst.changes 2026-09-28 10:44:57.035176753 +0200 +++ /work/SRC/openSUSE:Factory/.os-autoinst.new.383539/os-autoinst.changes 2026-09-28 13:24:45.148985286 +0200 @@ -1,0 +2,24 @@ +Mon Sep 28 08:24:13 UTC 2026 - [email protected] + +- Update to version 5.1790583840.ff0d715: + * fix(svirt): Verify VMware images while copying them into the datastore + * feat: gracefully handle temporarily unreachable hypervisors + * fix: remove redundant and invalid ruff selectors + * fix(test): Ignore flaky test on ppc64le as well + * feat: support holding key for duration in send_key + * feat: Identify executed commands directly from timeline step boxes + * fix(git): remove leading slash from cache path + +------------------------------------------------------------------- +Mon Sep 28 04:47:31 UTC 2026 - [email protected] + +- Update to version 5.1790570842.2a70c16: + * feat: gracefully handle temporarily unreachable hypervisors + * fix: remove redundant and invalid ruff selectors + * fix(test): Ignore flaky test on ppc64le as well + * test(t/18-backend-qemu): mock IO::Socket::IP to fix port check race + * feat: support holding key for duration in send_key + * feat: Identify executed commands directly from timeline step boxes + * fix(git): remove leading slash from cache path + +------------------------------------------------------------------- Old: ---- os-autoinst-5.1790352490.8968207.obscpio New: ---- os-autoinst-5.1790583840.ff0d715.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ os-autoinst-devel-test.spec ++++++ --- /var/tmp/diff_new_pack.dzb5BD/_old 2026-09-28 13:24:47.827097563 +0200 +++ /var/tmp/diff_new_pack.dzb5BD/_new 2026-09-28 13:24:47.828097605 +0200 @@ -18,7 +18,7 @@ %define short_name os-autoinst-devel Name: %{short_name}-test -Version: 5.1790352490.8968207 +Version: 5.1790583840.ff0d715 Release: 0 Summary: Test package for %{short_name} License: GPL-2.0-or-later ++++++ os-autoinst-openvswitch-test.spec ++++++ --- /var/tmp/diff_new_pack.dzb5BD/_old 2026-09-28 13:24:47.855098737 +0200 +++ /var/tmp/diff_new_pack.dzb5BD/_new 2026-09-28 13:24:47.856098779 +0200 @@ -19,7 +19,7 @@ %define name_ext -test %define short_name os-autoinst-openvswitch Name: %{short_name}%{?name_ext} -Version: 5.1790352490.8968207 +Version: 5.1790583840.ff0d715 Release: 0 Summary: test package for %{short_name} License: GPL-2.0-or-later ++++++ os-autoinst-test.spec ++++++ --- /var/tmp/diff_new_pack.dzb5BD/_old 2026-09-28 13:24:47.877099659 +0200 +++ /var/tmp/diff_new_pack.dzb5BD/_new 2026-09-28 13:24:47.879099743 +0200 @@ -19,7 +19,7 @@ %define name_ext -test %define short_name os-autoinst Name: %{short_name}%{?name_ext} -Version: 5.1790352490.8968207 +Version: 5.1790583840.ff0d715 Release: 0 Summary: test package for os-autoinst License: GPL-2.0-or-later ++++++ os-autoinst.spec ++++++ --- /var/tmp/diff_new_pack.dzb5BD/_old 2026-09-28 13:24:47.910101043 +0200 +++ /var/tmp/diff_new_pack.dzb5BD/_new 2026-09-28 13:24:47.911101085 +0200 @@ -17,7 +17,7 @@ Name: os-autoinst -Version: 5.1790352490.8968207 +Version: 5.1790583840.ff0d715 Release: 0 Summary: OS-level test automation License: GPL-2.0-or-later ++++++ os-autoinst-5.1790352490.8968207.obscpio -> os-autoinst-5.1790583840.ff0d715.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/os-autoinst-5.1790352490.8968207/consoles/VMWare.pm new/os-autoinst-5.1790583840.ff0d715/consoles/VMWare.pm --- old/os-autoinst-5.1790352490.8968207/consoles/VMWare.pm 2026-09-25 18:08:10.000000000 +0200 +++ new/os-autoinst-5.1790583840.ff0d715/consoles/VMWare.pm 2026-09-28 10:24:00.000000000 +0200 @@ -9,6 +9,8 @@ use Mojo::UserAgent; use Mojo::URL; use Mojo::Util qw(xml_escape); +use Carp 'croak'; +use File::Basename; use bmwqemu; use log; @@ -141,4 +143,315 @@ return $self; } + +# Provisioning of images into the ESXi datastore via shell scripts run over the SSH +# connection of a consoles::sshVirtsh console ($svirt) + +# Indents a shell snippet for interpolation and strips its trailing newline +sub _indent ($script, $level = 0) { + chomp $script; + return $script unless $level; + my $indentation = ' ' x $level; + $script =~ s/^(?=.)/$indentation/mg; + return $script; +} + +# Sets $vmid to the VM named exactly $name, so openQA-SUT-1 does not match openQA-SUT-10 +sub vmid_script ($name) { qq{vmid=\$(vim-cmd vmsvc/getallvms | awk '\$2 == "$name" { print \$1 }')} } + +# Path of the temporary copy; contains the VM name so the job's leftover cleanup removes it +sub _tmp_image_path ($dest, $name) { "$dest.$name.part" } + +# File telling waiting jobs why this job's copy failed its verification +sub _failure_note_path ($dest, $name) { "$dest.$name.failed" } + +# File recording the checksum an image was verified against +sub _verified_record_path ($dest) { "$dest.verified" } + +# The expected digest from CHECKSUM_<VAR> for the image named by <VAR>, or undef +sub _expected_checksum ($file_basename) { + for my $checksum_var (sort grep { /^CHECKSUM_/ } keys %bmwqemu::vars) { + my $image = $bmwqemu::vars{$checksum_var =~ s/^CHECKSUM_//r}; + next unless defined $image && basename($image) eq $file_basename; + my $checksum = $bmwqemu::vars{$checksum_var} // next; + # only a plain digest is safe to interpolate into the shell script + return lc $checksum if $checksum =~ /^(?:[0-9a-f]{64}|[0-9a-f]{128})$/i; + bmwqemu::diag "Ignoring $checksum_var, '$checksum' is not a SHA-256 or SHA-512 digest"; + } + return undef; +} + +# Sets $_digest to compare and $_digests to list what was computed in a mismatch message. +# A 64 character checksum is either SHA-256 or SHA-512 truncated, so try SHA-256 first +# and then the other, in the same order as verify_checksum() of the test distribution. +sub _digest_script ($file, $checksum) { + return <<~"EOF" if length($checksum) == 128; + _digest=\$(sha512sum "$file" | awk '{print \$1}') + _digests="SHA-512 \$_digest" + EOF + return <<~"EOF"; + _digest=\$(sha256sum "$file" | awk '{print \$1}') + _digests="SHA-256 \$_digest" + if [ "\$_digest" != "$checksum" ]; then + _digest=\$(sha512sum "$file" | awk '{print \$1}' | cut -c1-64) + _digests="\$_digests, SHA-512 truncated \$_digest" + fi + EOF +} + +# VMFS has no atomic copy, so copy to a temporary file, verify it and only then rename it +# into place (a metadata operation). Other jobs thus never see a partial or corrupt image, +# and no VM holds a VMFS lock on the temporary file that would block the verification. +# Note: This script must be in POSIX shell as ESXi uses busybox for /bin/sh +sub _atomic_copy_script ($source, $dest, $tmp, $checksum = undef, $failure_note = undef) { + my $copy = _replace_notice_script($dest) . <<~"EOF"; + if ! cp "$source" "$tmp"; then + rm -f "$tmp" + echo "Unable to copy $source to $dest" + exit 1 + fi + EOF + # waiting jobs fail with the same reason instead of copying the corrupt source again + my $note = $failure_note ? qq{echo "\$_mismatch" > "$failure_note"} : ':'; + my $verify = !$checksum ? '' : _digest_script($tmp, $checksum) . <<~"EOF"; + if [ "\$_digest" != "$checksum" ]; then + rm -f "$tmp" + _mismatch="Checksum mismatch for $source, expected $checksum but computed \$_digests" + echo "\$_mismatch" + $note + exit 1 + fi + echo "Verified $source against its published checksum" + EOF + my $old_record = _verified_record_path($dest); + my $publish = <<~"EOF" . _record_verified_script($dest, $checksum); + if ! rm -f "$old_record" || ! mv "$tmp" "$dest"; then + rm -f "$tmp" + echo "Unable to publish $dest, it might be held by a running VM" + exit 1 + fi + EOF + return $copy . $verify . $publish . qq{echo "Copied $source to $dest"\n}; +} + +# Like _atomic_copy_script() but decompressing. The image is recorded with the checksum of +# the compressed asset, as that is what later jobs publish for it. +sub _atomic_decompress_script ($source, $dest, $tmp, $checksum = undef) { + my $notice = _replace_notice_script($dest); + my $record = _indent(_record_verified_script($dest, $checksum), 1); + my $old_record = _verified_record_path($dest); + return <<~"EOF"; + $notice + if xz --decompress --stdout "$source" > "$tmp" && rm -f "$old_record" && mv "$tmp" "$dest"; then + $record + echo "Decompressed $source to $dest" + else + rm -f "$tmp" + echo "Unable to decompress $source to $dest" + exit 1 + fi + EOF +} + +# Writes the record; callers remove the old one before the rename so it never vouches +# for new content +sub _record_verified_script ($dest, $checksum = undef) { + my $record = _verified_record_path($dest); + return $checksum ? qq{echo "$checksum" > "$record"\n} : ''; +} + +# Defines _verified(): an existing image counts as present only if its record matches this +# job's checksum. Re-reading the image instead would be slow and fails with "Device or +# resource busy" while another job's VM runs from it. +# Note: This script must be in POSIX shell as ESXi uses busybox for /bin/sh +sub _verified_script ($checksum = undef) { + return qq{_verified() { test -e "\$1"; }\n} unless $checksum; + my $record = _verified_record_path('$1'); + return <<~"EOF"; + _verified() { + test -e "\$1" || return 1 + test "\$(cat "$record" 2>/dev/null)" = "$checksum" + } + EOF +} + +# Announces that an unverified image (older, corrupt or republished) gets replaced by a +# verified copy. Replacing heals the datastore without manual cleanup, the log keeps it visible. +sub _replace_notice_script ($dest) { + return qq{[ ! -e "$dest" ] || echo "Replacing $dest, it is not verified against the published checksum"\n}; +} + +# Marker directory claimed by the job copying an image +sub _copy_marker_path ($dest) { "$dest.copying" } + +# Ensures only one of the jobs arriving together copies an image: the one whose mkdir of +# the marker succeeds sets $_claimed, the others wait for the image to appear. +# - The owner writes a heartbeat, as the temporary file stops growing during verification; +# a marker whose heartbeat stalls for $stall_timeout seconds is taken over. +# - The marker records its owner, so a job that lost it to a takeover does not remove it. +# - If the owner's copy failed verification, waiters fail too instead of copying again. +# Note: This script must be in POSIX shell as ESXi uses busybox for /bin/sh +sub _claim_copy_script ($dest, $owner, $checksum = undef, $interval = 10, $stall_timeout = 300) { + my $marker = _copy_marker_path($dest); + my $own_note = _failure_note_path($dest, $owner); + # the owner waited for is only known at runtime, so the path contains the shell variable + my $waited_note = _failure_note_path($dest, '$_waited_for'); + my $check_waited = !$checksum ? '' : _indent(<<~"EOF", 1); + _owner=\$(cat "$marker/owner" 2>/dev/null) + _owner_left='' + [ -z "\$_waited_for" ] || [ "\$_owner" = "\$_waited_for" ] || _owner_left=1 + if [ -n "\$_owner_left" ] && grep -qF "$checksum" "$waited_note" 2>/dev/null; then + cat "$waited_note" + echo "Not copying $dest again, the copy of \$_waited_for failed its verification" + exit 1 + fi + [ -z "\$_owner" ] || _waited_for=\$_owner + EOF + return <<~"EOF"; + _claimed='' + _heartbeat_pid='' + _owns_marker() { test "\$(cat "$marker/owner" 2>/dev/null)" = "$owner"; } + _heartbeat() { + _beat=0 + while kill -0 \$\$ 2>/dev/null && _owns_marker; do + _beat=\$((_beat + 1)) + echo "\$_beat" > "$marker/heartbeat" + sleep $interval + done + } + _release_claim() { + if [ -n "\$_heartbeat_pid" ]; then + { kill "\$_heartbeat_pid"; wait "\$_heartbeat_pid"; } 2>/dev/null + fi + if [ -n "\$_claimed" ] && _owns_marker; then + rm -rf "$marker" + fi + } + trap _release_claim EXIT + _seen='' + _stalled=0 + _waited_for='' + until _verified "$dest"; do + $check_waited + if mkdir "$marker" 2>/dev/null; then + echo "$owner" > "$marker/owner" + rm -f "$own_note" + _claimed=1 + _heartbeat > /dev/null 2>&1 & + _heartbeat_pid=\$! + break + fi + _marker_state=\$(cat "$marker/owner" "$marker/heartbeat" 2>/dev/null) + if [ "\$_marker_state" != "\$_seen" ]; then + _seen=\$_marker_state + _stalled=0 + else + _stalled=\$((_stalled + $interval)) + fi + if [ "\$_stalled" -ge $stall_timeout ]; then + echo "No heartbeat from the job copying $dest for ${stall_timeout}s, taking the copy over" + rm -rf "$marker" + _stalled=0 + continue + fi + echo "Waiting for another job to copy $dest" + sleep $interval + done + EOF +} + +# Verifies that vmware image is present in the host datastore, otherwise copies from input. +sub provide_image_in_datastore ($svirt, $input_file, $vmware_openqa_datastore, %args) { + my $nfs_dir = ($args{backingfile}) ? 'hdd' : 'iso'; + my $vmware_nfs_datastore = $bmwqemu::vars{VMWARE_NFS_DATASTORE} or die 'Need variable VMWARE_NFS_DATASTORE'; + my $debug = ($bmwqemu::vars{VMWARE_NFS_DATASTORE_DEBUG} // 0) ? 'set -x;' : ''; + my $base_dir = $bmwqemu::vars{VIRSH_OPENQA_BASEDIR} // '/vmfs/volumes'; + my $basefile = basename($input_file); + # expected name of uncompressed image + my $baseimage = basename($input_file) =~ s/\.xz$//r; + my $dest_image = "$vmware_openqa_datastore/${baseimage}"; + # Use the standard folder for an input file without full path + my $file_origin = ($input_file eq $basefile) ? "$base_dir/$vmware_nfs_datastore/$nfs_dir/$basefile" : $input_file; + my $dest_xz = "$dest_image.xz"; + my $name = $svirt->name; + # the checksum belongs to the asset as named in the job, possibly the .xz file + my $checksum = _expected_checksum($basefile); + # check image is present and verified; copy and decompression are atomic. Claiming the + # final image also covers the .xz copy, as only the job producing the image needs it. + # Note: This script must be in POSIX shell as ESXi uses busybox for /bin/sh + my $verified = _indent(_verified_script($checksum)); + my $claim = _indent(_claim_copy_script($dest_image, $name, $checksum)); + my $failure_note = _failure_note_path($dest_image, $name); + my $copy_xz = _indent(_atomic_copy_script($file_origin, $dest_xz, _tmp_image_path($dest_xz, $name), $checksum, $failure_note), 3); + my $decompress = _indent(_atomic_decompress_script($dest_xz, $dest_image, _tmp_image_path($dest_image, $name), $checksum), 2); + my $copy_image = _indent(_atomic_copy_script($file_origin, $dest_image, _tmp_image_path($dest_image, $name), $checksum, $failure_note), 2); + my $cmd = <<~"EOF"; + $debug + input_file="$input_file" + $verified + $claim + if _verified "$dest_image"; then + echo "VMware image $dest_image ready" + else + if [ "\${input_file##*.}" = "xz" ]; then + if ! _verified "$dest_xz"; then + $copy_xz + fi + $decompress + else + $copy_image + fi + fi + echo "Done: origin:" $file_origin* " ; dest.:" $dest_image* + EOF + + my $ret = $svirt->run_cmd($cmd, domain => 'sshVMwareServer'); + croak "Error on VMware image $input_file preparation." if $ret; + return $dest_image; +} + +sub copy_image_to_datastore ($svirt, $name, $backingfile, $file_basename, %args) { + my $vmware_openqa_datastore = $args{vmware_openqa_datastore}; + my $vmware_disk_path = $args{vmware_disk_path}; + my $vmware_disk_path_thinfile = $args{vmware_disk_path_thinfile}; + my $copy_timeout = $args{copy_timeout} // 600; + + # Copy the image from the NFS datastore unless it is already present and verified + my $nfs_dir = $backingfile ? 'hdd' : 'iso'; + my $vmware_nfs_datastore = $bmwqemu::vars{VMWARE_NFS_DATASTORE} or die 'Need variable VMWARE_NFS_DATASTORE'; + # cmd debugging activable by setting VMWARE_NFS_DATASTORE_DEBUG=1 + my $ds_debug = ($bmwqemu::vars{VMWARE_NFS_DATASTORE_DEBUG} // 0) ? 'set -x;' : ''; + my $dest_image = "$vmware_openqa_datastore$file_basename"; + my $file_origin = "/vmfs/volumes/$vmware_nfs_datastore/$nfs_dir/$file_basename"; + my $checksum = _expected_checksum($file_basename); + my $verified = _indent(_verified_script($checksum)); + my $claim = _indent(_claim_copy_script($dest_image, $name, $checksum)); + my $copy_image = _indent(_atomic_copy_script($file_origin, $dest_image, _tmp_image_path($dest_image, $name), $checksum, _failure_note_path($dest_image, $name)), 1); + # Note: This script must be in POSIX shell as ESXi uses busybox for /bin/sh + my $cmd = <<~"EOF"; + $ds_debug + $verified + $claim + if _verified "$dest_image"; then + echo "VMware image $dest_image is already present and verified" + else + $copy_image + fi + EOF + my $retval = $svirt->run_cmd($cmd, domain => 'sshVMwareServer', timeout => $copy_timeout); + die "Can't copy VMware image $file_basename" if $retval; + return unless $backingfile; + # Power VM off, delete its disk image, and create it again. + # Then wait for some time for the VM to *really* turn off. + $cmd = + '( set -x; ' . vmid_script($name) . ';' . + 'if [ $vmid ]; then ' . + 'vim-cmd vmsvc/power.off $vmid;' . + 'fi;' . + "vmkfstools -v1 -U $vmware_disk_path_thinfile;" . + "vmkfstools -v1 -i $vmware_disk_path --diskformat thin $vmware_disk_path_thinfile; sleep 10 ) 2>&1"; + $retval = $svirt->run_cmd($cmd, domain => 'sshVMwareServer'); + die q{Can't create thin VMware image} if $retval; +} + 1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/os-autoinst-5.1790352490.8968207/consoles/sshVirtsh.pm new/os-autoinst-5.1790583840.ff0d715/consoles/sshVirtsh.pm --- old/os-autoinst-5.1790352490.8968207/consoles/sshVirtsh.pm 2026-09-25 18:08:10.000000000 +0200 +++ new/os-autoinst-5.1790583840.ff0d715/consoles/sshVirtsh.pm 2026-09-28 10:24:00.000000000 +0200 @@ -19,6 +19,7 @@ use Time::Seconds; use Carp 'croak'; use backend::svirt; +use consoles::VMWare; has [qw(instance name vmm_family vmm_type vmm_firmware)]; @@ -41,6 +42,13 @@ # initialize SSH console(s) $self->_init_ssh($args); + # Pre-check: Ensure hypervisor is reachable before starting Xvnc/activating console + my $hostname = $self->{ssh_credentials}->{default}->{hostname}; + my $timeout = $bmwqemu::vars{SVIRT_HYPERVISOR_SSH_TIMEOUT} // 1800; + if (!$self->wait_for_ssh_port($hostname, timeout => $timeout)) { + die "backend died: hypervisor host $hostname is not reachable after $timeout seconds\n"; + } + # start Xvnc $self->SUPER::activate; @@ -343,7 +351,7 @@ # Power VM off, delete it's disk image, and create it again. # Poll the power state until the VM is *really* turned off. my $cmd = - "( set -x; vmid=\$(vim-cmd vmsvc/getallvms | awk \'/$name/ { print \$1 }\');" . + '( set -x; ' . consoles::VMWare::vmid_script($name) . ';' . 'if [ $vmid ]; then ' . 'vim-cmd vmsvc/power.off $vmid;' . 'for i in $(seq 1 10); do vim-cmd vmsvc/power.getstate $vmid | grep -q "Powered off" && break; sleep 1; done;' . @@ -362,78 +370,8 @@ return $file; } -# Verifies that vmware image is present in the host datastore, otherwhise copies from input -sub provide_image_vmware_in_ds ($self, $input_file, $vmware_openqa_datastore, %args) { - my $nfs_dir = ($args{backingfile}) ? 'hdd' : 'iso'; - my $vmware_nfs_datastore = $bmwqemu::vars{VMWARE_NFS_DATASTORE} or die 'Need variable VMWARE_NFS_DATASTORE'; - my $debug = ($bmwqemu::vars{VMWARE_NFS_DATASTORE_DEBUG} // 0) ? 'set -x;' : ''; - my $base_dir = $bmwqemu::vars{VIRSH_OPENQA_BASEDIR} // '/vmfs/volumes'; - my $basefile = basename($input_file); - # expected name of uncompressed image - my $baseimage = basename($input_file) =~ s/\.xz$//r; - my $dest_image = "$vmware_openqa_datastore/${baseimage}"; - # Use the standard folder for an input file without full path - my $file_origin = ($input_file eq $basefile) ? "$base_dir/$vmware_nfs_datastore/$nfs_dir/$basefile" : $input_file; - # check image is present - # Note: This script must be in POSIX shell as ESXi uses busybox for /bin/sh - my $cmd = <<~"EOF"; - $debug - input_file="$input_file" - if [ -e "$dest_image" ]; then - echo "Waiting while $input_file is loading:" - while ps -v | grep -F -e "$baseimage" -e "$basefile" | grep -v grep - do sleep 5; done - echo "VMware image $dest_image ready" - elif [ "\${input_file##*.}" = "xz" ]; then - if [ -e "$dest_image.xz" ] || cp "$file_origin" "$vmware_openqa_datastore"; then - xz --decompress --keep "$dest_image.xz" - fi - else - cp "$file_origin" "$vmware_openqa_datastore" - fi - echo "Done: origin:" $file_origin* " ; dest.:" $dest_image* - EOF - - my $ret = $self->run_cmd($cmd, domain => 'sshVMwareServer'); - croak "Error on VMware image $input_file preparation." if $ret; - return $dest_image; -} - -sub _copy_image_vmware ($self, $name, $backingfile, $file_basename, %args) { - my $vmware_openqa_datastore = $args{vmware_openqa_datastore}; - my $vmware_disk_path = $args{vmware_disk_path}; - my $vmware_disk_path_thinfile = $args{vmware_disk_path_thinfile}; - my $copy_timeout = $args{copy_timeout} // 600; - - # If the file exists, make sure someone else is not copying it there right now, - # otherwise copy image from NFS datastore. - my $nfs_dir = $backingfile ? 'hdd' : 'iso'; - my $vmware_nfs_datastore = $bmwqemu::vars{VMWARE_NFS_DATASTORE} or die 'Need variable VMWARE_NFS_DATASTORE'; - # cmd debugging activable by setting VMWARE_NFS_DATASTORE_DEBUG=1 - my $ds_debug = ($bmwqemu::vars{VMWARE_NFS_DATASTORE_DEBUG} // 0) ? 'set -x;' : ''; - my $cmd = - "$ds_debug if test -e $vmware_openqa_datastore$file_basename; then " . - "while lsof | grep 'cp.*$file_basename'; do " . - "echo File $file_basename is being copied by other process, sleeping for 60 seconds; sleep 60;" . - 'done;' . - 'else ' . - "cp /vmfs/volumes/$vmware_nfs_datastore/$nfs_dir/$file_basename $vmware_openqa_datastore;" . - 'fi;'; - my $retval = $self->run_cmd($cmd, domain => 'sshVMwareServer', timeout => $copy_timeout); - die "Can't copy VMware image $file_basename" if $retval; - return unless $backingfile; - # Power VM off, delete it's disk image, and create it again. - # Than wait for some time for the VM to *really* turn off. - $cmd = - "( set -x; vmid=\$(vim-cmd vmsvc/getallvms | awk \'/$name/ { print \$1 }\');" . - 'if [ $vmid ]; then ' . - 'vim-cmd vmsvc/power.off $vmid;' . - 'fi;' . - "vmkfstools -v1 -U $vmware_disk_path_thinfile;" . - "vmkfstools -v1 -i $vmware_disk_path --diskformat thin $vmware_disk_path_thinfile; sleep 10 ) 2>&1"; - $retval = $self->run_cmd($cmd, domain => 'sshVMwareServer'); - die q{Can't create thin VMware image} if $retval; -} +# Provides an image in the VMware datastore, see consoles::VMWare +sub provide_image_vmware_in_ds ($self, @args) { consoles::VMWare::provide_image_in_datastore($self, @args) } sub _copy_nvram_vmware ($self, $name, $vmware_openqa_datastore, $vmware_disk_path) { # If the nvram exists in the source vmx file, then copy the source file as destination nvram. @@ -483,8 +421,8 @@ my $vmware_disk_path_thinfile = $vmware_disk_path =~ s/\.vmdk/_${name}_thinfile\.vmdk/r; if ($cdrom || $backingfile) { if ($self->vmm_family eq 'vmware') { - $self->_copy_image_vmware( - $name, $backingfile, $file_basename, + consoles::VMWare::copy_image_to_datastore( + $self, $name, $backingfile, $file_basename, vmware_openqa_datastore => $vmware_openqa_datastore, vmware_disk_path => $vmware_disk_path, vmware_disk_path_thinfile => $vmware_disk_path_thinfile diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/os-autoinst-5.1790352490.8968207/doc/backends.md new/os-autoinst-5.1790583840.ff0d715/doc/backends.md --- old/os-autoinst-5.1790352490.8968207/doc/backends.md 2026-09-25 18:08:10.000000000 +0200 +++ new/os-autoinst-5.1790583840.ff0d715/doc/backends.md 2026-09-28 10:24:00.000000000 +0200 @@ -165,6 +165,33 @@ production instance for existing VMWare jobs. The variables from the worker config should apply automatically so jobs can be cloned as-is. +#### Image provisioning in the datastore +Before a VM boots, `consoles::VMWare` copies its ISO or HDD image from the NFS +datastore (`VMWARE_NFS_DATASTORE`) into the openQA datastore of the ESXi host. +Jobs of the same build share that copy, so the workflow ensures that no job +ever boots from a partial or corrupt image: + +1. An image already in the datastore is reused only if + `<image>.verified` records the checksum the job publishes for it in + `CHECKSUM_<VAR>` (e.g. `CHECKSUM_ISO`). Without a published checksum the + image is reused as it is. +2. Otherwise the job claims the copy by creating the directory + `<image>.copying`. Only one of the jobs arriving together succeeds, the + others wait for the image to appear. +3. The claiming job copies to `<image>.<VM name>.part`, verifies it against + the checksum and only then renames it into place and writes the record. An + unverified image with the same name is replaced this way. +4. While copying, the job writes a heartbeat into the marker. A marker without + a heartbeat for 5 minutes, e.g. after the job died, is taken over by a + waiting job. +5. On a checksum mismatch nothing is published and the reason is left in + `<image>.<VM name>.failed`, so the waiting jobs fail with the same error + instead of copying the corrupt source again. + +Leftovers of a job (`*.part`, `*.failed`) contain its VM name and are removed +by the cleanup of the test distribution. The shell scripts must stay POSIX +compliant as ESXi uses busybox for `/bin/sh`. + #### Further notes * Within the ESXi web interface you can monitor events and tasks which is useful to keep track of what's going on from VMWare's side. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/os-autoinst-5.1790352490.8968207/t/22-svirt.t new/os-autoinst-5.1790583840.ff0d715/t/22-svirt.t --- old/os-autoinst-5.1790352490.8968207/t/22-svirt.t 2026-09-25 18:08:10.000000000 +0200 +++ new/os-autoinst-5.1790583840.ff0d715/t/22-svirt.t 2026-09-28 10:24:00.000000000 +0200 @@ -35,6 +35,7 @@ my $ssh_xterm_vt_mock = Test::MockModule->new('consoles::sshXtermVt'); $ssh_xterm_vt_mock->noop('activate'); +$ssh_xterm_vt_mock->mock(wait_for_ssh_port => 1); my $distri = $testapi::distri = distribution->new; my $svirt = backend::svirt->new; @@ -52,6 +53,13 @@ password => 'foo', }, 'reading SSH credentials via sshVirtsh console, username defaults to "root"'; +subtest 'activate dies if ssh port unreachable' => sub { + my $mock = Test::MockModule->new('consoles::sshXtermVt'); + $mock->mock(wait_for_ssh_port => 0); + my $ssh_virtsh_unreachable = consoles::sshVirtsh->new(undef, {hostname => 'unreachable-host', password => 'foo'}); + throws_ok { $ssh_virtsh_unreachable->activate } qr/backend died: hypervisor host unreachable-host is not reachable after 1800 seconds/, 'activate dies when hypervisor is unreachable'; +}; + $svirt->do_start_vm; $distri->add_console('root-sut-serial', 'ssh-virtsh-serial', {}); $distri->add_console('user-sut-serial', 'ssh-virtsh-serial', {}); @@ -642,7 +650,11 @@ set_var(VMWARE_NFS_DATASTORE => 'nfs_data_store'); @last_ssh_commands = (); $svirt->add_disk({cdrom => 1, dev_id => $dev_id, file => '/my/path/to/this/file/' . $filename}); - like $last_ssh_commands[0], qr%cp\s+/vmfs/volumes/nfs_data_store/iso/$filename\s+$vmware_openqa_datastore\s*;%, "Copy iso to $vmware_openqa_datastore"; + my $tmp_file = "$vmware_openqa_datastore$filename." . $svirt->name . '.part'; + like $last_ssh_commands[0], qr%cp\s+"/vmfs/volumes/nfs_data_store/iso/$filename"\s+"\Q$tmp_file\E"%, "Copy iso to temporary file in $vmware_openqa_datastore"; + like $last_ssh_commands[0], qr%mv\s+"\Q$tmp_file\E"\s+"$vmware_openqa_datastore$filename"%, 'Temporary file renamed into place so the copy is atomic'; + like $last_ssh_commands[0], qr%mkdir\s+"$vmware_openqa_datastore$filename\.copying"%, 'Right to copy claimed so jobs arriving together do not all transfer the image'; + unlike $last_ssh_commands[0], qr/lsof/, 'No guessing from a process list whether someone else is copying needed anymore'; svirt_xml_validate($svirt, disk_device => 'cdrom', @@ -1117,4 +1129,20 @@ }; }; + +subtest 'VMware VM lookup only finds the VM of the job itself' => sub { + my $bin = tempdir($dir . '/vimcmd_XXXX'); + $bin->child('vim-cmd')->spew(<<~'EOF')->chmod(0755); + #!/bin/sh + cat <<'LIST' + Vmid Name File Guest OS Version Annotation + 10 openQA-SUT-10 [datastore1] openQA/openQA-SUT-10.vmx sles15_64Guest vmx-19 openQA WebUI: host (10): openQA-SUT-1 is not me + 11 openQA-SUT-1 [datastore1] openQA/openQA-SUT-1.vmx sles15_64Guest vmx-19 openQA WebUI: host (1): + 12 openQA-SUT-11 [datastore1] openQA/openQA-SUT-11.vmx sles15_64Guest vmx-19 openQA WebUI: host (11): + LIST + EOF + my $vmid_script = consoles::VMWare::vmid_script('openQA-SUT-1'); + is qx{PATH="$bin:\$PATH"; $vmid_script; echo "\$vmid"}, "11\n", 'only the VM registered under exactly the name is found'; +}; + done_testing; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/os-autoinst-5.1790352490.8968207/t/27-consoles-vmware.t new/os-autoinst-5.1790583840.ff0d715/t/27-consoles-vmware.t --- old/os-autoinst-5.1790352490.8968207/t/27-consoles-vmware.t 2026-09-25 18:08:10.000000000 +0200 +++ new/os-autoinst-5.1790583840.ff0d715/t/27-consoles-vmware.t 2026-09-28 10:24:00.000000000 +0200 @@ -27,6 +27,8 @@ use Mojo::IOLoop::Server; use Mojo::Server::Daemon; use Scalar::Util qw(blessed); +use Mojo::File qw(tempdir path); +use Digest::SHA; use consoles::VMWare; @@ -286,4 +288,171 @@ } }; +subtest 'VMware images are verified against their published checksum before publishing' => sub { + my $my_test_basedir = tempdir('/tmp/27-vmware-checksum-XXXX'); + my $nfs_ds = 'openqa_checksum'; + my $ds = 'datastore_checksum'; + $bmwqemu::vars{VMWARE_NFS_DATASTORE_DEBUG} = '0'; + $bmwqemu::vars{VIRSH_OPENQA_BASEDIR} = $my_test_basedir; + $bmwqemu::vars{VMWARE_NFS_DATASTORE} = $nfs_ds; + my $iso = 'checksum-mock.iso'; + my $source = path($my_test_basedir, $nfs_ds, 'iso')->make_path->child($iso); + $source->spew('VMware image with a published checksum'); + my $vmware_openqa_datastore = path($my_test_basedir, $ds, 'openQA')->make_path; + my $dest = path($vmware_openqa_datastore, $iso); + my $digest = Digest::SHA->new(256)->addfile($source->to_string)->hexdigest; + my $last_output; + # stands in for the sshVirtsh console, running the generated script on the local host + my $svirt = Test::MockObject->new->set_always(name => 'openQA-SUT-1'); + $svirt->mock(run_cmd => sub ($self, $cmd, %args) { $last_output = qx{($cmd) 2>&1}; $? >> 8 }); + $bmwqemu::vars{ISO} = "/var/lib/openqa/share/factory/iso/$iso"; + my $provide = sub { consoles::VMWare::provide_image_in_datastore($svirt, $source, $vmware_openqa_datastore) }; + + subtest 'matching checksum publishes the image' => sub { + $dest->remove; + $bmwqemu::vars{CHECKSUM_ISO} = uc $digest; + lives_ok { $provide->() } 'image provided'; + is $dest->slurp, $source->slurp, 'published image matches the source'; + like $last_output, qr/Verified .*against its published checksum/, 'verification is reported'; + is_deeply [glob "$dest.*.part"], [], 'no temporary file left behind'; + }; + + subtest 'mismatching checksum leaves the destination untouched' => sub { + $dest->remove; + $bmwqemu::vars{CHECKSUM_ISO} = 'b' x 64; + throws_ok { $provide->() } qr/Error on VMware image .* preparation/, 'image preparation fails'; + ok !-e $dest, 'a corrupt image is never published'; + like $last_output, qr/Checksum mismatch .*expected b{64} but computed SHA-256 \Q$digest\E/, 'mismatch names both digests'; + like path(consoles::VMWare::_failure_note_path($dest, 'openQA-SUT-1'))->slurp, qr/Checksum mismatch .*expected b{64}/, 'the mismatch is left for the jobs waiting for this one'; + is_deeply [glob "$dest.*.part"], [], 'temporary file removed'; + }; + + subtest 'image without a published checksum is copied unverified' => sub { + $dest->remove; + delete $bmwqemu::vars{CHECKSUM_ISO}; + lives_ok { $provide->() } 'image provided'; + is $dest->slurp, $source->slurp, 'published image matches the source'; + unlike $last_output, qr/Verified/, 'nothing is verified'; + }; + + subtest 'an image already in the datastore which was never verified is replaced' => sub { + $bmwqemu::vars{CHECKSUM_ISO} = $digest; + $dest->spew('a corrupt image left behind by an earlier job'); + path(consoles::VMWare::_verified_record_path($dest))->remove; + lives_ok { $provide->() } 'image provided'; + like $last_output, qr/Replacing \Q$dest\E, it is not verified/, 'the replacement is announced'; + is $dest->slurp, $source->slurp, 'the unverified image is replaced by a verified copy'; + is path(consoles::VMWare::_verified_record_path($dest))->slurp, "$digest\n", 'the replacement is recorded as verified'; + }; + + subtest 'an image verified against the same checksum is reused' => sub { + $bmwqemu::vars{CHECKSUM_ISO} = $digest; + $dest->spew('published earlier and verified back then'); + path(consoles::VMWare::_verified_record_path($dest))->spew($digest); + lives_ok { $provide->() } 'image provided'; + is $dest->slurp, 'published earlier and verified back then', 'no multi-GB image is transferred twice'; + like $last_output, qr/\Q$dest\E ready/, 'the image is reported as ready'; + }; + + subtest 'an image verified against a different checksum is replaced' => sub { + $bmwqemu::vars{CHECKSUM_ISO} = $digest; + $dest->spew('a different image published under the same name'); + path(consoles::VMWare::_verified_record_path($dest))->spew('c' x 64); + lives_ok { $provide->() } 'image provided'; + is $dest->slurp, $source->slurp, 'a stale image of the same name is replaced'; + }; + + subtest 'a failed replacement leaves the existing image and its record alone' => sub { + $bmwqemu::vars{CHECKSUM_ISO} = 'b' x 64; + $dest->spew('verified against an older checksum'); + path(consoles::VMWare::_verified_record_path($dest))->spew('c' x 64); + throws_ok { $provide->() } qr/Error on VMware image .* preparation/, 'image preparation fails on a mismatch'; + is $dest->slurp, 'verified against an older checksum', 'the existing image is left alone'; + is path(consoles::VMWare::_verified_record_path($dest))->slurp, 'c' x 64, 'its record still describes it'; + is_deeply [glob "$dest.*.part"], [], 'temporary file removed'; + }; + + subtest 'a job without a published checksum keeps using whatever is there' => sub { + delete $bmwqemu::vars{CHECKSUM_ISO}; + $dest->spew('published by an earlier job'); + lives_ok { $provide->() } 'image provided'; + is $dest->slurp, 'published by an earlier job', 'there is nothing to vouch for, so nothing is rejected'; + }; + + subtest 'only a usable checksum of the right image is picked up' => sub { + $bmwqemu::vars{CHECKSUM_ISO} = $digest; + is consoles::VMWare::_expected_checksum($iso), $digest, 'checksum found by image name'; + is consoles::VMWare::_expected_checksum('other.iso'), undef, 'no checksum for an unrelated image'; + $bmwqemu::vars{CHECKSUM_ISO} = 'deadbeef'; + my $checksum; + combined_like { $checksum = consoles::VMWare::_expected_checksum($iso) } qr/Ignoring CHECKSUM_ISO, 'deadbeef' is not a SHA-256/, 'ignoring the value is logged'; + is $checksum, undef, 'a value which is not a digest is ignored'; + }; + delete $bmwqemu::vars{CHECKSUM_ISO}; + delete $bmwqemu::vars{ISO}; +}; + +subtest 'Only one of the VMware jobs arriving together copies the image' => sub { + my $datastore = tempdir('/tmp/27-vmware-claim-XXXX'); + my $dest = path($datastore, 'concurrent-mock.iso')->to_string; + my $marker = consoles::VMWare::_copy_marker_path($dest); + my $verified = consoles::VMWare::_verified_script; + # a one second poll interval keeps the test quick + my $claim = sub ($owner, $stall_timeout = 2) { $verified . consoles::VMWare::_claim_copy_script($dest, $owner, undef, 1, $stall_timeout) }; + + subtest 'a single job copies although four of them start at the same time' => sub { + # the copy outlasts the stall timeout, only the heartbeat keeps it claimed + my $copy = qq{sleep 3; echo published > "$dest"}; + my $jobs = join ' ', map { '( ' . $claim->("job$_") . qq{ test -z "\$_claimed" || { $copy; }; echo "claimed=[\$_claimed]" ) &} } 1 .. 4; + my $output = qx{($jobs wait) 2>&1}; + is scalar(grep { $_ eq 'claimed=[1]' } split /\n/, $output), 1, 'exactly one job transfers the image'; + like $output, qr/Waiting for another job to copy \Q$dest\E/, 'the other jobs wait for it instead'; + unlike $output, qr/taking the copy over/, 'a copy which is still alive is not taken over'; + ok -e $dest, 'the image is published'; + ok !-e $marker, 'the marker is released once the copy finished'; + }; + + subtest 'a marker whose owner died is taken over' => sub { + path($dest)->remove; + path($marker)->make_path->child('owner')->spew("job1\n"); + my $script = $claim->('job2', 1); + my $output = qx{($script echo "claimed=[\$_claimed]") 2>&1}; + like $output, qr/No heartbeat from the job copying \Q$dest\E for 1s, taking the copy over/, 'the abandoned copy is reported'; + like $output, qr/claimed=\[1\]/, 'a job which died in the middle of a copy does not block the others forever'; + ok !-e $marker, 'the marker taken over is released by its new owner'; + }; + + subtest 'a job which lost its marker to a takeover does not release it' => sub { + my $script = $claim->('job1'); + my $output = qx{($script echo job2 > "$marker/owner"; echo "claimed=[\$_claimed]") 2>&1}; + like $output, qr/claimed=\[1\]/, 'the marker was claimed'; + ok -e $marker, 'the marker of the job which took it over is kept'; + path($marker)->remove_tree; + }; + + subtest 'a job does not copy again what failed verification for the job it waited for' => sub { + my $checksum = 'd' x 64; + my $verified_checksum = consoles::VMWare::_verified_script($checksum); + my $note = consoles::VMWare::_failure_note_path($dest, 'job1'); + my $claim_checksum = sub ($owner) { $verified_checksum . consoles::VMWare::_claim_copy_script($dest, $owner, $checksum, 1, 2) }; + my $failing_copy = qq{sleep 1; echo "Checksum mismatch, expected $checksum" > "$note"; exit 1}; + my ($owner, $waiter) = map { $claim_checksum->($_) } qw(job1 job2); + # job2 starts shortly after job1 so it finds the marker and waits for job1 + my $output = qx{(( $owner $failing_copy ) & sleep 0.2; ( $waiter echo "claimed=[\$_claimed]" ); wait) 2>&1}; + like $output, qr/Not copying \Q$dest\E again, the copy of job1 failed its verification/, 'the waiting job gives up'; + like $output, qr/Checksum mismatch, expected $checksum/, 'and names the reason'; + unlike $output, qr/claimed=/, 'without claiming the copy itself'; + ok !-e $marker, 'the marker is released'; + path($note)->remove; + }; + + subtest 'an image which is already present is not claimed at all' => sub { + path($dest)->spew('published'); + my $script = $claim->('job1'); + my $output = qx{($script echo "claimed=[\$_claimed]") 2>&1}; + like $output, qr/claimed=\[\]/, 'nothing is claimed'; + unlike $output, qr/Waiting/, 'and nothing is waited for'; + }; +}; + done_testing; ++++++ os-autoinst.obsinfo ++++++ --- /var/tmp/diff_new_pack.dzb5BD/_old 2026-09-28 13:24:49.420164351 +0200 +++ /var/tmp/diff_new_pack.dzb5BD/_new 2026-09-28 13:24:49.423164476 +0200 @@ -1,5 +1,5 @@ name: os-autoinst -version: 5.1790352490.8968207 -mtime: 1790352490 -commit: 89682075eaef1bb69db68f9a024b539d29bceed6 +version: 5.1790583840.ff0d715 +mtime: 1790583840 +commit: ff0d71527a3bde3a2d7737384cd497a74d21a435
