Am Mi., 7. Okt. 2026 um 15:27 Uhr schrieb Peter Xu <[email protected]>:
>
> On Mon, Sep 21, 2026 at 08:03:45PM +0100, Alex Bennée wrote:
> > This is a hacky patch for comment but I've included the diagnostic
> > hacks to the migration test as well as my workaround to get itworking.
> >
> > This test was failing on my KVM system. On investigation I found the
> > nVME activity meant the migration never completed. I ended up
> > inserting some sleeps into the workload script until I was able to
> > get it to run.
>
> Indeed, I can reproduce it too:
Hi Peter,
thank you checking on your system and sharing a stacktrace.
Unfortunately, I wasn't able to reproduce this failure on my system,
but I think I have a good theory what's going on there. I think it is
because we use `snapshot=on` for a disk device and at the same time
keep important stuff like `/tmp/test_nvme_mig_workload.pid` on a root
filesystem. Easy fix is to use tmpfs for /tmp, so all data is actually
migrated.
Please, can you try the following patch and let me know if this fixes
the issue or not:
--- a/tests/functional/x86_64/test_nvme_migration.py
+++ b/tests/functional/x86_64/test_nvme_migration.py
@@ -75,6 +75,11 @@ def configure_machine(self, vm):
def launch_source_vm(self, vm):
vm.launch()
+ vm.cmd('migrate-set-parameters', **{
+ 'downtime-limit': 2000000,
+ 'max-bandwidth': 10 * 1024**3,
+ })
+
self.wait_for_console_pattern('Entering emergency mode.', vm)
prompt = '# '
self.wait_for_console_pattern(prompt, vm)
@@ -94,6 +99,8 @@ def launch_source_vm(self, vm):
prompt, vm=vm)
exec_command_and_wait_for_pattern(self, 'mount -t sysfs sysfs /sys',
prompt, vm=vm)
+ exec_command_and_wait_for_pattern(self, 'mount -t tmpfs tmpfs /tmp',
+ prompt, vm=vm)
# Run workload before migration to check if it continues
# to run properly after migration.
Kind regards,
Alex
>
> $ ./run ../tests/functional/x86_64/test_nvme_migration.py
> X8664NVMeMigrationTest.test_migration_with_unix
> TAP version 13
> Traceback (most recent call last):
> File "/home/zhexu/git/qemu/tests/functional/x86_64/test_nvme_migration.py",
> line 162, in test_migration_with_unix
> self.migration_with_unix()
> ~~~~~~~~~~~~~~~~~~~~~~~~^^
> File "/home/zhexu/git/qemu/tests/functional/migration.py", line 98, in
> migration_with_unix
> self.migrate(dst_uri)
> ~~~~~~~~~~~~^^^^^^^^^
> File "/home/zhexu/git/qemu/tests/functional/migration.py", line 78, in
> migrate
> self.migrate_vms(dst_uri, src_uri, dst_vm, src_vm)
> ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> File "/home/zhexu/git/qemu/tests/functional/migration.py", line 63, in
> migrate_vms
> self.assert_dest_vm(dst_vm)
> ~~~~~~~~~~~~~~~~~~~^^^^^^^^
> File "/home/zhexu/git/qemu/tests/functional/x86_64/test_nvme_migration.py",
> line 128, in assert_dest_vm
> self.exec_command_and_check(
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~^
> "kill -0 $(cat /tmp/test_nvme_mig_workload.pid)", vm)
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> File "/home/zhexu/git/qemu/tests/functional/x86_64/test_nvme_migration.py",
> line 48, in exec_command_and_check
> wait_for_console_pattern(self, 'OK', failure_message="FAIL", vm=vm)
> ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> File "/home/zhexu/git/qemu/tests/functional/qemu_test/cmd.py", line 191, in
> wait_for_console_pattern
> return _console_interaction(test, success_message, failure_message,
> None, vm=vm)
> File "/home/zhexu/git/qemu/tests/functional/qemu_test/cmd.py", line 134, in
> _console_interaction
> done, line = _console_read_line_until_match(test, vm,
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
> success_message_b,
> ^^^^^^^^^^^^^^^^^^
> failure_message_b)
> ^^^^^^^^^^^^^^^^^^
> File "/home/zhexu/git/qemu/tests/functional/qemu_test/cmd.py", line 70, in
> _console_read_line_until_match
> test.fail(
> ~~~~~~~~~^
> f"'{failure}' found in console, expected '{success}'")
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> AssertionError: 'b'FAIL'' found in console, expected 'b'OK''
>
> not ok 1 test_nvme_migration.X8664NVMeMigrationTest.test_migration_with_unix
> 1..1
>
> Copy the author too (Alexander).
>
> >
> > *NOTE*: obviously this can't be merged as is but I think we could
> > certainly do with more tracking in the test to see when things aren't
> > working with possibly the sleep as another patch.
> >
> > Signed-off-by: Alex Bennée <[email protected]>
> > ---
> > tests/functional/migration.py | 37 +++++++++++++++----
> > .../functional/x86_64/test_nvme_migration.py | 1 +
> > 2 files changed, 31 insertions(+), 7 deletions(-)
> >
> > diff --git a/tests/functional/migration.py b/tests/functional/migration.py
> > index 8d2428efc24..e51a82c22f4 100644
> > --- a/tests/functional/migration.py
> > +++ b/tests/functional/migration.py
> > @@ -22,18 +22,41 @@ class MigrationTest(QemuSystemTest):
> > timeout = 10
> >
> > @staticmethod
> > - def migration_finished(vm):
> > - return vm.cmd('query-migrate')['status'] in ('completed', 'failed')
> > + def migration_finished(query):
> > + return query['status'] in ('completed', 'failed')
> >
> > - def assert_migration(self, src_vm, dst_vm):
> > + def wait_for_vm(self, src_or_dst, vm):
> > + "Track the state of one VM in the migration"
> > + current_state = last_state = vm.cmd('query-migrate')
> >
> > - end = time.monotonic() + self.timeout
> > - while time.monotonic() < end and not
> > self.migration_finished(src_vm):
> > - time.sleep(0.1)
> > + self.log.info(f"waiting for {src_or_dst} to complete")
> >
> > end = time.monotonic() + self.timeout
> > - while time.monotonic() < end and not
> > self.migration_finished(dst_vm):
> > + while time.monotonic() < end:
> > + if self.migration_finished(current_state):
> > + break
> > + else:
> > + last_remain = last_state['remaining']
> > + current_remain = current_state['remaining']
> > +
> > + if current_remain > last_remain:
> > + self.log.warning(f"slow migration {current_remain} >
> > {last_remain}")
> > +
> > time.sleep(0.1)
> > + last_state = current_state
> > + current_state = vm.cmd('query-migrate')
> > +
> > +
> > + def assert_migration(self, src_vm, dst_vm):
> > +
> > + self.wait_for_vm("src", src_vm)
> > +
> > + # if we hit this we never completed...
> > + self.assertEqual(src_vm.cmd('query-migrate')['status'],
> > 'completed')
> > +
> > + self.wait_for_vm("dst", dst_vm)
> > +
> > + self.log.info("checking everything is OK")
> >
> > self.assertEqual(src_vm.cmd('query-migrate')['status'],
> > 'completed')
> > self.assertEqual(dst_vm.cmd('query-migrate')['status'],
> > 'completed')
> > diff --git a/tests/functional/x86_64/test_nvme_migration.py
> > b/tests/functional/x86_64/test_nvme_migration.py
> > index 890f0aab6d6..58596f63cd3 100755
> > --- a/tests/functional/x86_64/test_nvme_migration.py
> > +++ b/tests/functional/x86_64/test_nvme_migration.py
> > @@ -109,6 +109,7 @@ def launch_source_vm(self, vm):
> > rm -f /tmp/test_nvme_mig_workload.iter_finished; \
> > echo 3 > /proc/sys/vm/drop_caches; \
> > find /usr/bin -type f -exec cksum {} \\;; \
> > + sleep 0.3; \
> > touch /tmp/test_nvme_mig_workload.iter_finished; \
> > done) > /dev/null 2> /tmp/test_nvme_mig_workload.errors &",
> > prompt, vm=vm)
> > --
> > 2.47.3
> >
>
> --
> Peter Xu
>
