On Mon, Sep 21, 2026 at 08:03:45PM +0100, Alex Bennée wrote:
> This is a hacky patch for comment but I've included the diagnostic
> hacks to the migration test as well as my workaround to get itworking.
>
> This test was failing on my KVM system. On investigation I found the
> nVME activity meant the migration never completed. I ended up
> inserting some sleeps into the workload script until I was able to
> get it to run.
Indeed, I can reproduce it too:
$ ./run ../tests/functional/x86_64/test_nvme_migration.py
X8664NVMeMigrationTest.test_migration_with_unix
TAP version 13
Traceback (most recent call last):
File "/home/zhexu/git/qemu/tests/functional/x86_64/test_nvme_migration.py",
line 162, in test_migration_with_unix
self.migration_with_unix()
~~~~~~~~~~~~~~~~~~~~~~~~^^
File "/home/zhexu/git/qemu/tests/functional/migration.py", line 98, in
migration_with_unix
self.migrate(dst_uri)
~~~~~~~~~~~~^^^^^^^^^
File "/home/zhexu/git/qemu/tests/functional/migration.py", line 78, in migrate
self.migrate_vms(dst_uri, src_uri, dst_vm, src_vm)
~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/zhexu/git/qemu/tests/functional/migration.py", line 63, in
migrate_vms
self.assert_dest_vm(dst_vm)
~~~~~~~~~~~~~~~~~~~^^^^^^^^
File "/home/zhexu/git/qemu/tests/functional/x86_64/test_nvme_migration.py",
line 128, in assert_dest_vm
self.exec_command_and_check(
~~~~~~~~~~~~~~~~~~~~~~~~~~~^
"kill -0 $(cat /tmp/test_nvme_mig_workload.pid)", vm)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/zhexu/git/qemu/tests/functional/x86_64/test_nvme_migration.py",
line 48, in exec_command_and_check
wait_for_console_pattern(self, 'OK', failure_message="FAIL", vm=vm)
~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/zhexu/git/qemu/tests/functional/qemu_test/cmd.py", line 191, in
wait_for_console_pattern
return _console_interaction(test, success_message, failure_message,
None, vm=vm)
File "/home/zhexu/git/qemu/tests/functional/qemu_test/cmd.py", line 134, in
_console_interaction
done, line = _console_read_line_until_match(test, vm,
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
success_message_b,
^^^^^^^^^^^^^^^^^^
failure_message_b)
^^^^^^^^^^^^^^^^^^
File "/home/zhexu/git/qemu/tests/functional/qemu_test/cmd.py", line 70, in
_console_read_line_until_match
test.fail(
~~~~~~~~~^
f"'{failure}' found in console, expected '{success}'")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: 'b'FAIL'' found in console, expected 'b'OK''
not ok 1 test_nvme_migration.X8664NVMeMigrationTest.test_migration_with_unix
1..1
Copy the author too (Alexander).
>
> *NOTE*: obviously this can't be merged as is but I think we could
> certainly do with more tracking in the test to see when things aren't
> working with possibly the sleep as another patch.
>
> Signed-off-by: Alex Bennée <[email protected]>
> ---
> tests/functional/migration.py | 37 +++++++++++++++----
> .../functional/x86_64/test_nvme_migration.py | 1 +
> 2 files changed, 31 insertions(+), 7 deletions(-)
>
> diff --git a/tests/functional/migration.py b/tests/functional/migration.py
> index 8d2428efc24..e51a82c22f4 100644
> --- a/tests/functional/migration.py
> +++ b/tests/functional/migration.py
> @@ -22,18 +22,41 @@ class MigrationTest(QemuSystemTest):
> timeout = 10
>
> @staticmethod
> - def migration_finished(vm):
> - return vm.cmd('query-migrate')['status'] in ('completed', 'failed')
> + def migration_finished(query):
> + return query['status'] in ('completed', 'failed')
>
> - def assert_migration(self, src_vm, dst_vm):
> + def wait_for_vm(self, src_or_dst, vm):
> + "Track the state of one VM in the migration"
> + current_state = last_state = vm.cmd('query-migrate')
>
> - end = time.monotonic() + self.timeout
> - while time.monotonic() < end and not self.migration_finished(src_vm):
> - time.sleep(0.1)
> + self.log.info(f"waiting for {src_or_dst} to complete")
>
> end = time.monotonic() + self.timeout
> - while time.monotonic() < end and not self.migration_finished(dst_vm):
> + while time.monotonic() < end:
> + if self.migration_finished(current_state):
> + break
> + else:
> + last_remain = last_state['remaining']
> + current_remain = current_state['remaining']
> +
> + if current_remain > last_remain:
> + self.log.warning(f"slow migration {current_remain} >
> {last_remain}")
> +
> time.sleep(0.1)
> + last_state = current_state
> + current_state = vm.cmd('query-migrate')
> +
> +
> + def assert_migration(self, src_vm, dst_vm):
> +
> + self.wait_for_vm("src", src_vm)
> +
> + # if we hit this we never completed...
> + self.assertEqual(src_vm.cmd('query-migrate')['status'], 'completed')
> +
> + self.wait_for_vm("dst", dst_vm)
> +
> + self.log.info("checking everything is OK")
>
> self.assertEqual(src_vm.cmd('query-migrate')['status'], 'completed')
> self.assertEqual(dst_vm.cmd('query-migrate')['status'], 'completed')
> diff --git a/tests/functional/x86_64/test_nvme_migration.py
> b/tests/functional/x86_64/test_nvme_migration.py
> index 890f0aab6d6..58596f63cd3 100755
> --- a/tests/functional/x86_64/test_nvme_migration.py
> +++ b/tests/functional/x86_64/test_nvme_migration.py
> @@ -109,6 +109,7 @@ def launch_source_vm(self, vm):
> rm -f /tmp/test_nvme_mig_workload.iter_finished; \
> echo 3 > /proc/sys/vm/drop_caches; \
> find /usr/bin -type f -exec cksum {} \\;; \
> + sleep 0.3; \
> touch /tmp/test_nvme_mig_workload.iter_finished; \
> done) > /dev/null 2> /tmp/test_nvme_mig_workload.errors &",
> prompt, vm=vm)
> --
> 2.47.3
>
--
Peter Xu
