Re: [PATCH v1 00/12] libxl: bump LIBXL_API_VERSION

[Jim Fehlig]

On 3/17/21 4:57 AM, Olaf Hering wrote:

Various changes to handle libxl API variants.


Thanks for taking this on! It is much simpler than the crap I've cooked up in 
the past. I haven't taken another stab since we've moved to meson, and I'd like 
to blame autotools, but...


We'll need to sort out the naming I mentioned in patch 1, which will affect the 
others. I also pointed out a few small problems in patches 8 and 11, but 
otherwise looking good!


Regards,
Jim



Olaf

Olaf Hering (12):
   libxl: add API wrapper for libxl_domain_create_restore
   libxl: add API wrapper for libxl_retrieve_domain_configuration
   libxl: add API wrapper for libxl_domain_shutdown
   libxl: add API wrapper for libxl_domain_reboot
   libxl: add API wrapper for libxl_domain_pause
   libxl: add API wrapper for libxl_domain_unpause
   libxl: add API wrapper for libxl_domain_need_memory
   libxl: add API wrapper for libxl_get_free_memory
   libxl: add API wrapper for libxl_set_vcpuonline
   libxl: add API wrapper for libxl_send_trigger
   libxl: add API wrapper for libxl_set_memory_target
   libxl: use API 4.13 to support domUs with more than 4TB

  meson.build |   7 +-
  src/libxl/libxl_api.h   | 219 
  src/libxl/libxl_conf.c  |   5 +-
  src/libxl/libxl_domain.c|  23 ++--
  src/libxl/libxl_driver.c|  21 ++--
  src/libxl/libxl_migration.c |   3 +-
  tests/libxlmock.c   |   7 +-
  7 files changed, 259 insertions(+), 26 deletions(-)
  create mode 100644 src/libxl/libxl_api.h

Re: [PATCH v1 11/12] libxl: add API wrapper for libxl_set_memory_target

[Jim Fehlig]

On 3/17/21 4:57 AM, Olaf Hering wrote:

Upcoming changes will use different LIBXL_API_VERSION variants.

Prepare libxl_set_memory_target, which changed the storage size of
parameter "target_memkb" in Xen 4.8.

No functional change intended.

Signed-off-by: Olaf Hering 
---
  src/libxl/libxl_api.h| 29 +
  src/libxl/libxl_domain.c |  4 ++--
  src/libxl/libxl_driver.c |  2 +-
  3 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/src/libxl/libxl_api.h b/src/libxl/libxl_api.h
index c7174f5133..1507dbd38e 100644
--- a/src/libxl/libxl_api.h
+++ b/src/libxl/libxl_api.h
@@ -20,6 +20,7 @@
  
  #pragma once
  
+#include 

  #include 
  
  static inline int

@@ -188,3 +189,31 @@ Libxl_Send_Trigger(libxl_ctx *ctx,
  
  return ret;

  }
+
+static inline int
+Libxl_Set_Memory_Target(libxl_ctx *ctx,
+uint32_t domid,
+uint64_t target_memkb,
+int relative,
+int enforce)
+{
+int ret = -1;
+
+/* Technically this guard could be LIBXL_HAVE_MEMKB_64BITS */
+#if LIBXL_API_VERSION < 0x040800
+if (target_memkb < UINT_MAX)
+{


Here


+uint32_t val32 = target_memkb;
+
+ret = libxl_set_memory_target(ctx, domid, val32, relative, enforce);
+}
+#else
+if (target_memkb < LLONG_MAX)
+{


and here fail syntax-check

make: Entering directory '/home/jfehlig/virt/gitlab/libvirt/build/build-aux'
curly_braces_style
/home/jfehlig/virt/gitlab/libvirt/src/libxl/libxl_api.h-205-{
/home/jfehlig/virt/gitlab/libvirt/src/libxl/libxl_api.h-212-{

Regards,
Jim


+int64_t val64 = target_memkb;
+ret = libxl_set_memory_target(ctx, domid, val64, relative, enforce);
+}
+#endif
+
+return ret;
+}
diff --git a/src/libxl/libxl_domain.c b/src/libxl/libxl_domain.c
index 264a730c6c..0b0c3865fa 100644
--- a/src/libxl/libxl_domain.c
+++ b/src/libxl/libxl_domain.c
@@ -1010,7 +1010,7 @@ libxlDomainFreeMem(libxl_ctx *ctx, libxl_domain_config 
*d_config)
  {
  uint64_t needed_mem;
  uint64_t free_mem;
-int32_t target_mem;
+uint64_t target_mem;
  int tries = 3;
  int wait_secs = 10;
  
@@ -1025,7 +1025,7 @@ libxlDomainFreeMem(libxl_ctx *ctx, libxl_domain_config *d_config)

  return 0;
  
  target_mem = free_mem - needed_mem;

-if (libxl_set_memory_target(ctx, 0, target_mem,
+if (Libxl_Set_Memory_Target(ctx, 0, target_mem,
  /* relative */ 1, 0) < 0)
  goto error;
  
diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c

index cf3a2d9775..97aed9949d 100644
--- a/src/libxl/libxl_driver.c
+++ b/src/libxl/libxl_driver.c
@@ -1695,7 +1695,7 @@ libxlDomainSetMemoryFlags(virDomainPtr dom, unsigned long 
newmem,
  
  /* Unlock virDomainObj while ballooning memory */

  virObjectUnlock(vm);
-res = libxl_set_memory_target(cfg->ctx, vm->def->id, newmem, 0,
+res = Libxl_Set_Memory_Target(cfg->ctx, vm->def->id, newmem, 0,
/* force */ 1);
  virObjectLock(vm);
  if (res < 0) {

Re: [PATCH v1 08/12] libxl: add API wrapper for libxl_get_free_memory

[Jim Fehlig]

On 3/17/21 4:57 AM, Olaf Hering wrote:

Upcoming changes will use different LIBXL_API_VERSION variants.

Prepare libxl_get_free_memory, which changed storage size of parameter
"memkb" in Xen 4.8.

No functional change intended.

Signed-off-by: Olaf Hering 
---
  src/libxl/libxl_api.h| 19 +++
  src/libxl/libxl_conf.c   |  5 +++--
  src/libxl/libxl_domain.c |  4 ++--
  tests/libxlmock.c|  7 ++-
  4 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/src/libxl/libxl_api.h b/src/libxl/libxl_api.h
index 5c1545c3c0..1c13781ae6 100644
--- a/src/libxl/libxl_api.h
+++ b/src/libxl/libxl_api.h
@@ -138,3 +138,22 @@ Libxl_Domain_Need_Memory(libxl_ctx *ctx,
  
  return ret;

  }
+
+static inline int
+Libxl_Get_Free_Memory(libxl_ctx *ctx, uint64_t *memkb)
+{
+int ret;
+
+#if LIBXL_API_VERSION < 0x040800
+{
+uint32_t val32 = 0;
+
+ret = libxl_get_free_memory(ctx, );
+*memkb = val32;
+}
+#else
+ret = libxl_get_free_memory(ctx, memkb);
+#endif
+
+return ret;
+}
diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
index 4b6a7e6096..74551ff804 100644
--- a/src/libxl/libxl_conf.c
+++ b/src/libxl/libxl_conf.c
@@ -36,6 +36,7 @@
  #include "viruuid.h"
  #include "vircommand.h"
  #include "virsocketaddr.h"
+#include "libxl_api.h"
  #include "libxl_domain.h"
  #include "libxl_conf.h"
  #include "libxl_utils.h"
@@ -1785,7 +1786,7 @@ libxlDriverConfigNew(void)
  int
  libxlDriverConfigInit(libxlDriverConfigPtr cfg)
  {
-unsigned int free_mem;
+uint64_t free_mem;
  
  if (g_mkdir_with_parents(cfg->logDir, 0777) < 0) {

  virReportError(VIR_ERR_INTERNAL_ERROR,
@@ -1817,7 +1818,7 @@ libxlDriverConfigInit(libxlDriverConfigPtr cfg)
  
  /* This will fill xenstore info about free and dom0 memory if missing,

   * should be called before starting first domain */
-if (libxl_get_free_memory(cfg->ctx, _mem)) {
+if (Libxl_Get_Free_Memory(cfg->ctx, _mem)) {
  VIR_ERROR(_("Unable to configure libxl's memory management 
parameters"));
  return -1;
  }
diff --git a/src/libxl/libxl_domain.c b/src/libxl/libxl_domain.c
index 337764b5c7..264a730c6c 100644
--- a/src/libxl/libxl_domain.c
+++ b/src/libxl/libxl_domain.c
@@ -1009,7 +1009,7 @@ static int
  libxlDomainFreeMem(libxl_ctx *ctx, libxl_domain_config *d_config)
  {
  uint64_t needed_mem;
-uint32_t free_mem;
+uint64_t free_mem;
  int32_t target_mem;
  int tries = 3;
  int wait_secs = 10;
@@ -1018,7 +1018,7 @@ libxlDomainFreeMem(libxl_ctx *ctx, libxl_domain_config 
*d_config)
  goto error;
  
  do {

-if (libxl_get_free_memory(ctx, _mem) < 0)
+if (Libxl_Get_Free_Memory(ctx, _mem) < 0)
  goto error;
  
  if (free_mem >= needed_mem)

diff --git a/tests/libxlmock.c b/tests/libxlmock.c
index a36ca135f6..7a43f9196b 100644
--- a/tests/libxlmock.c
+++ b/tests/libxlmock.c
@@ -67,7 +67,12 @@ VIR_MOCK_IMPL_RET_ARGS(libxl_get_version_info,
  VIR_MOCK_STUB_RET_ARGS(libxl_get_free_memory,
 int, 0,
 libxl_ctx *, ctx,
-   uint32_t *, memkb);
+#if LIBXL_API_VERSION < 0x040800
+   uint32_t *,
+#else
+   uint64_t *,
+#endif
+   memkb);


Fails syntax-check

cppi: /home/jfehlig/virt/gitlab/libvirt/tests/libxlmock.c: line 70: not properly 
indented
cppi: /home/jfehlig/virt/gitlab/libvirt/tests/libxlmock.c: line 72: not properly 
indented
cppi: /home/jfehlig/virt/gitlab/libvirt/tests/libxlmock.c: line 74: not properly 
indented

build-aux/syntax-check.mk: incorrect preprocessor indentation

You can install cppi in your build env to catch those.

Regards,
Jim

Re: [PATCH v1 01/12] libxl: add API wrapper for libxl_domain_create_restore

[Jim Fehlig]

On 3/17/21 4:57 AM, Olaf Hering wrote:

Upcoming changes will use different LIBXL_API_VERSION variants.

Prepare libxl_domain_create_restore, which got a new parameter
"send_back_fd" in Xen 4.7. libvirt does not use this parameter.

No functional change intended.

Signed-off-by: Olaf Hering 
---
  src/libxl/libxl_api.h| 44 
  src/libxl/libxl_domain.c |  5 +++--
  2 files changed, 47 insertions(+), 2 deletions(-)
  create mode 100644 src/libxl/libxl_api.h

diff --git a/src/libxl/libxl_api.h b/src/libxl/libxl_api.h
new file mode 100644
index 00..64ccd7a428
--- /dev/null
+++ b/src/libxl/libxl_api.h


What do you think of a more descriptive name for this file? E.g. 
libxl_api_quirks.h or libxl_api_wrap.h?



@@ -0,0 +1,44 @@
+/*
+ * libxl_api.h: handle various libxl API variants


Or maybe libxl_api_variants.h?


+ *
+ * Copyright (C) 2021 SUSE LLC
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * .
+ */
+
+#pragma once
+
+#include 
+
+static inline int
+Libxl_Domain_Create_Restore(libxl_ctx *ctx,


libvirt doesn't use this format for function names. See 'Function names' under 
naming conventions


https://libvirt.org/coding-style.html#naming-conventions

I struggled with naming when toying with this in the past. Maybe 
libxlDomainCreateRestoreWrap? The 'Wrap' suffix compliments the libxl_api_wrap.h 
name suggestion.


Regards,
Jim


+libxl_domain_config *d_config,
+uint32_t *domid,
+int restore_fd,
+const libxl_domain_restore_params *params,
+const libxl_asyncprogress_how *aop_console_how)
+{
+int ret;
+
+#if LIBXL_API_VERSION < 0x040700
+ret = libxl_domain_create_restore(ctx, d_config, domid, restore_fd, params,
+  NULL, aop_console_how);
+#else
+ret = libxl_domain_create_restore(ctx, d_config, domid, restore_fd, -1,
+  params, NULL, aop_console_how);
+#endif
+
+return ret;
+}
diff --git a/src/libxl/libxl_domain.c b/src/libxl/libxl_domain.c
index 014f6aceca..2fe1d34e19 100644
--- a/src/libxl/libxl_domain.c
+++ b/src/libxl/libxl_domain.c
@@ -22,6 +22,7 @@
  
  #include 
  
+#include "libxl_api.h"

  #include "libxl_domain.h"
  #include "libxl_capabilities.h"
  
@@ -1396,8 +1397,8 @@ libxlDomainStart(libxlDriverPrivatePtr driver,

  #ifdef LIBXL_HAVE_SRM_V2
  params.stream_version = restore_ver;
  #endif
-ret = libxl_domain_create_restore(cfg->ctx, _config, ,
-  restore_fd, , NULL,
+ret = Libxl_Domain_Create_Restore(cfg->ctx, _config, ,
+  restore_fd, ,
_console_how);
  libxl_domain_restore_params_dispose();
  }

Re: [PATCH v1 01/12] libxl: add API wrapper for libxl_domain_create_restore

[Olaf Hering]

Am Thu, 18 Mar 2021 16:26:14 -0600
schrieb Jim Fehlig :

> Maybe libxlDomainCreateRestoreWrap?
> The 'Wrap' suffix compliments the libxl_api_wrap.h name suggestion.

"Naming conventions" does not cover API wrapping.

Some of the names are already taken, like 
libxl_domain_shutdown/libxlDomainShutdown.
This is the reason why I just used uppercase for the first letter of the libxl 
function.

I assume virLibxlDomainShutdown may work, because in the end the wrappers are a 
libvirt thing.

Olaf


pgphjIrBcDi7i.pgp
Description: Digitale Signatur von OpenPGP

Re: [PULL v2 00/13] misc patches removing deprecated features

[Peter Maydell]

On Thu, 18 Mar 2021 at 09:30, Daniel P. Berrangé  wrote:
>
> The following changes since commit 571d413b5da6bc6f1c2aaca8484717642255ddb0:
>
>   Merge remote-tracking branch 'remotes/mcayland/tags/qemu-openbios-20210316' 
> into staging (2021-03-17 21:02:37 +)
>
> are available in the Git repository at:
>
>   https://gitlab.com/berrange/qemu tags/dep-many-pull-request
>
> for you to fetch changes up to 8d17adf34f501ded65a106572740760f0a75577c:
>
>   block: remove support for using "file" driver with block/char devices 
> (2021-03-18 09:22:55 +)
>
> 
> Remove many old deprecated features
>
> The following features have been deprecated for well over the 2
> release cycle we promise
>
>   ``-drive file=json:{...{'driver':'file'}}`` (since 3.0)
>   ``-vnc acl`` (since 4.0.0)
>   ``-mon ...,control=readline,pretty=on|off`` (since 4.1)
>   ``migrate_set_downtime`` and ``migrate_set_speed`` (since 2.8.0)
>   ``query-named-block-nodes`` result ``encryption_key_missing`` (since 2.10.0)
>   ``query-block`` result ``inserted.encryption_key_missing`` (since 2.10.0)
>   ``migrate-set-cache-size`` and ``query-migrate-cache-size`` (since 2.11.0)
>   ``query-named-block-nodes`` and ``query-block`` result 
> dirty-bitmaps[i].status (since 4.0)
>   ``query-cpus`` (since 2.12.0)
>   ``query-cpus-fast`` ``arch`` output member (since 3.0.0)
>   ``query-events`` (since 4.0)
>   chardev client socket with ``wait`` option (since 4.0)
>   ``acl_show``, ``acl_reset``, ``acl_policy``, ``acl_add``, ``acl_remove`` 
> (since 4.0.0)
>   ``ide-drive`` (since 4.2)
>   ``scsi-disk`` (since 4.2)
>
> 


Applied, thanks.

Please update the changelog at https://wiki.qemu.org/ChangeLog/6.0
for any user-visible changes.

-- PMM

[libvirt PATCH 2/2] build: teach run script how to temporarily stop systemd units

[Daniel P . Berrangé]

When testing locally built daemons on a systemd host there can be quite
a few systemd units that need temporarily stopping, and ideally
restarting after the test is complete. This becomes a massive burden
when modular daemons are running and you want to test libvirtd, as a
huge number of units need stopping.

The run script can facilitate this usage by looking at what units are
running and automatically stopping any that are known to conflict with
the daemon that is about to be run.

Signed-off-by: Daniel P. Berrangé 
---
 run.in | 86 --
 1 file changed, 84 insertions(+), 2 deletions(-)

diff --git a/run.in b/run.in
index 99c67c586a..72d299dc4a 100644
--- a/run.in
+++ b/run.in
@@ -44,6 +44,7 @@ import os
 import os.path
 import random
 import sys
+import subprocess
 
 # Function to intelligently prepend a path to an environment variable.
 # See https://stackoverflow.com/a/9631350
@@ -77,5 +78,86 @@ env["LIBVIRT_DIR_OVERRIDE"] = "1"
 # read problems when using glibc.
 env["MALLOC_PERTURB_"] = "%d" % random.randint(1, 255)
 
-# Run the program.
-os.execve(prog, args, env)
+modular_daemons = [
+"virtinterfaced",
+"virtlxcd",
+"virtnetworkd",
+"virtnodedevd",
+"virtnwfilterd",
+"virtproxyd",
+"virtqemud",
+"virtsecretd",
+"virtstoraged",
+"virtvboxd",
+"virtvzd",
+"virtxend",
+]
+
+def is_modular_daemon(name):
+return name in modular_daemons
+
+def is_monolithic_daemon(name):
+return name == "libvirtd"
+
+def is_systemd_host():
+if os.getuid() != 0:
+return False
+return os.path.exists("/run/systemd/system")
+
+def daemon_units(name):
+return [name + suffix for suffix in [
+".service", ".socket", "-ro.socket", "-admin.socket"]]
+
+def is_unit_active(name):
+ret = subprocess.call(["systemctl", "is-active", "-q", name])
+return ret == 0
+
+def change_unit(name, action):
+ret = subprocess.call(["systemctl", action, "-q", name])
+return ret == 0
+
+try_stop_units = []
+if is_systemd_host():
+name = os.path.basename(prog)
+
+maybe_stopped_units = []
+if is_modular_daemon(name):
+# Only need to stop libvirtd or this specific modular unit
+maybe_stopped_units += daemon_units("libvirtd")
+maybe_stopped_units += daemon_units(name)
+elif is_monolithic_daemon(name):
+# Need to stop libvirtd and/or all modular units
+maybe_stopped_units += daemon_units("libvirtd")
+for entry in modular_daemons:
+maybe_stopped_units += daemon_units(entry)
+
+for unit in maybe_stopped_units:
+if is_unit_active(unit):
+try_stop_units.append(unit)
+
+if len(try_stop_units) == 0:
+# Run the program directly, replacing ourselves
+os.execve(prog, args, env)
+else:
+print("Temporarily stopping systemd units...")
+stopped_units = []
+
+try:
+for unit in try_stop_units:
+print(" > %s" % unit)
+if not change_unit(unit, "stop"):
+raise Exception("Unable to stop '%s'" % unit)
+
+stopped_units.append(unit)
+
+print("Running %s..." % prog)
+ret = subprocess.call(prog, env=env)
+except KeyboardInterrupt as ex:
+pass
+finally:
+print("Re-starting original systemd units...")
+stopped_units.reverse()
+for unit in stopped_units:
+print(" > %s" % unit)
+if not change_unit(unit, "start"):
+print(" ! unable to restart %s" % unit, file=sys.stderr)
-- 
2.30.2

[libvirt PATCH 0/2] build: make the run script more intelligent on systemd hosts

[Daniel P . Berrangé]

Running virtqemud, we just stop the virqemud daemon units

Temporarily stopping systemd units...
 > virtqemud.socket
 > virtqemud-ro.socket
 > virtqemud-admin.socket
Running ./src/virtqemud...
^CRe-starting original systemd units...
 > virtqemud-admin.socket
 > virtqemud-ro.socket
 > virtqemud.socket

Running libvirtd, we stop all modular demon units

Temporarily stopping systemd units...
 > virtnetworkd.socket
 > virtproxyd.socket
 > virtqemud.socket
 > virtqemud-ro.socket
 > virtqemud-admin.socket
Running ./src/libvirtd...
^CRe-starting original systemd units...
 > virtqemud-admin.socket
 > virtqemud-ro.socket
 > virtqemud.socket
 > virtproxyd.socket
 > virtnetworkd.socket

Daniel P. Berrangé (2):
  build: convert the run script to use Python
  build: teach run script how to temporarily stop systemd units

 run.in | 125 -
 1 file changed, 105 insertions(+), 20 deletions(-)

-- 
2.30.2

[libvirt PATCH 1/2] build: convert the run script to use Python

[Daniel P . Berrangé]

This fits with the goal of eliminating non-Python scripting languages,
and makes forthcoming changes far easier.

Signed-off-by: Daniel P. Berrangé 
---
 run.in | 45 ++---
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/run.in b/run.in
index 6ddf5fc58f..99c67c586a 100644
--- a/run.in
+++ b/run.in
@@ -1,6 +1,6 @@
-#!/bin/sh
+#!/usr/bin/env python3
 # libvirt 'run' programs locally script
-# Copyright (C) 2012-2013 Red Hat, Inc.
+# Copyright (C) 2012-2021 Red Hat, Inc.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -40,35 +40,42 @@
 #
 #--
 
+import os
+import os.path
+import random
+import sys
+
 # Function to intelligently prepend a path to an environment variable.
 # See https://stackoverflow.com/a/9631350
-prepend()
-{
-eval $1="$2\${$1:+:\$$1}"
-}
+def prepend(env, varname, extradir):
+if varname in os.environ:
+env[varname] = extradir + ":" + env[varname]
+else:
+env[varname] = extradir
+
+here = "@abs_builddir@"
 
-# Find this script.
-b=@abs_builddir@
+if len(sys.argv) < 2:
+print("syntax: %s BINARY [ARGS...]" % sys.argv[0], file=sys.stderr)
+sys.exit(1)
 
-prepend LD_LIBRARY_PATH "$b/src"
-export LD_LIBRARY_PATH
+prog = sys.argv[1]
+args = sys.argv[1:]
+env = os.environ
 
-prepend PKG_CONFIG_PATH "$b/src"
-export PKG_CONFIG_PATH
 
-prepend PATH "$b/tools"
-export PATH
+prepend(env, "LD_LIBRARY_PATH", os.path.join(here, "src"))
+prepend(env, "PKG_CONFIG_PATH", os.path.join(here, "src"))
+prepend(env, "PATH", os.path.join(here, "tools"))
 
 # Ensure that any 3rd party apps using libvirt.so from the build tree get
 # files resolved to the build/source tree too. Typically useful for language
 # bindings running tests against non-installed libvirt.
-LIBVIRT_DIR_OVERRIDE=1
-export LIBVIRT_DIR_OVERRIDE
+env["LIBVIRT_DIR_OVERRIDE"] = "1"
 
 # This is a cheap way to find some use-after-free and uninitialized
 # read problems when using glibc.
-random_val="$(awk 'BEGIN{srand(); print 1+int(255*rand())}' < /dev/null)"
-export MALLOC_PERTURB_=$random_val
+env["MALLOC_PERTURB_"] = "%d" % random.randint(1, 255)
 
 # Run the program.
-exec "$@"
+os.execve(prog, args, env)
-- 
2.30.2

[PATCH 3/3] qemu: Update paused reason on unexpected 'STOP' event

[Peter Krempa]

The qemu STOP event doesn't really report why the VM was stopped. In
certain cases we do expect this by storing the expectation in the
private data.

In cases we've encountered an unexpected STOP event we can offload to
the event thread a job to refresh the state using 'query-status'.

For all other paused reasons which are expected we keep the original
logic.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_domain.c  |  1 +
 src/qemu/qemu_domain.h  |  1 +
 src/qemu/qemu_driver.c  | 37 +
 src/qemu/qemu_process.c | 22 +++---
 4 files changed, 58 insertions(+), 3 deletions(-)

diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index dde9ba92c6..106a292ef3 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -11049,6 +11049,7 @@ qemuProcessEventFree(struct qemuProcessEvent *event)
 virObjectUnref(event->data);
 break;
 case QEMU_PROCESS_EVENT_PR_DISCONNECT:
+case QEMU_PROCESS_EVENT_STOP:
 case QEMU_PROCESS_EVENT_LAST:
 break;
 }
diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
index 949307229b..b057e94df0 100644
--- a/src/qemu/qemu_domain.h
+++ b/src/qemu/qemu_domain.h
@@ -441,6 +441,7 @@ typedef enum {
 QEMU_PROCESS_EVENT_PR_DISCONNECT,
 QEMU_PROCESS_EVENT_RDMA_GID_STATUS_CHANGED,
 QEMU_PROCESS_EVENT_GUEST_CRASHLOADED,
+QEMU_PROCESS_EVENT_STOP,

 QEMU_PROCESS_EVENT_LAST
 } qemuProcessEventType;
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index f3f8caab44..53be363338 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -4316,6 +4316,40 @@ processGuestCrashloadedEvent(virQEMUDriverPtr driver,
 }


+static void
+processGuestStopEvent(virDomainObjPtr vm)
+{
+qemuDomainObjPrivatePtr priv = vm->privateData;
+virObjectEventPtr event = NULL;
+virDomainPausedReason reason = VIR_DOMAIN_PAUSED_UNKNOWN;
+
+if (qemuDomainObjBeginJob(priv->driver, vm, QEMU_JOB_MODIFY) == 0) {
+bool running;
+int rc;
+
+qemuDomainObjEnterMonitor(priv->driver, vm);
+
+rc = qemuMonitorGetStatus(priv->mon, , );
+
+/* update reason only on successful state update */
+if (qemuDomainObjExitMonitor(priv->driver, vm) < 0)
+rc = -1;
+
+qemuDomainObjEndJob(priv->driver, vm);
+
+if (rc < 0)
+return;
+}
+
+virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, reason);
+
+event = virDomainEventLifecycleNewFromObj(vm, VIR_DOMAIN_EVENT_SUSPENDED,
+  
qemuDomainPausedReasonToSuspendedEvent(reason));
+
+virObjectEventStateQueue(priv->driver->domainEventState, event);
+}
+
+
 static void qemuProcessEventHandler(void *data, void *opaque)
 {
 struct qemuProcessEvent *processEvent = data;
@@ -4365,6 +4399,9 @@ static void qemuProcessEventHandler(void *data, void 
*opaque)
 case QEMU_PROCESS_EVENT_GUEST_CRASHLOADED:
 processGuestCrashloadedEvent(driver, vm);
 break;
+case QEMU_PROCESS_EVENT_STOP:
+processGuestStopEvent(vm);
+break;
 case QEMU_PROCESS_EVENT_LAST:
 break;
 }
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 5f31260221..7003c92046 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -689,10 +689,26 @@ qemuProcessHandleStop(qemuMonitorPtr mon G_GNUC_UNUSED,
 if (priv->signalStop)
 virDomainObjBroadcast(vm);

+/* In case of VIR_DOMAIN_PAUSED_UNKNOWN we'll update it later, but we
+ * must set the state to PAUSED right away. We delay the event until
+ * the update. */
 virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, reason);
-event = virDomainEventLifecycleNewFromObj(vm,
-  VIR_DOMAIN_EVENT_SUSPENDED,
-  detail);
+if (reason == VIR_DOMAIN_PAUSED_UNKNOWN) {
+/* offload unknown state to thread updating state */
+struct qemuProcessEvent *processEvent = g_new0(struct 
qemuProcessEvent, 1);
+
+processEvent->eventType = QEMU_PROCESS_EVENT_STOP;
+processEvent->vm = virObjectRef(vm);
+
+if (virThreadPoolSendJob(driver->workerPool, 0, processEvent) < 0) 
{
+virObjectUnref(vm);
+qemuProcessEventFree(processEvent);
+}
+} else {
+event = virDomainEventLifecycleNewFromObj(vm,
+  
VIR_DOMAIN_EVENT_SUSPENDED,
+  detail);
+}

 VIR_FREE(priv->lockState);
 if (virDomainLockProcessPause(driver->lockManager, vm, 
>lockState) < 0)
-- 
2.29.2

[PATCH 2/3] API: Add VIR_DOMAIN_PAUSED_HYPERVISOR_FAULT enum value

[Peter Krempa]

In certain cases hypervisors on encountering an internal fault pause the
domain to allow gathering useful debugging information. Libvirt for now
would report 'VIR_DOMAIN_PAUSED_UNKNOWN' which isn't entirely helpful.

Add a new paused reason and the corresponding value for the suspended
event, so that hypervisor drivers can report this state properly.

In qemu this will be mapped to the 'internal-error' paused reason.

Signed-off-by: Peter Krempa 
---
 examples/c/misc/event-test.c | 3 +++
 include/libvirt/libvirt-domain.h | 2 ++
 src/conf/domain_conf.c   | 1 +
 src/qemu/qemu_domain.c   | 3 +++
 src/qemu/qemu_monitor.c  | 4 +++-
 tools/virsh-domain-monitor.c | 1 +
 tools/virsh-domain.c | 3 ++-
 7 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/examples/c/misc/event-test.c b/examples/c/misc/event-test.c
index 76d4f3f6e8..c7d60d4c61 100644
--- a/examples/c/misc/event-test.c
+++ b/examples/c/misc/event-test.c
@@ -187,6 +187,9 @@ eventDetailToString(int event,
 case VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED:
 return "Post-copy Error";

+case VIR_DOMAIN_EVENT_SUSPENDED_HYPERVISOR_FAULT:
+return "Hypervisor fault";
+
 case VIR_DOMAIN_EVENT_SUSPENDED_LAST:
 break;
 }
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h
index 03c119fe26..f1c0e960f9 100644
--- a/include/libvirt/libvirt-domain.h
+++ b/include/libvirt/libvirt-domain.h
@@ -119,6 +119,7 @@ typedef enum {
 VIR_DOMAIN_PAUSED_STARTING_UP = 11, /* the domain is being started */
 VIR_DOMAIN_PAUSED_POSTCOPY = 12,/* paused for post-copy migration */
 VIR_DOMAIN_PAUSED_POSTCOPY_FAILED = 13, /* paused after failed post-copy */
+VIR_DOMAIN_PAUSED_HYPERVISOR_FAULT = 14, /* paused after a hypervisor 
failure */

 # ifdef VIR_ENUM_SENTINELS
 VIR_DOMAIN_PAUSED_LAST
@@ -3108,6 +3109,7 @@ typedef enum {
 VIR_DOMAIN_EVENT_SUSPENDED_API_ERROR = 6, /* suspended after failure 
during libvirt API call */
 VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY = 7, /* suspended for post-copy 
migration */
 VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY_FAILED = 8, /* suspended after failed 
post-copy */
+VIR_DOMAIN_EVENT_SUSPENDED_HYPERVISOR_FAULT = 9, /* suspended after a 
hypervisor fault */

 # ifdef VIR_ENUM_SENTINELS
 VIR_DOMAIN_EVENT_SUSPENDED_LAST
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 47756ff0be..f49db930ca 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -1082,6 +1082,7 @@ VIR_ENUM_IMPL(virDomainPausedReason,
   "starting up",
   "post-copy",
   "post-copy failed",
+  "hypervisor fault",
 );

 VIR_ENUM_IMPL(virDomainShutdownReason,
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index ed2a1481d4..dde9ba92c6 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -11180,6 +11180,9 @@ 
qemuDomainPausedReasonToSuspendedEvent(virDomainPausedReason reason)
 case VIR_DOMAIN_PAUSED_POSTCOPY:
 return VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY;

+case VIR_DOMAIN_PAUSED_HYPERVISOR_FAULT:
+return VIR_DOMAIN_EVENT_SUSPENDED_HYPERVISOR_FAULT;
+
 case VIR_DOMAIN_PAUSED_UNKNOWN:
 case VIR_DOMAIN_PAUSED_USER:
 case VIR_DOMAIN_PAUSED_SAVE:
diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
index c43c6f180e..ca6bd79516 100644
--- a/src/qemu/qemu_monitor.c
+++ b/src/qemu/qemu_monitor.c
@@ -3508,10 +3508,12 @@ qemuMonitorVMStatusToPausedReason(const char *status)

 switch ((qemuMonitorVMStatus) st) {
 case QEMU_MONITOR_VM_STATUS_DEBUG:
-case QEMU_MONITOR_VM_STATUS_INTERNAL_ERROR:
 case QEMU_MONITOR_VM_STATUS_RESTORE_VM:
 return VIR_DOMAIN_PAUSED_UNKNOWN;

+case QEMU_MONITOR_VM_STATUS_INTERNAL_ERROR:
+return VIR_DOMAIN_PAUSED_HYPERVISOR_FAULT;
+
 case QEMU_MONITOR_VM_STATUS_INMIGRATE:
 case QEMU_MONITOR_VM_STATUS_POSTMIGRATE:
 case QEMU_MONITOR_VM_STATUS_FINISH_MIGRATE:
diff --git a/tools/virsh-domain-monitor.c b/tools/virsh-domain-monitor.c
index c4d7464695..169fd5931b 100644
--- a/tools/virsh-domain-monitor.c
+++ b/tools/virsh-domain-monitor.c
@@ -195,6 +195,7 @@ VIR_ENUM_IMPL(virshDomainPausedReason,
   N_("starting up"),
   N_("post-copy"),
   N_("post-copy failed"),
+  N_("hypervisor fault"),
 );

 VIR_ENUM_DECL(virshDomainShutdownReason);
diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c
index 7db88f700a..05b75edcda 100644
--- a/tools/virsh-domain.c
+++ b/tools/virsh-domain.c
@@ -12837,7 +12837,8 @@ VIR_ENUM_IMPL(virshDomainEventSuspended,
   N_("Snapshot"),
   N_("API error"),
   N_("Post-copy"),
-  N_("Post-copy Error"));
+  N_("Post-copy Error"),
+  N_("Hypervisor fault"));

 VIR_ENUM_DECL(virshDomainEventResumed);
 VIR_ENUM_IMPL(virshDomainEventResumed,

[PATCH 1/3] qemuMonitorJSONGetStatus: Refactor cleanup

[Peter Krempa]

Use g_autofree for the JSON values to remove cleanup label and ret
variable.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_monitor_json.c | 18 ++
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c
index b669630bc8..e62adcb5b5 100644
--- a/src/qemu/qemu_monitor_json.c
+++ b/src/qemu/qemu_monitor_json.c
@@ -1729,10 +1729,9 @@ qemuMonitorJSONGetStatus(qemuMonitorPtr mon,
  bool *running,
  virDomainPausedReason *reason)
 {
-int ret = -1;
 const char *status;
-virJSONValuePtr cmd;
-virJSONValuePtr reply = NULL;
+g_autoptr(virJSONValue) cmd = NULL;
+g_autoptr(virJSONValue) reply = NULL;
 virJSONValuePtr data;

 if (reason)
@@ -1742,17 +1741,17 @@ qemuMonitorJSONGetStatus(qemuMonitorPtr mon,
 return -1;

 if (qemuMonitorJSONCommand(mon, cmd, ) < 0)
-goto cleanup;
+return -1;

 if (qemuMonitorJSONCheckReply(cmd, reply, VIR_JSON_TYPE_OBJECT) < 0)
-goto cleanup;
+return -1;

 data = virJSONValueObjectGetObject(reply, "return");

 if (virJSONValueObjectGetBoolean(data, "running", running) < 0) {
 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("query-status reply was missing running state"));
-goto cleanup;
+return -1;
 }

 if ((status = virJSONValueObjectGetString(data, "status"))) {
@@ -1762,12 +1761,7 @@ qemuMonitorJSONGetStatus(qemuMonitorPtr mon,
 VIR_DEBUG("query-status reply was missing status details");
 }

-ret = 0;
-
- cleanup:
-virJSONValueFree(cmd);
-virJSONValueFree(reply);
-return ret;
+return 0;
 }


-- 
2.29.2

[PATCH 0/3] Add event for hypervisor faults and implement it for qemu

[Peter Krempa]

Peter Krempa (3):
  qemuMonitorJSONGetStatus: Refactor cleanup
  API: Add VIR_DOMAIN_PAUSED_HYPERVISOR_FAULT enum value
  qemu: Update paused reason on unexpected 'STOP' event

 examples/c/misc/event-test.c |  3 +++
 include/libvirt/libvirt-domain.h |  2 ++
 src/conf/domain_conf.c   |  1 +
 src/qemu/qemu_domain.c   |  4 
 src/qemu/qemu_domain.h   |  1 +
 src/qemu/qemu_driver.c   | 37 
 src/qemu/qemu_monitor.c  |  4 +++-
 src/qemu/qemu_monitor_json.c | 18 ++--
 src/qemu/qemu_process.c  | 22 ---
 tools/virsh-domain-monitor.c |  1 +
 tools/virsh-domain.c |  3 ++-
 11 files changed, 79 insertions(+), 17 deletions(-)

-- 
2.29.2

Re: [libvirt PATCH 8/9] conf: introduce support for firmware auto-selection feature filtering

[Pavel Hrdina]

On Thu, Mar 18, 2021 at 06:10:45PM +0100, Pavel Hrdina wrote:
> On Thu, Mar 18, 2021 at 05:18:36PM +0100, Michal Privoznik wrote:
> > On 3/18/21 1:26 PM, Pavel Hrdina wrote:
> > > When the firmware auto-selection was introduced it always picked first
> > > usable firmware based on the JSON descriptions on the host. It is
> > > possible to add/remove/change the JSON files but it will always be for
> > > the whole host.
> > > 
> > > This patch introduces support for configuring the auto-selection per VM
> > > by adding users an option to limit what features they would like to have
> > > available in the firmware.
> > > 
> > > Signed-off-by: Pavel Hrdina 
> > > ---
> > >   docs/formatdomain.rst | 31 +++
> > >   docs/schemas/domaincommon.rng | 23 +
> > >   src/conf/domain_conf.c| 83 ++-
> > >   src/conf/domain_conf.h| 10 +++
> > >   .../os-firmware-efi-invalid-type.xml  | 28 +++
> > >   ...os-firmware-invalid-type.x86_64-latest.err |  1 +
> > >   .../os-firmware-invalid-type.xml  | 28 +++
> > >   tests/qemuxml2argvtest.c  |  1 +
> > >   ...aarch64-os-firmware-efi.aarch64-latest.xml |  1 +
> > >   .../os-firmware-bios.x86_64-latest.xml|  1 +
> > >   .../os-firmware-efi-secboot.x86_64-latest.xml |  1 +
> > >   .../os-firmware-efi.x86_64-latest.xml |  1 +
> > >   tests/vmx2xmldata/vmx2xml-firmware-efi.xml|  1 +
> > >   13 files changed, 207 insertions(+), 3 deletions(-)
> > 
> > >   create mode 100644 
> > > tests/qemuxml2argvdata/os-firmware-efi-invalid-type.xml
> > >   create mode 100644 tests/qemuxml2argvdata/os-firmware-invalid-type.xml
> > 
> > These two are identical. Have you intended them to be different?
> 
> Nice catch, the first one is leftover after rename, I'll drop it.
> 
> > >   create mode 100644 
> > > tests/qemuxml2argvdata/os-firmware-invalid-type.x86_64-latest.err
> > 
> > > 
> > > diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
> > > index c101d5a1f1..dd063b0794 100644
> > > --- a/docs/formatdomain.rst
> > > +++ b/docs/formatdomain.rst
> > > @@ -155,6 +155,37 @@ harddisk, cdrom, network) determining where to 
> > > obtain/find the boot image.
> > >  the host native arch will be chosen. For the ``test``, ``ESX`` and 
> > > ``VMWare``
> > >  hypervisor drivers, however, the ``i686`` arch will always be chosen 
> > > even on
> > >  an ``x86_64`` host. :since:`Since 0.0.1`
> > > +``firmware``
> > > +   :since:`Since 7.2.0 QEMU/KVM only`
> > > +
> > > +   When used together with ``firmware`` attribute of ``os`` element the 
> > > ``type``
> > > +   attribute must have the same value.
> > > +
> > > +   List of mandatory attributes:
> > > +
> > > +   - ``type`` (accepted values are ``bios`` and ``efi``) same as the 
> > > ``firmware``
> > > + attribute of ``os`` element.
> > > +
> > > +   When using firmware auto-selection there are different features 
> > > enabled in
> > > +   the firmwares. The list of features can be used to limit what 
> > > firmware should
> > > +   be automatically selected for the VM. The list of features can be 
> > > specified
> > > +   using zero or more ``feature`` elements. Libvirt will take into 
> > > consideration
> > > +   only the listed features and ignore the rest when selecting the 
> > > firmware.
> > > +
> > > +   ``feature``
> > > +  The list of mandatory attributes:
> > > +
> > > +  - ``enabled`` (accepted values are ``yes`` and ``no``) is used to 
> > > tell libvirt
> > > +if the feature must be enabled or not in the automatically 
> > > selected firmware
> > > +
> > > +  - ``name`` the name of the feature, the list of the features:
> > > +
> > > +- ``enrolled-keys`` whether the selected nvram template has 
> > > default
> > > +  certificate enrolled. Firmware with Secure Boot feature but 
> > > without
> > > +  enrolled keys will successfully boot non-signed binaries as 
> > > well.
> > > +  Valid only for firmwares with Secure Boot feature.
> > > +
> > > +- ``secure-boot`` whether the firmware implements UEFI Secure 
> > > boot feature.
> > >   ``loader``
> > >  The optional ``loader`` tag refers to a firmware blob, which is 
> > > specified by
> > >  absolute path, used to assist the domain creation process. It is 
> > > used by Xen
> > > diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng
> > > index e6db2f5b74..1dbfc68f18 100644
> > > --- a/docs/schemas/domaincommon.rng
> > > +++ b/docs/schemas/domaincommon.rng
> > > @@ -276,6 +276,29 @@
> > > 
> > >   
> > >   
> > > +
> > > +  
> > > +
> > > +  
> > > +bios
> > > +efi
> > > +  
> > > +
> > > +
> > > +  
> > > +
> > > +  
> > > +   

Re: [libvirt PATCH 1/9] docs: improve description of secure attribute for loader element

[Pavel Hrdina]

On Thu, Mar 18, 2021 at 05:42:47PM +0100, Kashyap Chamarthy wrote:
> On Thu, Mar 18, 2021 at 01:26:38PM +0100, Pavel Hrdina wrote:
> > The original text was not explaining what this attribute actually
> > controls and could have been interpreted as a control switch for the
> > Secure boot feature in firmwares.
> 
> Yep, I've indeed seen people misread it as such.
> 
> > Signed-off-by: Pavel Hrdina 
> > ---
> >  docs/formatdomain.rst | 4 +++-
> >  1 file changed, 3 insertions(+), 1 deletion(-)
> > 
> > diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
> > index a2ea2690a5..c101d5a1f1 100644
> > --- a/docs/formatdomain.rst
> > +++ b/docs/formatdomain.rst
> > @@ -167,7 +167,9 @@ harddisk, cdrom, network) determining where to 
> > obtain/find the boot image.
> > in the guest memory the file should be mapped. For instance, if the 
> > loader
> > path points to an UEFI image, ``type`` should be ``pflash``. Moreover, 
> > some
> > firmwares may implement the Secure boot feature. Attribute ``secure`` 
> > can be
> > -   used then to control it. :since:`Since 2.1.0`
> > +   used to tell the hypervisor that the firmware implements Secure Boot 
> > Feature.
> 
> s/Feature/feature/
> 
> Perhaps: "firmware is capable of Secure Boot feature"

Sounds reasonable, will change it.

> > +   It cannot be used to enable or disable the feature itself in the 
> > firmware.
> > +   :since:`Since 2.1.0`
> 
> This additional clarification is good.
> 
> (Nit-pick: not this patch's fault: consistently use "Secure Boot"; I see
> both "Secure boot" and "Secure Boot".)

If you check our documentation we lack consistency almost everywhere :)
until this is enforced by some check it will happen all the time.

> Address the above only if you're respinning.  FWIW:
> 
> Reviewed-by: Kashyap Chamarthy 

Thanks, I need to do some other changes to the series before pushing so
I'll apply this suggestions as well before pushing.

Pavel


signature.asc
Description: PGP signature

Re: [libvirt PATCH 9/9] qemu: implement support for firmware auto-selection feature filtering

[Pavel Hrdina]

On Thu, Mar 18, 2021 at 05:18:38PM +0100, Michal Privoznik wrote:
> On 3/18/21 1:26 PM, Pavel Hrdina wrote:
> > Signed-off-by: Pavel Hrdina 
> > ---
> >   src/qemu/qemu_firmware.c  | 40 +++
> >   ...re-efi-no-enrolled-keys.x86_64-latest.args | 49 ++
> >   .../os-firmware-efi-no-enrolled-keys.xml  | 25 ++
> >   tests/qemuxml2argvtest.c  |  1 +
> >   ...are-efi-no-enrolled-keys.x86_64-latest.xml | 50 +++
> >   tests/qemuxml2xmltest.c   |  1 +
> >   6 files changed, 166 insertions(+)
> >   create mode 100644 
> > tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args
> >   create mode 100644 
> > tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.xml
> >   create mode 100644 
> > tests/qemuxml2xmloutdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.xml
> > 
> > diff --git a/src/qemu/qemu_firmware.c b/src/qemu/qemu_firmware.c
> > index d3198e2d45..f6f371f51f 100644
> > --- a/src/qemu/qemu_firmware.c
> > +++ b/src/qemu/qemu_firmware.c
> > @@ -930,6 +930,10 @@ qemuFirmwareMatchDomain(const virDomainDef *def,
> >   bool supportsS4 = false;
> >   bool requiresSMM = false;
> >   bool supportsSEV = false;
> > +bool supportsSecureBoot = false;
> > +bool hasEnrolledKeys = false;
> > +int reqSecureBoot;
> > +int reqEnrolledKeys;
> >   want = qemuFirmwareOSInterfaceTypeFromOsDefFirmware(def->os.firmware);
> > @@ -979,7 +983,13 @@ qemuFirmwareMatchDomain(const virDomainDef *def,
> >   break;
> >   case QEMU_FIRMWARE_FEATURE_SECURE_BOOT:
> > +supportsSecureBoot = true;
> > +break;
> > +
> >   case QEMU_FIRMWARE_FEATURE_ENROLLED_KEYS:
> > +hasEnrolledKeys = true;
> > +break;
> > +
> >   case QEMU_FIRMWARE_FEATURE_VERBOSE_DYNAMIC:
> >   case QEMU_FIRMWARE_FEATURE_VERBOSE_STATIC:
> >   case QEMU_FIRMWARE_FEATURE_NONE:
> > @@ -1000,6 +1010,36 @@ qemuFirmwareMatchDomain(const virDomainDef *def,
> >   return false;
> >   }
> > +if (def->os.firmwareFeatures) {
> > +reqSecureBoot = 
> > def->os.firmwareFeatures[VIR_DOMAIN_OS_DEF_FIRMWARE_FEATURE_SECURE_BOOT];
> > +if (reqSecureBoot != VIR_TRISTATE_BOOL_ABSENT) {
> > +if (reqSecureBoot == VIR_TRISTATE_BOOL_YES && 
> > !supportsSecureBoot) {
> > +VIR_DEBUG("User requested Secure Boot, firmware '%s' 
> > doesn't support it",
> > +  path);
> > +return false;
> > +}
> > +
> > +if (reqSecureBoot == VIR_TRISTATE_BOOL_NO && 
> > supportsSecureBoot) {
> > +VIR_DEBUG("User refused Secure Boot, firmware '%s' 
> > supports it", path);
> > +return false;
> > +}
> > +}
> > +
> > +reqEnrolledKeys = 
> > def->os.firmwareFeatures[VIR_DOMAIN_OS_DEF_FIRMWARE_FEATURE_ENROLLED_KEYS];
> > +if (reqEnrolledKeys != VIR_TRISTATE_BOOL_ABSENT) {
> > +if (reqEnrolledKeys == VIR_TRISTATE_BOOL_YES && 
> > !hasEnrolledKeys) {
> > +VIR_DEBUG("User requested Enrolled keys, firmware '%s' 
> > doesn't support it",
> 
> "doesn't have them" perhaps?
> 
> > +  path);
> > +return false;
> > +}
> > +
> > +if (reqEnrolledKeys == VIR_TRISTATE_BOOL_NO && 
> > hasEnrolledKeys) {
> > +VIR_DEBUG("User refused Enrolled keys, firmware '%s' 
> > supports it", path);
> 
> "has them" perhaps?

Sounds better, I wanted to change it after copy of the secureBoot
part, but as we can see it did not happen. :)

> > +return false;
> > +}
> > +}
> > +}
> > +
> >   if (def->os.loader &&
> >   def->os.loader->secure == VIR_TRISTATE_BOOL_YES &&
> >   !requiresSMM) {
> > diff --git 
> > a/tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args
> >  
> > b/tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args
> > new file mode 100644
> > index 00..561a905e78
> > --- /dev/null
> > +++ 
> > b/tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args
> > @@ -0,0 +1,49 @@
> > +LC_ALL=C \
> > +PATH=/bin \
> > +HOME=/tmp/lib/domain--1-fedora \
> > +USER=test \
> > +LOGNAME=test \
> > +XDG_DATA_HOME=/tmp/lib/domain--1-fedora/.local/share \
> > +XDG_CACHE_HOME=/tmp/lib/domain--1-fedora/.cache \
> > +XDG_CONFIG_HOME=/tmp/lib/domain--1-fedora/.config \
> > +/usr/bin/qemu-system-x86_64 \
> > +-name guest=fedora,debug-threads=on \
> > +-S \
> > +-object secret,id=masterKey0,format=raw,\
> > +file=/tmp/lib/domain--1-fedora/master-key.aes \
> > +-blockdev '{"driver":"file","filename":"/usr/share/OVMF/OVMF_CODE.fd",\
> > +"node-name":"libvirt-pflash0-storage","auto-read-only":true,\
> > +"discard":"unmap"}' \
> > +-blockdev 

Re: [libvirt PATCH 8/9] conf: introduce support for firmware auto-selection feature filtering

[Pavel Hrdina]

On Thu, Mar 18, 2021 at 05:18:36PM +0100, Michal Privoznik wrote:
> On 3/18/21 1:26 PM, Pavel Hrdina wrote:
> > When the firmware auto-selection was introduced it always picked first
> > usable firmware based on the JSON descriptions on the host. It is
> > possible to add/remove/change the JSON files but it will always be for
> > the whole host.
> > 
> > This patch introduces support for configuring the auto-selection per VM
> > by adding users an option to limit what features they would like to have
> > available in the firmware.
> > 
> > Signed-off-by: Pavel Hrdina 
> > ---
> >   docs/formatdomain.rst | 31 +++
> >   docs/schemas/domaincommon.rng | 23 +
> >   src/conf/domain_conf.c| 83 ++-
> >   src/conf/domain_conf.h| 10 +++
> >   .../os-firmware-efi-invalid-type.xml  | 28 +++
> >   ...os-firmware-invalid-type.x86_64-latest.err |  1 +
> >   .../os-firmware-invalid-type.xml  | 28 +++
> >   tests/qemuxml2argvtest.c  |  1 +
> >   ...aarch64-os-firmware-efi.aarch64-latest.xml |  1 +
> >   .../os-firmware-bios.x86_64-latest.xml|  1 +
> >   .../os-firmware-efi-secboot.x86_64-latest.xml |  1 +
> >   .../os-firmware-efi.x86_64-latest.xml |  1 +
> >   tests/vmx2xmldata/vmx2xml-firmware-efi.xml|  1 +
> >   13 files changed, 207 insertions(+), 3 deletions(-)
> 
> >   create mode 100644 tests/qemuxml2argvdata/os-firmware-efi-invalid-type.xml
> >   create mode 100644 tests/qemuxml2argvdata/os-firmware-invalid-type.xml
> 
> These two are identical. Have you intended them to be different?

Nice catch, the first one is leftover after rename, I'll drop it.

> >   create mode 100644 
> > tests/qemuxml2argvdata/os-firmware-invalid-type.x86_64-latest.err
> 
> > 
> > diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
> > index c101d5a1f1..dd063b0794 100644
> > --- a/docs/formatdomain.rst
> > +++ b/docs/formatdomain.rst
> > @@ -155,6 +155,37 @@ harddisk, cdrom, network) determining where to 
> > obtain/find the boot image.
> >  the host native arch will be chosen. For the ``test``, ``ESX`` and 
> > ``VMWare``
> >  hypervisor drivers, however, the ``i686`` arch will always be chosen 
> > even on
> >  an ``x86_64`` host. :since:`Since 0.0.1`
> > +``firmware``
> > +   :since:`Since 7.2.0 QEMU/KVM only`
> > +
> > +   When used together with ``firmware`` attribute of ``os`` element the 
> > ``type``
> > +   attribute must have the same value.
> > +
> > +   List of mandatory attributes:
> > +
> > +   - ``type`` (accepted values are ``bios`` and ``efi``) same as the 
> > ``firmware``
> > + attribute of ``os`` element.
> > +
> > +   When using firmware auto-selection there are different features enabled 
> > in
> > +   the firmwares. The list of features can be used to limit what firmware 
> > should
> > +   be automatically selected for the VM. The list of features can be 
> > specified
> > +   using zero or more ``feature`` elements. Libvirt will take into 
> > consideration
> > +   only the listed features and ignore the rest when selecting the 
> > firmware.
> > +
> > +   ``feature``
> > +  The list of mandatory attributes:
> > +
> > +  - ``enabled`` (accepted values are ``yes`` and ``no``) is used to 
> > tell libvirt
> > +if the feature must be enabled or not in the automatically 
> > selected firmware
> > +
> > +  - ``name`` the name of the feature, the list of the features:
> > +
> > +- ``enrolled-keys`` whether the selected nvram template has default
> > +  certificate enrolled. Firmware with Secure Boot feature but 
> > without
> > +  enrolled keys will successfully boot non-signed binaries as well.
> > +  Valid only for firmwares with Secure Boot feature.
> > +
> > +- ``secure-boot`` whether the firmware implements UEFI Secure boot 
> > feature.
> >   ``loader``
> >  The optional ``loader`` tag refers to a firmware blob, which is 
> > specified by
> >  absolute path, used to assist the domain creation process. It is used 
> > by Xen
> > diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng
> > index e6db2f5b74..1dbfc68f18 100644
> > --- a/docs/schemas/domaincommon.rng
> > +++ b/docs/schemas/domaincommon.rng
> > @@ -276,6 +276,29 @@
> > 
> >   
> >   
> > +
> > +  
> > +
> > +  
> > +bios
> > +efi
> > +  
> > +
> > +
> > +  
> > +
> > +  
> > +
> > +
> > +  
> > +enrolled-keys
> > +secure-boot
> > +  
> > +
> > +  
> > +
> > +  
> > +
> >   
> > 
> >   
> > diff --git 

[PATCH] qemu: backup: Restore security label on backup disk store image on VM termination

[Peter Krempa]

When the backup job is terminated normally the security label is
restored by the blockjob finishing handler.

If the VM dies or is destroyed that wouldn't happen as the blockjob
handler wouldn't be called.

Restore the security label on disk store where we remember that the job
was running at the point when 'qemuBackupJobTerminate' was called.

Not resetting the security label means that we also leak the xattr
attributes remembering the label which prevents any further use of the
file, which is a problem for block devices.

This also requires that the call to 'qemuBackupJobTerminate' from
'qemuProcessStop' happens only after 'vm->pid' was reset as otherwise
the security subdrivers attempt to enter the process namespace which
fails if the process isn't running any more.

Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1939082
Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_backup.c  | 36 ++--
 src/qemu/qemu_process.c |  8 
 2 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/src/qemu/qemu_backup.c b/src/qemu/qemu_backup.c
index f91d632715..430c11762c 100644
--- a/src/qemu/qemu_backup.c
+++ b/src/qemu/qemu_backup.c
@@ -28,6 +28,7 @@
 #include "qemu_monitor_json.h"
 #include "qemu_checkpoint.h"
 #include "qemu_command.h"
+#include "qemu_security.h"

 #include "storage_source.h"
 #include "storage_source_conf.h"
@@ -558,25 +559,40 @@ qemuBackupJobTerminate(virDomainObjPtr vm,

 {
 qemuDomainObjPrivatePtr priv = vm->privateData;
+g_autoptr(virQEMUDriverConfig) cfg = NULL;
 size_t i;

-if (!(priv->backup->apiFlags & VIR_DOMAIN_BACKUP_BEGIN_REUSE_EXTERNAL) &&
-(priv->backup->type == VIR_DOMAIN_BACKUP_TYPE_PULL ||
- (priv->backup->type == VIR_DOMAIN_BACKUP_TYPE_PUSH &&
-  jobstatus != QEMU_DOMAIN_JOB_STATUS_COMPLETED))) {
+for (i = 0; i < priv->backup->ndisks; i++) {
+virDomainBackupDiskDefPtr backupdisk = priv->backup->disks + i;

-g_autoptr(virQEMUDriverConfig) cfg = 
virQEMUDriverGetConfig(priv->driver);
+if (!backupdisk->store)
+continue;
+
+/* restore security label on the images in case the blockjob finishing
+ * handler didn't do so, such as when the VM was destroyed */
+if (backupdisk->state == VIR_DOMAIN_BACKUP_DISK_STATE_RUNNING ||
+backupdisk->state == VIR_DOMAIN_BACKUP_DISK_STATE_NONE) {
+if (qemuSecurityRestoreImageLabel(priv->driver, vm, 
backupdisk->store,
+  false) < 0)
+VIR_WARN("Unable to restore security label on %s",
+ NULLSTR(backupdisk->store->path));
+}
+
+/* delete unneeded images created by libvirt */
+if (backupdisk->store->type == VIR_STORAGE_TYPE_FILE &&
+!(priv->backup->apiFlags & VIR_DOMAIN_BACKUP_BEGIN_REUSE_EXTERNAL) 
&&
+(priv->backup->type == VIR_DOMAIN_BACKUP_TYPE_PULL ||
+ (priv->backup->type == VIR_DOMAIN_BACKUP_TYPE_PUSH &&
+  jobstatus != QEMU_DOMAIN_JOB_STATUS_COMPLETED))) {

-for (i = 0; i < priv->backup->ndisks; i++) {
-virDomainBackupDiskDefPtr backupdisk = priv->backup->disks + i;
 uid_t uid;
 gid_t gid;

-if (!backupdisk->store ||
-backupdisk->store->type != VIR_STORAGE_TYPE_FILE)
-continue;
+if (!cfg)
+cfg = virQEMUDriverGetConfig(priv->driver);

 qemuDomainGetImageIds(cfg, vm, backupdisk->store, NULL, , 
);
+
 if (virFileRemove(backupdisk->store->path, uid, gid) < 0)
 VIR_WARN("failed to remove scratch file '%s'",
  backupdisk->store->path);
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 5f31260221..0b79dde2c3 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -7822,10 +7822,6 @@ void qemuProcessStop(virQEMUDriverPtr driver,
 virResctrlAllocRemove(vm->def->resctrls[i]->alloc);
 }

-/* clean up a possible backup job */
-if (priv->backup)
-qemuBackupJobTerminate(vm, QEMU_DOMAIN_JOB_STATUS_CANCELED);
-
 qemuProcessRemoveDomainStatus(driver, vm);

 /* Remove VNC and Spice ports from port reservation bitmap, but only if
@@ -7877,6 +7873,10 @@ void qemuProcessStop(virQEMUDriverPtr driver,
 for (i = 0; i < vm->def->niothreadids; i++)
 vm->def->iothreadids[i]->thread_id = 0;

+/* clean up a possible backup job */
+if (priv->backup)
+qemuBackupJobTerminate(vm, QEMU_DOMAIN_JOB_STATUS_CANCELED);
+
 /* Do this explicitly after vm->pid is reset so that security drivers don't
  * try to enter the domain's namespace which is non-existent by now as qemu
  * is no longer running. */
-- 
2.29.2

[PATCH 2/3] qemu: backup: Enable full backup support

[Peter Krempa]

qemuBackupBegin can take a full backup of the disks (excluding any
operations with bitmaps) without the need to wait for the
blockdev-reopen support in qemu.

Add a check that no checkpoint creation is required and the disk backup
mode isn't VIR_DOMAIN_BACKUP_DISK_BACKUP_MODE_INCREMENTAL.

Call to virDomainBackupAlignDisks is moved earlier as it initializes the
disk backup mode if not present in user config.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_backup.c | 28 +++-
 src/qemu/qemu_domain.c |  3 ---
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/qemu/qemu_backup.c b/src/qemu/qemu_backup.c
index f91d632715..d89122f225 100644
--- a/src/qemu/qemu_backup.c
+++ b/src/qemu/qemu_backup.c
@@ -788,12 +788,33 @@ qemuBackupBegin(virDomainObjPtr vm,
 goto endjob;
 }

-if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_INCREMENTAL_BACKUP)) {
+if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_BLOCKDEV_BACKUP)) {
 virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
-   _("incremental backup is not supported yet"));
+   _("backup is not supported with this QEMU"));
 goto endjob;
 }

+if (virDomainBackupAlignDisks(def, vm->def, suffix) < 0)
+goto endjob;
+
+if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_INCREMENTAL_BACKUP)) {
+size_t i;
+
+if (chkdef) {
+virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
+   _("creating checkpoint for incremental backup is 
not supported yet"));
+goto endjob;
+}
+
+for (i = 0; i < def->ndisks; i++) {
+if (def->disks[i].backupmode == 
VIR_DOMAIN_BACKUP_DISK_BACKUP_MODE_INCREMENTAL) {
+virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
+   _("incremental backup is not supported yet"));
+goto endjob;
+}
+}
+}
+
 if (priv->backup) {
 virReportError(VIR_ERR_OPERATION_INVALID, "%s",
_("another backup job is already running"));
@@ -806,9 +827,6 @@ qemuBackupBegin(virDomainObjPtr vm,
 if (qemuBackupBeginPrepareTLS(vm, cfg, def, , ) < 
0)
 goto endjob;

-if (virDomainBackupAlignDisks(def, vm->def, suffix) < 0)
-goto endjob;
-
 actions = virJSONValueNewArray();

 /* The 'chk' checkpoint must be rolled back if the transaction command
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index ed2a1481d4..5c98f8ff1a 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -2322,9 +2322,6 @@ qemuDomainObjPrivateXMLFormatBackups(virBufferPtr buf,
 g_auto(virBuffer) attrBuf = VIR_BUFFER_INITIALIZER;
 g_auto(virBuffer) childBuf = VIR_BUFFER_INIT_CHILD(buf);

-if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_INCREMENTAL_BACKUP))
-return 0;
-
 if (priv->backup &&
 virDomainBackupDefFormat(, priv->backup, true) < 0)
 return -1;
-- 
2.29.2

[PATCH 3/3] NEWS: Mention support for full backups via virDomainBackupBegin

[Peter Krempa]

Signed-off-by: Peter Krempa 
---
 NEWS.rst | 9 +
 1 file changed, 9 insertions(+)

diff --git a/NEWS.rst b/NEWS.rst
index 9d819a3cf2..c2013ecac9 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -24,6 +24,15 @@ v7.2.0 (unreleased)
 The memory dirty rate stats can be obtained through ``virsh domstats
 --dirtyrate`` via the virConnectGetAllDomainStats API.

+  * qemu: Full disk backups via ``virDomainBackupBegin``
+
+The qemu hypervisor dirver now allows taking full disk backups via the
+``virDomainBackupBegin`` API and the corresponding virsh wrapper.
+
+In future releases the feature will be extended to also support incremental
+backups (where only the difference since the last backup is copied) when
+qemu adds the required functionality.
+
 * **Improvements**

 * **Bug fixes**
-- 
2.29.2

[PATCH 1/3] qemu: capabilities: Introduce QEMU_CAPS_BLOCKDEV_BACKUP

[Peter Krempa]

Upcoming commit will enable full backup support (incremental part
requires blockdev-reopen, which won't happen in qemu for at least
another release).

Add a capability that the 'blockdev-backup' job is supported by qemu
capped, but limited to when qemu supports QEMU_CAPS_BLOCKDEV.

We can also use it in the expression to enable
QEMU_CAPS_INCREMENTAL_BACKUP since it's a pre-requisite too.

Signed-off-by: Peter Krempa 
---
 src/qemu/qemu_capabilities.c  | 7 ++-
 src/qemu/qemu_capabilities.h  | 1 +
 tests/qemucapabilitiesdata/caps_4.2.0.aarch64.xml | 1 +
 tests/qemucapabilitiesdata/caps_4.2.0.s390x.xml   | 1 +
 tests/qemucapabilitiesdata/caps_4.2.0.x86_64.xml  | 1 +
 tests/qemucapabilitiesdata/caps_5.0.0.aarch64.xml | 1 +
 tests/qemucapabilitiesdata/caps_5.0.0.ppc64.xml   | 1 +
 tests/qemucapabilitiesdata/caps_5.0.0.riscv64.xml | 1 +
 tests/qemucapabilitiesdata/caps_5.0.0.x86_64.xml  | 1 +
 tests/qemucapabilitiesdata/caps_5.1.0.sparc.xml   | 1 +
 tests/qemucapabilitiesdata/caps_5.1.0.x86_64.xml  | 1 +
 tests/qemucapabilitiesdata/caps_5.2.0.aarch64.xml | 1 +
 tests/qemucapabilitiesdata/caps_5.2.0.ppc64.xml   | 1 +
 tests/qemucapabilitiesdata/caps_5.2.0.riscv64.xml | 1 +
 tests/qemucapabilitiesdata/caps_5.2.0.s390x.xml   | 1 +
 tests/qemucapabilitiesdata/caps_5.2.0.x86_64.xml  | 1 +
 tests/qemucapabilitiesdata/caps_6.0.0.x86_64.xml  | 1 +
 17 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c
index 447cf77875..08bd0e13ce 100644
--- a/src/qemu/qemu_capabilities.c
+++ b/src/qemu/qemu_capabilities.c
@@ -622,6 +622,7 @@ VIR_ENUM_IMPL(virQEMUCaps,
   /* 395 */
   "vnc-power-control",
   "audiodev",
+  "blockdev-backup",
 );


@@ -1557,6 +1558,7 @@ static struct virQEMUCapsStringFlags 
virQEMUCapsQMPSchemaQueries[] = {
 { "netdev_add/arg-type/+vhost-vdpa", QEMU_CAPS_NETDEV_VHOST_VDPA },
 { "migrate-set-parameters/arg-type/block-bitmap-mapping/bitmaps/transform",
   QEMU_CAPS_MIGRATION_PARAM_BLOCK_BITMAP_MAPPING },
+{ "blockdev-backup", QEMU_CAPS_BLOCKDEV_BACKUP },
 };

 typedef struct _virQEMUCapsObjectTypeProps virQEMUCapsObjectTypeProps;
@@ -5168,7 +5170,10 @@ virQEMUCapsInitQMPVersionCaps(virQEMUCapsPtr qemuCaps)
 void
 virQEMUCapsInitProcessCapsInterlock(virQEMUCapsPtr qemuCaps)
 {
-if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_BLOCKDEV) &&
+if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_BLOCKDEV))
+virQEMUCapsClear(qemuCaps, QEMU_CAPS_BLOCKDEV_BACKUP);
+
+if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_BLOCKDEV_BACKUP) &&
 virQEMUCapsGet(qemuCaps, QEMU_CAPS_BLOCKDEV_REOPEN) &&
 virQEMUCapsGet(qemuCaps, 
QEMU_CAPS_MIGRATION_PARAM_BLOCK_BITMAP_MAPPING))
 virQEMUCapsSet(qemuCaps, QEMU_CAPS_INCREMENTAL_BACKUP);
diff --git a/src/qemu/qemu_capabilities.h b/src/qemu/qemu_capabilities.h
index ee321df66c..792cb91ce1 100644
--- a/src/qemu/qemu_capabilities.h
+++ b/src/qemu/qemu_capabilities.h
@@ -602,6 +602,7 @@ typedef enum { /* virQEMUCapsFlags grouping marker for 
syntax-check */
 /* 395 */
 QEMU_CAPS_VNC_POWER_CONTROL, /* -vnc power-control option */
 QEMU_CAPS_AUDIODEV, /* -audiodev instead of QEMU_AUDIO_DRV */
+QEMU_CAPS_BLOCKDEV_BACKUP, /* qemu supports the blockdev-backup job */

 QEMU_CAPS_LAST /* this must always be the last item */
 } virQEMUCapsFlags;
diff --git a/tests/qemucapabilitiesdata/caps_4.2.0.aarch64.xml 
b/tests/qemucapabilitiesdata/caps_4.2.0.aarch64.xml
index b53584cfb8..371e6192e2 100644
--- a/tests/qemucapabilitiesdata/caps_4.2.0.aarch64.xml
+++ b/tests/qemucapabilitiesdata/caps_4.2.0.aarch64.xml
@@ -198,6 +198,7 @@
   
   
   
+  
   4001050
   0
   61700242
diff --git a/tests/qemucapabilitiesdata/caps_4.2.0.s390x.xml 
b/tests/qemucapabilitiesdata/caps_4.2.0.s390x.xml
index 363c4372c1..1f6f2e5833 100644
--- a/tests/qemucapabilitiesdata/caps_4.2.0.s390x.xml
+++ b/tests/qemucapabilitiesdata/caps_4.2.0.s390x.xml
@@ -160,6 +160,7 @@
   
   
   
+  
   4002000
   0
   39100242
diff --git a/tests/qemucapabilitiesdata/caps_4.2.0.x86_64.xml 
b/tests/qemucapabilitiesdata/caps_4.2.0.x86_64.xml
index 4443974912..8f9972eb14 100644
--- a/tests/qemucapabilitiesdata/caps_4.2.0.x86_64.xml
+++ b/tests/qemucapabilitiesdata/caps_4.2.0.x86_64.xml
@@ -246,6 +246,7 @@
   
   
   
+  
   4002000
   0
   43100242
diff --git a/tests/qemucapabilitiesdata/caps_5.0.0.aarch64.xml 
b/tests/qemucapabilitiesdata/caps_5.0.0.aarch64.xml
index 9a2b1f2061..51d83b9f47 100644
--- a/tests/qemucapabilitiesdata/caps_5.0.0.aarch64.xml
+++ b/tests/qemucapabilitiesdata/caps_5.0.0.aarch64.xml
@@ -208,6 +208,7 @@
   
   
   
+  
   500
   0
   61700241
diff --git a/tests/qemucapabilitiesdata/caps_5.0.0.ppc64.xml 
b/tests/qemucapabilitiesdata/caps_5.0.0.ppc64.xml
index ab12f7e72c..4e6aec6fda 100644
--- a/tests/qemucapabilitiesdata/caps_5.0.0.ppc64.xml
+++ b/tests/qemucapabilitiesdata/caps_5.0.0.ppc64.xml
@@ -217,6 

[PATCH 0/3] qemu: Enable virDomainBackupBegin for full backups

[Peter Krempa]

Peter Krempa (3):
  qemu: capabilities: Introduce QEMU_CAPS_BLOCKDEV_BACKUP
  qemu: backup: Enable full backup support
  NEWS: Mention support for full backups via virDomainBackupBegin

 NEWS.rst  |  9 ++
 src/qemu/qemu_backup.c| 28 +++
 src/qemu/qemu_capabilities.c  |  7 -
 src/qemu/qemu_capabilities.h  |  1 +
 src/qemu/qemu_domain.c|  3 --
 .../caps_4.2.0.aarch64.xml|  1 +
 .../qemucapabilitiesdata/caps_4.2.0.s390x.xml |  1 +
 .../caps_4.2.0.x86_64.xml |  1 +
 .../caps_5.0.0.aarch64.xml|  1 +
 .../qemucapabilitiesdata/caps_5.0.0.ppc64.xml |  1 +
 .../caps_5.0.0.riscv64.xml|  1 +
 .../caps_5.0.0.x86_64.xml |  1 +
 .../qemucapabilitiesdata/caps_5.1.0.sparc.xml |  1 +
 .../caps_5.1.0.x86_64.xml |  1 +
 .../caps_5.2.0.aarch64.xml|  1 +
 .../qemucapabilitiesdata/caps_5.2.0.ppc64.xml |  1 +
 .../caps_5.2.0.riscv64.xml|  1 +
 .../qemucapabilitiesdata/caps_5.2.0.s390x.xml |  1 +
 .../caps_5.2.0.x86_64.xml |  1 +
 .../caps_6.0.0.x86_64.xml |  1 +
 20 files changed, 54 insertions(+), 9 deletions(-)

-- 
2.29.2

Re: [libvirt PATCH 1/9] docs: improve description of secure attribute for loader element

[Kashyap Chamarthy]

On Thu, Mar 18, 2021 at 01:26:38PM +0100, Pavel Hrdina wrote:
> The original text was not explaining what this attribute actually
> controls and could have been interpreted as a control switch for the
> Secure boot feature in firmwares.

Yep, I've indeed seen people misread it as such.

> Signed-off-by: Pavel Hrdina 
> ---
>  docs/formatdomain.rst | 4 +++-
>  1 file changed, 3 insertions(+), 1 deletion(-)
> 
> diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
> index a2ea2690a5..c101d5a1f1 100644
> --- a/docs/formatdomain.rst
> +++ b/docs/formatdomain.rst
> @@ -167,7 +167,9 @@ harddisk, cdrom, network) determining where to 
> obtain/find the boot image.
> in the guest memory the file should be mapped. For instance, if the loader
> path points to an UEFI image, ``type`` should be ``pflash``. Moreover, 
> some
> firmwares may implement the Secure boot feature. Attribute ``secure`` can 
> be
> -   used then to control it. :since:`Since 2.1.0`
> +   used to tell the hypervisor that the firmware implements Secure Boot 
> Feature.

s/Feature/feature/

Perhaps: "firmware is capable of Secure Boot feature"

> +   It cannot be used to enable or disable the feature itself in the firmware.
> +   :since:`Since 2.1.0`

This additional clarification is good.

(Nit-pick: not this patch's fault: consistently use "Secure Boot"; I see
both "Secure boot" and "Secure Boot".)

Address the above only if you're respinning.  FWIW:

Reviewed-by: Kashyap Chamarthy 

>  ``nvram``
> Some UEFI firmwares may want to use a non-volatile memory to store some
> variables. In the host, this is represented as a file and the absolute 
> path
> -- 
> 2.30.2
> 

-- 
/kashyap

Re: [libvirt PATCH v4 5/6] ci: util: Add a registry checker for stale images

[Andrea Bolognani]

On Thu, 2021-03-18 at 17:11 +0100, Erik Skultety wrote:
> +msg = textwrap.dedent(f"""
> +The following images are stale and can be purged from the 
> registry:
> +
> +STALE_DETAILS
> +
> +You can delete the images listed above using this shell 
> snippet:
> +
> +$ for image_id in {stale_ids}; do
> +curl --request DELETE --header "PRIVATE-TOKEN: 
> " \\
> +{registry_uri}/$image_id;

There should be one additional leading space here, to align things
better.

Please also move the check_stale_image() function after the
refresh_cirrus() image - basically all action_*() functions should
continue being grouped together.

With those tweaks,

  Reviewed-by: Andrea Bolognani 

-- 
Andrea Bolognani / Red Hat / Virtualization

Re: [libvirt PATCH 9/9] qemu: implement support for firmware auto-selection feature filtering

[Michal Privoznik]

On 3/18/21 1:26 PM, Pavel Hrdina wrote:

Signed-off-by: Pavel Hrdina 
---
  src/qemu/qemu_firmware.c  | 40 +++
  ...re-efi-no-enrolled-keys.x86_64-latest.args | 49 ++
  .../os-firmware-efi-no-enrolled-keys.xml  | 25 ++
  tests/qemuxml2argvtest.c  |  1 +
  ...are-efi-no-enrolled-keys.x86_64-latest.xml | 50 +++
  tests/qemuxml2xmltest.c   |  1 +
  6 files changed, 166 insertions(+)
  create mode 100644 
tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args
  create mode 100644 tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.xml
  create mode 100644 
tests/qemuxml2xmloutdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.xml

diff --git a/src/qemu/qemu_firmware.c b/src/qemu/qemu_firmware.c
index d3198e2d45..f6f371f51f 100644
--- a/src/qemu/qemu_firmware.c
+++ b/src/qemu/qemu_firmware.c
@@ -930,6 +930,10 @@ qemuFirmwareMatchDomain(const virDomainDef *def,
  bool supportsS4 = false;
  bool requiresSMM = false;
  bool supportsSEV = false;
+bool supportsSecureBoot = false;
+bool hasEnrolledKeys = false;
+int reqSecureBoot;
+int reqEnrolledKeys;
  
  want = qemuFirmwareOSInterfaceTypeFromOsDefFirmware(def->os.firmware);
  
@@ -979,7 +983,13 @@ qemuFirmwareMatchDomain(const virDomainDef *def,

  break;
  
  case QEMU_FIRMWARE_FEATURE_SECURE_BOOT:

+supportsSecureBoot = true;
+break;
+
  case QEMU_FIRMWARE_FEATURE_ENROLLED_KEYS:
+hasEnrolledKeys = true;
+break;
+
  case QEMU_FIRMWARE_FEATURE_VERBOSE_DYNAMIC:
  case QEMU_FIRMWARE_FEATURE_VERBOSE_STATIC:
  case QEMU_FIRMWARE_FEATURE_NONE:
@@ -1000,6 +1010,36 @@ qemuFirmwareMatchDomain(const virDomainDef *def,
  return false;
  }
  
+if (def->os.firmwareFeatures) {

+reqSecureBoot = 
def->os.firmwareFeatures[VIR_DOMAIN_OS_DEF_FIRMWARE_FEATURE_SECURE_BOOT];
+if (reqSecureBoot != VIR_TRISTATE_BOOL_ABSENT) {
+if (reqSecureBoot == VIR_TRISTATE_BOOL_YES && !supportsSecureBoot) 
{
+VIR_DEBUG("User requested Secure Boot, firmware '%s' doesn't 
support it",
+  path);
+return false;
+}
+
+if (reqSecureBoot == VIR_TRISTATE_BOOL_NO && supportsSecureBoot) {
+VIR_DEBUG("User refused Secure Boot, firmware '%s' supports 
it", path);
+return false;
+}
+}
+
+reqEnrolledKeys = 
def->os.firmwareFeatures[VIR_DOMAIN_OS_DEF_FIRMWARE_FEATURE_ENROLLED_KEYS];
+if (reqEnrolledKeys != VIR_TRISTATE_BOOL_ABSENT) {
+if (reqEnrolledKeys == VIR_TRISTATE_BOOL_YES && !hasEnrolledKeys) {
+VIR_DEBUG("User requested Enrolled keys, firmware '%s' doesn't 
support it",


"doesn't have them" perhaps?


+  path);
+return false;
+}
+
+if (reqEnrolledKeys == VIR_TRISTATE_BOOL_NO && hasEnrolledKeys) {
+VIR_DEBUG("User refused Enrolled keys, firmware '%s' supports 
it", path);


"has them" perhaps?


+return false;
+}
+}
+}
+
  if (def->os.loader &&
  def->os.loader->secure == VIR_TRISTATE_BOOL_YES &&
  !requiresSMM) {
diff --git 
a/tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args 
b/tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args
new file mode 100644
index 00..561a905e78
--- /dev/null
+++ b/tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args
@@ -0,0 +1,49 @@
+LC_ALL=C \
+PATH=/bin \
+HOME=/tmp/lib/domain--1-fedora \
+USER=test \
+LOGNAME=test \
+XDG_DATA_HOME=/tmp/lib/domain--1-fedora/.local/share \
+XDG_CACHE_HOME=/tmp/lib/domain--1-fedora/.cache \
+XDG_CONFIG_HOME=/tmp/lib/domain--1-fedora/.config \
+/usr/bin/qemu-system-x86_64 \
+-name guest=fedora,debug-threads=on \
+-S \
+-object secret,id=masterKey0,format=raw,\
+file=/tmp/lib/domain--1-fedora/master-key.aes \
+-blockdev '{"driver":"file","filename":"/usr/share/OVMF/OVMF_CODE.fd",\
+"node-name":"libvirt-pflash0-storage","auto-read-only":true,\
+"discard":"unmap"}' \
+-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,\
+"driver":"raw","file":"libvirt-pflash0-storage"}' \
+-blockdev '{"driver":"file",\
+"filename":"/var/lib/libvirt/qemu/nvram/fedora_VARS.fd",\
+"node-name":"libvirt-pflash1-storage","auto-read-only":true,\
+"discard":"unmap"}' \
+-blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,\
+"driver":"raw","file":"libvirt-pflash1-storage"}' \
+-machine pc-q35-4.0,accel=kvm,usb=off,dump-guest-core=off,\
+pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,\
+memory-backend=pc.ram \
+-cpu qemu64 \
+-m 8 \
+-object memory-backend-ram,id=pc.ram,size=8388608 \
+-overcommit mem-lock=off \
+-smp 

Re: [libvirt PATCH 8/9] conf: introduce support for firmware auto-selection feature filtering

[Michal Privoznik]

On 3/18/21 1:26 PM, Pavel Hrdina wrote:

When the firmware auto-selection was introduced it always picked first
usable firmware based on the JSON descriptions on the host. It is
possible to add/remove/change the JSON files but it will always be for
the whole host.

This patch introduces support for configuring the auto-selection per VM
by adding users an option to limit what features they would like to have
available in the firmware.

Signed-off-by: Pavel Hrdina 
---
  docs/formatdomain.rst | 31 +++
  docs/schemas/domaincommon.rng | 23 +
  src/conf/domain_conf.c| 83 ++-
  src/conf/domain_conf.h| 10 +++
  .../os-firmware-efi-invalid-type.xml  | 28 +++
  ...os-firmware-invalid-type.x86_64-latest.err |  1 +
  .../os-firmware-invalid-type.xml  | 28 +++
  tests/qemuxml2argvtest.c  |  1 +
  ...aarch64-os-firmware-efi.aarch64-latest.xml |  1 +
  .../os-firmware-bios.x86_64-latest.xml|  1 +
  .../os-firmware-efi-secboot.x86_64-latest.xml |  1 +
  .../os-firmware-efi.x86_64-latest.xml |  1 +
  tests/vmx2xmldata/vmx2xml-firmware-efi.xml|  1 +
  13 files changed, 207 insertions(+), 3 deletions(-)



  create mode 100644 tests/qemuxml2argvdata/os-firmware-efi-invalid-type.xml

>   create mode 100644 tests/qemuxml2argvdata/os-firmware-invalid-type.xml

These two are identical. Have you intended them to be different?


  create mode 100644 
tests/qemuxml2argvdata/os-firmware-invalid-type.x86_64-latest.err




diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
index c101d5a1f1..dd063b0794 100644
--- a/docs/formatdomain.rst
+++ b/docs/formatdomain.rst
@@ -155,6 +155,37 @@ harddisk, cdrom, network) determining where to obtain/find 
the boot image.
 the host native arch will be chosen. For the ``test``, ``ESX`` and 
``VMWare``
 hypervisor drivers, however, the ``i686`` arch will always be chosen even 
on
 an ``x86_64`` host. :since:`Since 0.0.1`
+``firmware``
+   :since:`Since 7.2.0 QEMU/KVM only`
+
+   When used together with ``firmware`` attribute of ``os`` element the 
``type``
+   attribute must have the same value.
+
+   List of mandatory attributes:
+
+   - ``type`` (accepted values are ``bios`` and ``efi``) same as the 
``firmware``
+ attribute of ``os`` element.
+
+   When using firmware auto-selection there are different features enabled in
+   the firmwares. The list of features can be used to limit what firmware 
should
+   be automatically selected for the VM. The list of features can be specified
+   using zero or more ``feature`` elements. Libvirt will take into 
consideration
+   only the listed features and ignore the rest when selecting the firmware.
+
+   ``feature``
+  The list of mandatory attributes:
+
+  - ``enabled`` (accepted values are ``yes`` and ``no``) is used to tell 
libvirt
+if the feature must be enabled or not in the automatically selected 
firmware
+
+  - ``name`` the name of the feature, the list of the features:
+
+- ``enrolled-keys`` whether the selected nvram template has default
+  certificate enrolled. Firmware with Secure Boot feature but without
+  enrolled keys will successfully boot non-signed binaries as well.
+  Valid only for firmwares with Secure Boot feature.
+
+- ``secure-boot`` whether the firmware implements UEFI Secure boot 
feature.
  ``loader``
 The optional ``loader`` tag refers to a firmware blob, which is specified 
by
 absolute path, used to assist the domain creation process. It is used by 
Xen
diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng
index e6db2f5b74..1dbfc68f18 100644
--- a/docs/schemas/domaincommon.rng
+++ b/docs/schemas/domaincommon.rng
@@ -276,6 +276,29 @@

  
  
+
+  
+
+  
+bios
+efi
+  
+
+
+  
+
+  
+
+
+  
+enrolled-keys
+secure-boot
+  
+
+  
+
+  
+
  

  
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 7729333897..dcfe5c0d03 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -1318,6 +1318,12 @@ VIR_ENUM_IMPL(virDomainOsDefFirmware,
"efi",
  );
  
+VIR_ENUM_IMPL(virDomainOsDefFirmwareFeature,

+  VIR_DOMAIN_OS_DEF_FIRMWARE_FEATURE_LAST,
+  "enrolled-keys",
+  "secure-boot",
+);
+
  VIR_ENUM_IMPL(virDomainCFPC,
VIR_DOMAIN_CFPC_LAST,
"none",
@@ -19600,22 +19606,67 @@ virDomainDefParseBootFirmwareOptions(virDomainDefPtr 
def,
   

[libvirt PATCH v4 5/6] ci: util: Add a registry checker for stale images

[Erik Skultety]

This function checks whether there are any stale Docker images in the
registry that can be purged. Since we're pulling available container
images from our GitLab registry with the 'list-images' action, it
could happen that we'd list old (already unsupported) images and make
them available for the user to consume and run a build in them.
Naturally, the build will most likely fail leaving the user confused.

Signed-off-by: Erik Skultety 
---
 ci/helper  | 46 +-
 ci/util.py | 41 +
 2 files changed, 86 insertions(+), 1 deletion(-)

diff --git a/ci/helper b/ci/helper
index 31cf72fbdf..0743f95e13 100755
--- a/ci/helper
+++ b/ci/helper
@@ -10,6 +10,7 @@ import pty
 import shutil
 import subprocess
 import sys
+import textwrap
 
 import util
 
@@ -130,7 +131,7 @@ class Parser:
 refreshparser = subparsers.add_parser(
 "refresh",
 help="refresh data generated with lcitool",
-parents=[lcitoolparser],
+parents=[lcitoolparser, gitlabparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 refreshparser.add_argument(
@@ -139,6 +140,13 @@ class Parser:
 default=False,
 help="refresh data silently"
 )
+refreshparser.add_argument(
+"--check-stale",
+action="store",
+choices=["yes", "no"],
+default="yes",
+help="check for existence of stale images on the GitLab instance"
+)
 refreshparser.set_defaults(func=Application.action_refresh)
 
 def parse(self):
@@ -287,10 +295,46 @@ class Application:
 print("Available cross-compiler container images:\n")
 print(spacing + ("\n" + spacing).join(cross))
 
+def check_stale_images(self):
+namespace = self.args.namespace
+gitlab_uri = self.args.gitlab_uri
+registry_uri = util.get_registry_uri(namespace, gitlab_uri)
+lcitool_hosts = self.lcitool_get_hosts()
+
+stale_images = util.get_registry_stale_images(registry_uri,
+  lcitool_hosts)
+if stale_images:
+spacing = "\n" + 4 * " "
+stale_fmt = [f"{k} (ID: {v})" for k, v in stale_images.items()]
+stale_details = spacing.join(stale_fmt)
+stale_ids = ' '.join([str(id) for id in stale_images.values()])
+registry_uri = util.get_registry_uri(namespace, gitlab_uri)
+
+msg = textwrap.dedent(f"""
+The following images are stale and can be purged from the 
registry:
+
+STALE_DETAILS
+
+You can delete the images listed above using this shell 
snippet:
+
+$ for image_id in {stale_ids}; do
+curl --request DELETE --header "PRIVATE-TOKEN: 
" \\
+{registry_uri}/$image_id;
+  done
+
+You can generate a personal access token here:
+
+{gitlab_uri}/-/profile/personal_access_tokens
+""")
+print(msg.replace("STALE_DETAILS", stale_details))
+
 def action_refresh(self):
 self.refresh_containers()
 self.refresh_cirrus()
 
+if self.args.check_stale == "yes" and not self.args.quiet:
+self.check_stale_images()
+
 def run(self):
 self.args.func(self)
 
diff --git a/ci/util.py b/ci/util.py
index f9f8320276..90d58454be 100644
--- a/ci/util.py
+++ b/ci/util.py
@@ -38,3 +38,44 @@ def get_registry_images(uri: str) -> List[Dict]:
 
 # read the HTTP response and load the JSON part of it
 return json.loads(r.read().decode())
+
+
+def get_image_distro(image_name: str) -> str:
+"""
+Extract the name of the distro in the GitLab image registry name, e.g.
+ci-debian-9-cross-mipsel --> debian-9
+
+:param image_name: name of the GitLab registry image
+:return: distro name as a string
+"""
+name_prefix = "ci-"
+name_suffix = "-cross-"
+
+distro = image_name[len(name_prefix):]
+
+index = distro.find(name_suffix)
+if index > 0:
+distro = distro[:index]
+
+return distro
+
+
+def get_registry_stale_images(registry_uri: str,
+  supported_distros: List[str]) -> Dict[str, int]:
+"""
+Check the GitLab image registry for images that we no longer support and
+which should be deleted.
+
+:param uri: URI pointing to a GitLab instance's image registry
+:param supported_distros: list of hosts supported by lcitool
+:return: dictionary formatted as: {: }
+"""
+
+images = get_registry_images(registry_uri)
+
+stale_images = {}
+for img in images:
+if get_image_distro(img["name"]) not in supported_distros:
+stale_images[img["name"]] = img["id"]
+
+return stale_images
-- 
2.30.2

[libvirt PATCH v4 6/6] ci: helper: Apply Python naming practice to private methods/attributes

[Erik Skultety]

As documented at [1], the common practice with respect to private
attributes/methods naming is to prefix them with an underscore.

[1] https://docs.python.org/3/tutorial/classes.html#private-variables

Signed-off-by: Erik Skultety 
---
 ci/helper | 122 +++---
 1 file changed, 61 insertions(+), 61 deletions(-)

diff --git a/ci/helper b/ci/helper
index 0743f95e13..70c9cb4f6b 100755
--- a/ci/helper
+++ b/ci/helper
@@ -84,8 +84,8 @@ class Parser:
 )
 
 # Main parser
-self.parser = argparse.ArgumentParser()
-subparsers = self.parser.add_subparsers(
+self._parser = argparse.ArgumentParser()
+subparsers = self._parser.add_subparsers(
 dest="action",
 metavar="ACTION",
 )
@@ -98,7 +98,7 @@ class Parser:
 parents=[containerparser, mesonparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
-buildparser.set_defaults(func=Application.action_build)
+buildparser.set_defaults(func=Application._action_build)
 
 # test action
 testparser = subparsers.add_parser(
@@ -107,7 +107,7 @@ class Parser:
 parents=[containerparser, mesonparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
-testparser.set_defaults(func=Application.action_test)
+testparser.set_defaults(func=Application._action_test)
 
 # shell action
 shellparser = subparsers.add_parser(
@@ -116,7 +116,7 @@ class Parser:
 parents=[containerparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
-shellparser.set_defaults(func=Application.action_shell)
+shellparser.set_defaults(func=Application._action_shell)
 
 # list-images action
 listimagesparser = subparsers.add_parser(
@@ -125,7 +125,7 @@ class Parser:
 parents=[gitlabparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
-listimagesparser.set_defaults(func=Application.action_list_images)
+listimagesparser.set_defaults(func=Application._action_list_images)
 
 # refresh action
 refreshparser = subparsers.add_parser(
@@ -147,56 +147,56 @@ class Parser:
 default="yes",
 help="check for existence of stale images on the GitLab instance"
 )
-refreshparser.set_defaults(func=Application.action_refresh)
+refreshparser.set_defaults(func=Application._action_refresh)
 
 def parse(self):
-return self.parser.parse_args()
+return self._parser.parse_args()
 
 
 class Application:
 def __init__(self):
-self.basedir = pathlib.Path(__file__).resolve().parent
-self.args = Parser().parse()
+self._basedir = pathlib.Path(__file__).resolve().parent
+self._args = Parser().parse()
 
-if self.args.action == "refresh":
-if not shutil.which(self.args.lcitool):
+if self._args.action == "refresh":
+if not shutil.which(self._args.lcitool):
 sys.exit("error: 'lcitool' not installed")
 
-def make_run(self, target):
+def _make_run(self, target):
 args = [
 "-C",
-self.basedir,
+self._basedir,
 target,
 ]
 
-if self.args.action in ["build", "test", "shell"]:
+if self._args.action in ["build", "test", "shell"]:
 args.extend([
-f"CI_ENGINE={self.args.engine}",
-f"CI_USER_LOGIN={self.args.login}",
-f"CI_IMAGE_PREFIX={self.args.image_prefix}",
-f"CI_IMAGE_TAG={self.args.image_tag}",
+f"CI_ENGINE={self._args.engine}",
+f"CI_USER_LOGIN={self._args.login}",
+f"CI_IMAGE_PREFIX={self._args.image_prefix}",
+f"CI_IMAGE_TAG={self._args.image_tag}",
 ])
 
-if self.args.action in ["build", "test"]:
+if self._args.action in ["build", "test"]:
 args.extend([
-f"CI_MESON_ARGS={self.args.meson_args}",
-f"CI_NINJA_ARGS={self.args.ninja_args}",
+f"CI_MESON_ARGS={self._args.meson_args}",
+f"CI_NINJA_ARGS={self._args.ninja_args}",
 ])
 
 if pty.spawn(["make"] + args) != 0:
 sys.exit("error: 'make' failed")
 
-def lcitool_run(self, args):
-output = subprocess.check_output([self.args.lcitool] + args)
+def _lcitool_run(self, args):
+output = subprocess.check_output([self._args.lcitool] + args)
 return output.decode("utf-8")
 
-def lcitool_get_hosts(self):
-output = self.lcitool_run(["hosts"])
+def _lcitool_get_hosts(self):
+output = self._lcitool_run(["hosts"])
 return output.splitlines()
 
-def generate_dockerfile(self, host, cross=None):
+def 

[libvirt PATCH v4 2/6] ci: helper: Introduce --quiet

[Erik Skultety]

Offer an option to silence all output to stdout, e.g. when generating
dockerfiles/varsfiles.

Signed-off-by: Erik Skultety 
Reviewed-by: Andrea Bolognani 
---
 ci/helper | 12 ++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/ci/helper b/ci/helper
index 8f34f4b59d..53baef3cb1 100755
--- a/ci/helper
+++ b/ci/helper
@@ -116,6 +116,12 @@ class Parser:
 parents=[lcitoolparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
+refreshparser.add_argument(
+"--quiet",
+action="store_true",
+default=False,
+help="refresh data silently"
+)
 refreshparser.set_defaults(func=Application.action_refresh)
 
 def parse(self):
@@ -173,7 +179,8 @@ class Application:
 outfile = f"ci-{host}-cross-{cross}.Dockerfile"
 
 outpath = outdir.joinpath(outfile)
-print(outpath)
+if not self.args.quiet:
+print(outpath)
 
 output = self.lcitool_run(args)
 with open(outpath, "w") as f:
@@ -185,7 +192,8 @@ class Application:
 outfile = f"{host}.vars"
 
 outpath = outdir.joinpath(outfile)
-print(outpath)
+if not self.args.quiet:
+print(outpath)
 
 output = self.lcitool_run(args)
 with open(outpath, "w") as f:
-- 
2.30.2

[libvirt PATCH v4 4/6] ci: helper: Rewrite image listing to Python

[Erik Skultety]

The corresponding Bash script is dropped.
After this patch's rewrite, the Makefile's original image listing
target remains intact only to notify the user to use the Python helper
instead.

Signed-off-by: Erik Skultety 
Reviewed-by: Andrea Bolognani 
---
 ci/Makefile   | 12 
 ci/helper | 38 +-
 ci/list-images.sh | 14 --
 3 files changed, 37 insertions(+), 27 deletions(-)
 delete mode 100644 ci/list-images.sh

diff --git a/ci/Makefile b/ci/Makefile
index 72f5bda942..02ce0df7ee 100644
--- a/ci/Makefile
+++ b/ci/Makefile
@@ -211,17 +211,6 @@ ci-build@%:
 ci-test@%:
$(MAKE) -C $(CI_ROOTDIR) ci-build@$* CI_NINJA_ARGS=test
 
-ci-list-images:
-   @echo
-   @echo "Available x86 container images:"
-   @echo
-   @sh list-images.sh "$(CI_IMAGE_PREFIX)" | grep -v cross
-   @echo
-   @echo "Available cross-compiler container images:"
-   @echo
-   @sh list-images.sh "$(CI_IMAGE_PREFIX)" | grep cross
-   @echo
-
 ci-help:
@echo
@echo
@@ -241,7 +230,6 @@ ci-help:
@echo "ci-build@\$$IMAGE - run a default 'ninja' build"
@echo "ci-test@\$$IMAGE  - run a 'ninja test'"
@echo "ci-shell@\$$IMAGE - run an interactive shell"
-   @echo "ci-list-images  - list available images"
@echo "ci-help - show this help message"
@echo
@echo "Available make variables:"
diff --git a/ci/helper b/ci/helper
index 53baef3cb1..31cf72fbdf 100755
--- a/ci/helper
+++ b/ci/helper
@@ -11,6 +11,8 @@ import shutil
 import subprocess
 import sys
 
+import util
+
 
 class Parser:
 def __init__(self):
@@ -66,6 +68,20 @@ class Parser:
 help="path to lcitool binary",
 )
 
+# Options that are common to actions communicating with a GitLab
+# instance
+gitlabparser = argparse.ArgumentParser(add_help=False)
+gitlabparser.add_argument(
+"--namespace",
+default="libvirt/libvirt",
+help="GitLab project namespace"
+)
+gitlabparser.add_argument(
+"--gitlab-uri",
+default="https://gitlab.com;,
+help="base GitLab URI"
+)
+
 # Main parser
 self.parser = argparse.ArgumentParser()
 subparsers = self.parser.add_subparsers(
@@ -105,6 +121,7 @@ class Parser:
 listimagesparser = subparsers.add_parser(
 "list-images",
 help="list known container images",
+parents=[gitlabparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 listimagesparser.set_defaults(func=Application.action_list_images)
@@ -249,7 +266,26 @@ class Application:
 self.make_run(f"ci-shell@{self.args.target}")
 
 def action_list_images(self):
-self.make_run(f"ci-list-images")
+registry_uri = util.get_registry_uri(self.args.namespace,
+ self.args.gitlab_uri)
+images = util.get_registry_images(registry_uri)
+
+# skip the "ci-" prefix each of our container images' name has
+name_prefix = "ci-"
+names = [i["name"][len(name_prefix):] for i in images]
+names.sort()
+
+native = [name for name in names if "-cross-" not in name]
+cross = [name for name in names if "-cross-" in name]
+
+spacing = 4 * " "
+print("Available x86 container images:\n")
+print(spacing + ("\n" + spacing).join(native))
+
+if cross:
+print()
+print("Available cross-compiler container images:\n")
+print(spacing + ("\n" + spacing).join(cross))
 
 def action_refresh(self):
 self.refresh_containers()
diff --git a/ci/list-images.sh b/ci/list-images.sh
deleted file mode 100644
index b85b132253..00
--- a/ci/list-images.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-prefix="${1##registry.gitlab.com/}"
-
-PROJECT_ID=192693
-
-all_repos() {
-  curl -s 
"https://gitlab.com/api/v4/projects/$PROJECT_ID/registry/repositories?per_page=100;
 \
-| tr , '\n' | grep '"path":' | sed 's,"path":",,g;s,"$,,g'
-}
-
-all_repos | grep "^$prefix" | sed "s,^$prefix,,g" | while read repo; do
-echo "$repo"
-done | sort -u
-- 
2.30.2

[libvirt PATCH v4 3/6] ci: Introduce a util module

[Erik Skultety]

With the gradual rewrite of the Makefile to the 'helper' script will
require helper functions that would better live in a separate util
module.

Signed-off-by: Erik Skultety 
Reviewed-by: Andrea Bolognani 
---
 ci/util.py | 40 
 1 file changed, 40 insertions(+)
 create mode 100644 ci/util.py

diff --git a/ci/util.py b/ci/util.py
new file mode 100644
index 00..f9f8320276
--- /dev/null
+++ b/ci/util.py
@@ -0,0 +1,40 @@
+import json
+import urllib.request
+import urllib.parse
+
+from typing import Dict, List
+
+
+def get_registry_uri(namespace: str,
+ gitlab_uri: str = "https://gitlab.com;) -> str:
+"""
+Construct a v4 API URI pointing the namespaced project's image registry.
+
+:param namespace: GitLab project namespace, e.g. "libvirt/libvirt"
+:param gitlab_uri: GitLab base URI, can be a private deployment
+:param api_version: GitLab REST API version number
+:return: URI pointing to a namespaced project's image registry
+"""
+
+# this converts something like "libvirt/libvirt" to "libvirt%2Flibvirt"
+namespace_urlenc = urllib.parse.quote_plus(namespace)
+
+project_uri = f"{gitlab_uri}/api/v4/projects/{namespace_urlenc}"
+
+uri = project_uri + "/registry/repositories"
+return uri
+
+
+def get_registry_images(uri: str) -> List[Dict]:
+"""
+List all container images that are currently available in the given GitLab
+project.
+
+:param uri: URI pointing to a GitLab instance's image registry
+:return: list of container image names
+"""
+
+r = urllib.request.urlopen(uri + "?per_page=100")
+
+# read the HTTP response and load the JSON part of it
+return json.loads(r.read().decode())
-- 
2.30.2

[libvirt PATCH v4 0/6] ci: helper: Rewrite image listing in Python and add stale Docker image checker

[Erik Skultety]

since v3:
- more formatting changes as requested per review comments
- get_registry_stale_images was simplified according to the comments
- class attributes were also converted to private in [6/6]

v1: https://listman.redhat.com/archives/libvir-list/2021-February/msg00641.ht=
ml
v2: https://listman.redhat.com/archives/libvir-list/2021-March/msg00789.html
v3: https://listman.redhat.com/archives/libvir-list/2021-March/msg00892.html

Erik Skultety (6):
  ci: helper: Use the ArgumentDefaultsHelpFormatter help formatter
  ci: helper: Introduce --quiet
  ci: Introduce a util module
  ci: helper: Rewrite image listing to Python
  ci: util: Add a registry checker for stale images
  ci: helper: Apply Python naming practice to private methods/attributes

 ci/Makefile   |  12 ---
 ci/helper | 203 +-
 ci/list-images.sh |  14 
 ci/util.py|  81 ++
 4 files changed, 229 insertions(+), 81 deletions(-)
 delete mode 100644 ci/list-images.sh
 create mode 100644 ci/util.py

--=20
2.30.2

[libvirt PATCH v4 1/6] ci: helper: Use the ArgumentDefaultsHelpFormatter help formatter

[Erik Skultety]

This help formatter class reports the defaults we use for options
taking an argument.

Signed-off-by: Erik Skultety 
Reviewed-by: Andrea Bolognani 
---
 ci/helper | 5 +
 1 file changed, 5 insertions(+)

diff --git a/ci/helper b/ci/helper
index 73a3f729da..8f34f4b59d 100755
--- a/ci/helper
+++ b/ci/helper
@@ -79,6 +79,7 @@ class Parser:
 "build",
 help="run a build in a container",
 parents=[containerparser, mesonparser],
+formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 buildparser.set_defaults(func=Application.action_build)
 
@@ -87,6 +88,7 @@ class Parser:
 "test",
 help="run a build in a container (including tests)",
 parents=[containerparser, mesonparser],
+formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 testparser.set_defaults(func=Application.action_test)
 
@@ -95,6 +97,7 @@ class Parser:
 "shell",
 help="start a shell in a container",
 parents=[containerparser],
+formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 shellparser.set_defaults(func=Application.action_shell)
 
@@ -102,6 +105,7 @@ class Parser:
 listimagesparser = subparsers.add_parser(
 "list-images",
 help="list known container images",
+formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 listimagesparser.set_defaults(func=Application.action_list_images)
 
@@ -110,6 +114,7 @@ class Parser:
 "refresh",
 help="refresh data generated with lcitool",
 parents=[lcitoolparser],
+formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 refreshparser.set_defaults(func=Application.action_refresh)
 
-- 
2.30.2

Re: [libvirt PATCH v3 6/6] ci: helper: Convert private methods to Python common naming practice

[Erik Skultety]

On Thu, Mar 18, 2021 at 12:09:19PM +0100, Andrea Bolognani wrote:
> On Thu, 2021-03-18 at 09:09 +0100, Erik Skultety wrote:
> > As documented at [1], the common practice wrt to private
> 
> s/wrt to/with respect to/
> 
> >  def parse(self):
> >  return self.parser.parse_args()
> 
> Shouldn't this change be applied to Parser.parser...
> 
> >  def run(self):
> >  self.args.func(self)
> 
> ... and Application.args too?
> 
> Everything else looks good.

Applied.

Erik

Re: [libvirt PATCH v3 5/6] ci: util: Add a registry checker for stale images

[Erik Skultety]

...

> > +def get_registry_stale_images(registry_uri: str,
> > +  supported_distros: List[str]) -> Dict[str, 
> > int]:
> > +"""
> > +Check the GitLab image registry for images that we no longer support 
> > and
> > +which should be deleted.
> > +
> > +:param uri: URI pointing to a GitLab instance's image registry
> > +:param supported_distros: list of hosts supported by lcitool
> > +:return: dictionary formatted as: {: 
> > }
> > +"""
> > +
> > +images = get_registry_images(registry_uri)
> > +
> > +# extract distro names from the list of registry images
> > +registry_distros = [get_image_distro(i["name"]) for i in images]
> > +
> > +# - compare the distros powering the images in GitLab registry with
> > +#   the list of host available from lcitool
> > +# - @unsupported is a set containing the distro names which we no 
> > longer
> > +#   support; we need to map these back to registry image names
> > +unsupported = set(registry_distros) - set(supported_distros)
> > +if unsupported:
> > +stale_images = {}
> > +for distro in unsupported:
> > +for img in images:
> > +# gitlab images are named like "ci--?"
> > +if distro in img["name"]:
> > +stale_images[img["name"]] = img["id"]
> > +
> > +return stale_images
> 
> As far as I can tell, this can be achieved in a much more
> straightforward way with
> 
>   def get_registry_stale_images(registry_uri, supported_distros):
> 
>   images = get_registry_images(registry_uri)
>   stale_images = {}
> 
>   for img in images:
>   if get_image_distro(img["name"]) not in supported_distros:
>   stale_images[img["name"]] = img["id"]
> 
>   return stale_images
> 
> At least from a quick test, the results appear to be the same. Am I
> missing something?

No, it works. I'll respin.

Erik

[PATCH v7 00/11] Configurable policy for handling deprecated interfaces

[Markus Armbruster]

New option -compat lets you configure what to do when deprecated
interfaces get used.  This is intended for testing users of the
management interfaces.  It is experimental.

-compat deprecated-input= configures what to do when
deprecated input is received.  Available policies:

* accept: Accept deprecated commands and arguments (default)
* reject: Reject them
* crash: Crash

-compat deprecated-output= configures what to do when
deprecated output is sent.  Available output policies:

* accept: Emit deprecated command results and events (default)
* hide: Suppress them

For now, -compat covers only deprecated syntactic aspects of QMP.  We
may want to extend it to cover semantic aspects, CLI, and experimental
features.

v7:
* Trivially rebased
* PATCH 01: New, bug fix from Paolo
* PATCH 02+11: Since: comments fixed, for real this time [Eric]
* PATCH 03+10: qobject_output_visitor_new_qmp() and
  qobject_input_visitor_new_qmp() moved to qmp-dispatch.c to not break
  --disable-system --disable-tools builds
* PATCH 05: Minor fallout from changed PATCH 03
* PATCH 06: Commit message typo fixed [Eric]

v6:
* Rebased, with straightforward conflicts
* PATCH 1: More verbose commit message, explaining intent, and the
  lack of introspection [Peter], comments updated for 6.0, Eric's R-by
  kept anyway
* PATCH 5+6: Split old PATCH 5 [Eric], Eric's R-by lept anyway
* PATCH 7: New, correcting a latent issue so it doesn't break PATCH 8
* PATCH 10: Doc fix [Eric]

v5:
* Old PATCH 01-26 merged in commit f57587c7d47.
* Rebased, non-trivial conflicts in PATCH 1 due to Meson, and in PATCH
  7 due to visitor changes
* PATCH 1: Comments updated for 5.2 [Eric]
* PATCH 2: Harmless missing initialization fixed [Eric]
* PATCH 3+4: Harmless missing has_FOO = true fixed [Eric]
* PATCH 6+7: Commit message tweaked

v4:
* PATCH 05+07: Temporary memory leak plugged [Marc-André]
* PATCH 23: Rewritten [Marc-André]
* PATCH 24: Comment typo [Marc-André]
* PATCH 30: Memory leaks plugged

v3:
* Rebased, non-trivial conflicts in PATCH 01+26+27+34 due to RST
  conversion and code motion
* PATCH 28-29: Old PATCH 28 split up to ease review
* PATCH 30-31: New
* PATCH 32-33: Old PATCH 29 split up to ease review

Comparison to RFC (24 Oct 2019):
* Cover arguments and results in addition to commands and events
* Half-baked "[RFC PATCH 18/19] qapi: Include a warning in the
  response to a deprecated command" dropped

See also last item of
Subject: Minutes of KVM Forum BoF on deprecating stuff
Date: Fri, 26 Oct 2018 16:03:51 +0200
Message-ID: <87mur0ls8o@dusky.pond.sub.org>
https://lists.nongnu.org/archive/html/qemu-devel/2018-10/msg05828.html

Cc: Lukáš Doktor 
Cc: libgues...@redhat.com
Cc: libvir-list@redhat.com
Cc: Daniel P. Berrange 
Cc: Peter Krempa 
Cc: Kevin Wolf 

Markus Armbruster (10):
  qemu-options: New -compat to set policy for deprecated interfaces
  qapi: Implement deprecated-output=hide for QMP command results
  qapi: Implement deprecated-output=hide for QMP events
  qapi: Implement deprecated-output=hide for QMP event data
  monitor: Drop query-qmp-schema 'gen': false hack
  qapi: Implement deprecated-output=hide for QMP introspection
  test-util-sockets: Add stub for monitor_set_cur()
  qapi: Implement deprecated-input=reject for QMP commands
  qapi: Implement deprecated-input=reject for QMP command arguments
  qapi: New -compat deprecated-input=crash

Paolo Bonzini (1):
  qemuutil: remove qemu_set_fd_handler duplicate symbol

 qapi/compat.json|  52 
 qapi/introspect.json|   2 +-
 qapi/qapi-schema.json   |   1 +
 include/qapi/compat-policy.h|  38 +
 include/qapi/qmp/dispatch.h |   1 +
 include/qapi/qobject-input-visitor.h|   4 +
 include/qapi/qobject-output-visitor.h   |   4 +
 include/qapi/visitor-impl.h |   6 ++
 include/qapi/visitor.h  |  18 +
 monitor/monitor-internal.h  |   3 -
 monitor/misc.c  |   2 -
 monitor/qmp-cmds-control.c  | 100 +---
 qapi/qapi-visit-core.c  |  18 +
 qapi/qmp-dispatch.c |  35 +
 qapi/qobject-input-visitor.c|  29 +++
 qapi/qobject-output-visitor.c   |  19 +
 softmmu/vl.c|  17 
 storage-daemon/qemu-storage-daemon.c|   2 -
 stubs/set-fd-handler.c  |  10 ---
 tests/unit/test-qmp-cmds.c  |  91 +++--
 tests/unit/test-qmp-event.c |  41 ++
 tests/unit/test-util-sockets.c  |   1 +
 qapi/meson.build|   1 +
 qapi/trace-events   |   2 +
 qemu-options.hx |  22 ++
 scripts/qapi/commands.py|  17 ++--
 scripts/qapi/events.py  |  21 -
 scripts/qapi/visit.py   |  15 
 stubs/meson.build  

Re: [libvirt PATCH 02/38] virxml: Add virXMLPropYesNo

[Pavel Hrdina]

On Thu, Mar 18, 2021 at 09:00:41AM +0100, Tim Wiederhake wrote:
> Convenience function to return value of a yes / no attribute.
> 
> Does not use virTristateBoolTypeFromString to disallow "default".
> 
> Signed-off-by: Tim Wiederhake 
> ---
>  src/util/virxml.c | 37 +
>  src/util/virxml.h |  4 
>  2 files changed, 41 insertions(+)
> 
> diff --git a/src/util/virxml.c b/src/util/virxml.c
> index 060b7530fc..47e8414bd5 100644
> --- a/src/util/virxml.c
> +++ b/src/util/virxml.c
> @@ -556,6 +556,43 @@ virXMLNodeContentString(xmlNodePtr node)
>  }
>  
>  
> +/**
> + * virXMLPropYesNo:
> + * @node: XML dom node pointer
> + * @name: Name of the property (attribute) to get
> + * @value: The returned virTristateBool value
> + *
> + * Convenience function to return value of a yes / no attribute.
> + *
> + * Returns 1 in case of success in which case @value is set,
> + * or 0 if the attribute is not present,
> + * or -1 and reports an error on failure.
> + */
> +int
> +virXMLPropYesNo(xmlNodePtr node, const char* name, virTristateBool *value)
> +{
> +g_autofree char *tmp = virXMLPropString(node, name);
> +
> +if (!tmp)
> +return 0;
> +

I was wondering if we can return -1 or 0 only by adding addition
argument to the function, for example "bool mandatory".

The condition here could be changed to this:

if (!tmp) {
if (mandatory) {
virReportError(VIR_ERR_XML_ERROR,
   _(Missing mandatory attribute '%s' in element '%s'.),
   name, node->name);

return -1;
}

return 0;
}

This would unify our error message for all the yes/no on/off options as
well.

> +if (STREQ("yes", tmp)) {
> +*value = VIR_TRISTATE_BOOL_YES;
> +return 1;
> +}
> +
> +if (STREQ("no", tmp)) {
> +*value = VIR_TRISTATE_BOOL_NO;
> +return 1;
> +}
> +
> +virReportError(VIR_ERR_XML_ERROR,
> +   _("Invalid value for attribute '%s' in node '%s': '%s'. 
> Expected 'yes' or 'no'"),

s/node/element/ ? We should unify our terminology and I'm not sure if we
prefer element or node.

Pavel


signature.asc
Description: PGP signature

Re: [libvirt PATCH 02/38] virxml: Add virXMLPropYesNo

[Daniel P . Berrangé]

On Thu, Mar 18, 2021 at 04:03:17PM +0100, Michal Privoznik wrote:
> On 3/18/21 9:00 AM, Tim Wiederhake wrote:
> > Convenience function to return value of a yes / no attribute.
> > 
> > Does not use virTristateBoolTypeFromString to disallow "default".
> > 
> > Signed-off-by: Tim Wiederhake 
> > ---
> >   src/util/virxml.c | 37 +
> >   src/util/virxml.h |  4 
> >   2 files changed, 41 insertions(+)
> > 
> > diff --git a/src/util/virxml.c b/src/util/virxml.c
> > index 060b7530fc..47e8414bd5 100644
> > --- a/src/util/virxml.c
> > +++ b/src/util/virxml.c
> > @@ -556,6 +556,43 @@ virXMLNodeContentString(xmlNodePtr node)
> >   }
> > +/**
> > + * virXMLPropYesNo:
> > + * @node: XML dom node pointer
> > + * @name: Name of the property (attribute) to get
> > + * @value: The returned virTristateBool value
> > + *
> > + * Convenience function to return value of a yes / no attribute.
> > + *
> > + * Returns 1 in case of success in which case @value is set,
> > + * or 0 if the attribute is not present,
> > + * or -1 and reports an error on failure.
> > + */
> > +int
> > +virXMLPropYesNo(xmlNodePtr node, const char* name, virTristateBool *value)

I kind of feel this should be called

   virXMLPropTristateBool


and the next patch virXMLPropTristateSwitch, so that they
both match their output variable types.

> > +{
> > +g_autofree char *tmp = virXMLPropString(node, name);
> > +
> > +if (!tmp)
> > +return 0;
> > +
> > +if (STREQ("yes", tmp)) {
> > +*value = VIR_TRISTATE_BOOL_YES;
> > +return 1;
> > +}
> > +
> > +if (STREQ("no", tmp)) {
> > +*value = VIR_TRISTATE_BOOL_NO;
> > +return 1;
> > +}
> > +
> > +virReportError(VIR_ERR_XML_ERROR,
> > +   _("Invalid value for attribute '%s' in node '%s': '%s'. 
> > Expected 'yes' or 'no'"),
> > +   name, node->name, tmp);
> > +return -1;
> 
> 
> How about:
> 
> int
> virXMLPropYesNo(xmlNodePtr node, const char* name, virTristateBool *value)
> {
> g_autofree char *tmp = virXMLPropString(node, name);
> int val;
> 
> if (!tmp)
> return 0;
> 
> if ((val = virTristateBoolTypeFromString(tmp)) <= 0) {
> virReportError(VIR_ERR_XML_ERROR,
>_("Invalid value for attribute '%s' in node '%s':
> '%s'. Expected 'yes' or 'no'"),
>name, node->name, tmp);
> return -1;
> }
> 
> *value = val;
> return 1;
> }

Yep, I think we really should do this, not open code the same
thing.


Regards,
Daniel
-- 
|: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o-https://fstop138.berrange.com :|
|: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|

Re: [libvirt PATCH 00/38] Refactor XML parsing boilerplate code

[Tim Wiederhake]

On Thu, 2021-03-18 at 16:03 +0100, Michal Privoznik wrote:
> On 3/18/21 9:00 AM, Tim Wiederhake wrote:
> > (...)
> >   10 files changed, 314 insertions(+), 663 deletions(-)
> > 
> 
> Reviewed-by: Michal Privoznik 
> 
> Nice cleanup!
> 
> And if you agree with my suggestions in 01-02/38 (suggestion from 02 
> affects also 03), I can squash the changes in and push.
> 
> Michal

Thanks for the review!

Don't push it yet, I found one more "virTristateBoolTypeFromString"
that can be changed and that I missed somehow. I will add that to the
series, rebase, incorporate your suggestions, and resend tomorrow.

My plan is to do something similar for the "int-like" attributes next.

Cheers,
Tim

Re: [PATCH v6 00/10] Configurable policy for handling deprecated interfaces

[Markus Armbruster]

This series needs fixups to not break --disable-system --disable-tools
builds.  It's not much (see appended diff), but enough to make me respin
it as v7.  I'll respin the pull request, too.


diff --git a/include/qapi/compat-policy.h b/include/qapi/compat-policy.h
index b8c6638156..1083f95122 100644
--- a/include/qapi/compat-policy.h
+++ b/include/qapi/compat-policy.h
@@ -17,4 +17,22 @@
 
 extern CompatPolicy compat_policy;
 
+/*
+ * Create a QObject input visitor for @obj for use with QMP
+ *
+ * This is like qobject_input_visitor_new(), except it obeys the
+ * policy for handling deprecated management interfaces set with
+ * -compat.
+ */
+Visitor *qobject_input_visitor_new_qmp(QObject *obj);
+
+/*
+ * Create a QObject output visitor for @obj for use with QMP
+ *
+ * This is like qobject_output_visitor_new(), except it obeys the
+ * policy for handling deprecated management interfaces set with
+ * -compat.
+ */
+Visitor *qobject_output_visitor_new_qmp(QObject **result);
+
 #endif
diff --git a/include/qapi/qobject-input-visitor.h 
b/include/qapi/qobject-input-visitor.h
index cbc54de4ac..8d69388810 100644
--- a/include/qapi/qobject-input-visitor.h
+++ b/include/qapi/qobject-input-visitor.h
@@ -15,6 +15,7 @@
 #ifndef QOBJECT_INPUT_VISITOR_H
 #define QOBJECT_INPUT_VISITOR_H
 
+#include "qapi/qapi-types-compat.h"
 #include "qapi/visitor.h"
 
 typedef struct QObjectInputVisitor QObjectInputVisitor;
@@ -58,14 +59,8 @@ typedef struct QObjectInputVisitor QObjectInputVisitor;
  */
 Visitor *qobject_input_visitor_new(QObject *obj);
 
-/*
- * Create a QObject input visitor for @obj for use with QMP
- *
- * This is like qobject_input_visitor_new(), except it obeys the
- * policy for handling deprecated management interfaces set with
- * -compat.
- */
-Visitor *qobject_input_visitor_new_qmp(QObject *obj);
+void qobject_input_visitor_set_policy(Visitor *v,
+  CompatPolicyInput deprecated);
 
 /*
  * Create a QObject input visitor for @obj for use with keyval_parse()
diff --git a/include/qapi/qobject-output-visitor.h 
b/include/qapi/qobject-output-visitor.h
index 29f4ea6aad..f2a2f92a00 100644
--- a/include/qapi/qobject-output-visitor.h
+++ b/include/qapi/qobject-output-visitor.h
@@ -15,6 +15,7 @@
 #define QOBJECT_OUTPUT_VISITOR_H
 
 #include "qapi/visitor.h"
+#include "qapi/qapi-types-compat.h"
 
 typedef struct QObjectOutputVisitor QObjectOutputVisitor;
 
@@ -53,13 +54,7 @@ typedef struct QObjectOutputVisitor QObjectOutputVisitor;
  */
 Visitor *qobject_output_visitor_new(QObject **result);
 
-/*
- * Create a QObject output visitor for @obj for use with QMP
- *
- * This is like qobject_output_visitor_new(), except it obeys the
- * policy for handling deprecated management interfaces set with
- * -compat.
- */
-Visitor *qobject_output_visitor_new_qmp(QObject **result);
+void qobject_output_visitor_set_policy(Visitor *v,
+   CompatPolicyOutput deprecated);
 
 #endif
diff --git a/qapi/qmp-dispatch.c b/qapi/qmp-dispatch.c
index 12657d635e..0f7e2e48fa 100644
--- a/qapi/qmp-dispatch.c
+++ b/qapi/qmp-dispatch.c
@@ -19,6 +19,8 @@
 #include "qapi/qmp/dispatch.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
+#include "qapi/qobject-input-visitor.h"
+#include "qapi/qobject-output-visitor.h"
 #include "sysemu/runstate.h"
 #include "qapi/qmp/qbool.h"
 #include "qemu/coroutine.h"
@@ -26,6 +28,22 @@
 
 CompatPolicy compat_policy;
 
+Visitor *qobject_input_visitor_new_qmp(QObject *obj)
+{
+Visitor *v = qobject_input_visitor_new(obj);
+
+qobject_input_visitor_set_policy(v, compat_policy.deprecated_input);
+return v;
+}
+
+Visitor *qobject_output_visitor_new_qmp(QObject **result)
+{
+Visitor *v = qobject_output_visitor_new(result);
+
+qobject_output_visitor_set_policy(v, compat_policy.deprecated_output);
+return v;
+}
+
 static QDict *qmp_dispatch_check_obj(QDict *dict, bool allow_oob,
  Error **errp)
 {
diff --git a/qapi/qobject-input-visitor.c b/qapi/qobject-input-visitor.c
index baad0dcd3c..04b790412e 100644
--- a/qapi/qobject-input-visitor.c
+++ b/qapi/qobject-input-visitor.c
@@ -739,12 +739,12 @@ Visitor *qobject_input_visitor_new(QObject *obj)
 return >visitor;
 }
 
-Visitor *qobject_input_visitor_new_qmp(QObject *obj)
+void qobject_input_visitor_set_policy(Visitor *v,
+   CompatPolicyInput deprecated)
 {
-QObjectInputVisitor *v = to_qiv(qobject_input_visitor_new(obj));
+QObjectInputVisitor *qiv = to_qiv(v);
 
-v->deprecated_policy = compat_policy.deprecated_input;
-return >visitor;
+qiv->deprecated_policy = deprecated;
 }
 
 Visitor *qobject_input_visitor_new_keyval(QObject *obj)
diff --git a/qapi/qobject-output-visitor.c b/qapi/qobject-output-visitor.c
index 5c4aa0f64d..e4873308d4 100644
--- a/qapi/qobject-output-visitor.c
+++ b/qapi/qobject-output-visitor.c
@@ -276,10 +276,10 @@ Visitor *qobject_output_visitor_new(QObject 

Re: [libvirt PATCH 02/38] virxml: Add virXMLPropYesNo

[Michal Privoznik]

On 3/18/21 9:00 AM, Tim Wiederhake wrote:

Convenience function to return value of a yes / no attribute.

Does not use virTristateBoolTypeFromString to disallow "default".

Signed-off-by: Tim Wiederhake 
---
  src/util/virxml.c | 37 +
  src/util/virxml.h |  4 
  2 files changed, 41 insertions(+)

diff --git a/src/util/virxml.c b/src/util/virxml.c
index 060b7530fc..47e8414bd5 100644
--- a/src/util/virxml.c
+++ b/src/util/virxml.c
@@ -556,6 +556,43 @@ virXMLNodeContentString(xmlNodePtr node)
  }
  
  
+/**

+ * virXMLPropYesNo:
+ * @node: XML dom node pointer
+ * @name: Name of the property (attribute) to get
+ * @value: The returned virTristateBool value
+ *
+ * Convenience function to return value of a yes / no attribute.
+ *
+ * Returns 1 in case of success in which case @value is set,
+ * or 0 if the attribute is not present,
+ * or -1 and reports an error on failure.
+ */
+int
+virXMLPropYesNo(xmlNodePtr node, const char* name, virTristateBool *value)
+{
+g_autofree char *tmp = virXMLPropString(node, name);
+
+if (!tmp)
+return 0;
+
+if (STREQ("yes", tmp)) {
+*value = VIR_TRISTATE_BOOL_YES;
+return 1;
+}
+
+if (STREQ("no", tmp)) {
+*value = VIR_TRISTATE_BOOL_NO;
+return 1;
+}
+
+virReportError(VIR_ERR_XML_ERROR,
+   _("Invalid value for attribute '%s' in node '%s': '%s'. Expected 
'yes' or 'no'"),
+   name, node->name, tmp);
+return -1;



How about:

int
virXMLPropYesNo(xmlNodePtr node, const char* name, virTristateBool *value)
{
g_autofree char *tmp = virXMLPropString(node, name);
int val;

if (!tmp)
return 0;

if ((val = virTristateBoolTypeFromString(tmp)) <= 0) {
virReportError(VIR_ERR_XML_ERROR,
   _("Invalid value for attribute '%s' in node 
'%s': '%s'. Expected 'yes' or 'no'"),

   name, node->name, tmp);
return -1;
}

*value = val;
return 1;
}


"default" which is VIR_TRISTATE_BOOL_ABSENT is explicitly set to 0, so 
that things like g_new0() initialize the value to _ABSENT.


Also, the function should be listed in private symbols we export:

diff --git i/src/libvirt_private.syms w/src/libvirt_private.syms
index 526dcee11a..85bebca23a 100644
--- i/src/libvirt_private.syms
+++ w/src/libvirt_private.syms
@@ -3545,6 +3545,7 @@ virXMLParseHelper;
 virXMLPickShellSafeComment;
 virXMLPropString;
 virXMLPropStringLimit;
+virXMLPropYesNo;
 virXMLSaveFile;
 virXMLValidateAgainstSchema;
 virXMLValidatorFree;

I'm surprised linked did not run into any issues.

No need to resend, I can fix it before pushing. The similar applies to 
the next patch.


Michal

Re: [libvirt PATCH 01/38] virStorageAdapterFCHost: Fix comment

[Michal Privoznik]

On 3/18/21 9:00 AM, Tim Wiederhake wrote:

Signed-off-by: Tim Wiederhake 
---
  src/conf/storage_adapter_conf.h | 2 +-
  1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/conf/storage_adapter_conf.h b/src/conf/storage_adapter_conf.h
index 4c7da7c8d9..93879acb6e 100644
--- a/src/conf/storage_adapter_conf.h
+++ b/src/conf/storage_adapter_conf.h
@@ -51,7 +51,7 @@ struct _virStorageAdapterFCHost {
  char *parent_fabric_wwn;
  char *wwnn;
  char *wwpn;
-int managed;/* enum virTristateSwitch */
+int managed;/* enum virTristateBool */
  };
  
  typedef struct _virStorageAdapter virStorageAdapter;




I'd expand the commit message a bit, like this:

virStorageAdapterFCHost: Fix comment for @managed

The enum that's used for @managed is virTristateBool and not
virTristateSwitch as the comment says.


Michal

Re: [libvirt PATCH 00/38] Refactor XML parsing boilerplate code

[Michal Privoznik]

On 3/18/21 9:00 AM, Tim Wiederhake wrote:

This series replaces some recurring boilerplate code in src/conf/ regarding
the extraction of a virTristate(Switch|Bool) XML attribute.

The boilerplate code looks roughly like this,

   g_autofree char *str = NULL;
   if (str = virXMLPropString(node, ...)) {
 int val;
 if ((val = virTristateBoolTypeFromString(str)) <= 0) {
   virReportError(...)
   return -1;
 }
 def->... = val;
   }

with some variations regarding how `str` is free'd in case of later re-use,
the exact error message for invalid values, whether or not
`VIR_TRISTATE_(SWITCH|BOOL)_ABSENT` is explicitly checked for (`val < 0` vs.
`val <= 0`), whether an intermediate variable is used or the value is assigned
directly, and in some cases the conditions in the two if-blocks are merged.

As a side effect, this makes the error messages for invalid values in these
attributes much more consistent and catches some instances where e.g.
`` would incorrectly be accepted.

Patch #11 (virDomainChrSourceReconnectDefParseXML) is a good example of what
this refactoring is about.

Tim Wiederhake (38):
   virStorageAdapterFCHost: Fix comment
   virxml: Add virXMLPropYesNo
   virxml: Add virXMLPropOnOff
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainKeyWrapCipherDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainVirtioOptionsParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainDeviceInfoParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainDiskSourceNetworkParse
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainDiskSourceNVMeParse
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainDiskDefDriverParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainActualNetDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainChrSourceReconnectDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainNetDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainChrSourceDefParseTCP
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainChrSourceDefParseFile
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainChrSourceDefParseLog
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainGraphicsDefParseXMLVNC
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainGraphicsDefParseXMLSDL
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainGraphicsDefParseXMLSpice
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainAudioCommonParse
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainAudioJackParse
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainAudioOSSParse
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainAudioDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainMemballoonDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainShmemDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainPerfEventDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainMemoryDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainIOMMUDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainVsockDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainFeaturesDefParse
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainLoaderDefParseXML
   domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainVcpuParse
   backup_conf: Use virXMLProp(OnOff|YesNo) in
 virDomainBackupDiskDefParseXML
   backup_conf: Use virXMLProp(OnOff|YesNo) in virDomainBackupDefParse
   device_conf: Use virXMLProp(OnOff|YesNo) in
 virPCIDeviceAddressParseXML
   network_conf: Use virXMLProp(OnOff|YesNo) in
 virNetworkForwardNatDefParseXML
   numa_conf: Use virXMLProp(OnOff|YesNo) in virDomainNumaDefParseXML
   storage_adapter_conf: Use virXMLProp(OnOff|YesNo) in
 virStorageAdapterParseXMLFCHost
   storage_conf: Use virXMLProp(OnOff|YesNo) in
 virStoragePoolDefParseSource

  src/conf/backup_conf.c  |  32 +-
  src/conf/device_conf.c  |  10 +-
  src/conf/domain_conf.c  | 791 +---
  src/conf/network_conf.c |  15 +-
  src/conf/numa_conf.c|  13 +-
  src/conf/storage_adapter_conf.c |  17 +-
  src/conf/storage_adapter_conf.h |   2 +-
  src/conf/storage_conf.c |  16 +-
  src/util/virxml.c   |  74 +++
  src/util/virxml.h   |   7 +
  10 files changed, 314 insertions(+), 663 deletions(-)



Reviewed-by: Michal Privoznik 

Nice cleanup!

And if you agree with my suggestions in 01-02/38 (suggestion from 02 
affects also 03), I can squash the changes in and push.


Michal

Re: [libvirt PATCH 06/38] domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainDeviceInfoParseXML

[Michal Privoznik]

On 3/18/21 9:00 AM, Tim Wiederhake wrote:

Signed-off-by: Tim Wiederhake 
---
  src/conf/domain_conf.c | 21 +
  1 file changed, 9 insertions(+), 12 deletions(-)



Had to rebase this, because meanwhile I pushed another patch that 
touched this area.



diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 3a4b01ad1e..e1b2baf621 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -6620,8 +6620,6 @@ virDomainDeviceInfoParseXML(virDomainXMLOptionPtr xmlopt,
  xmlNodePtr boot = NULL;
  xmlNodePtr rom = NULL;
  int ret = -1;
-g_autofree char *romenabled = NULL;
-g_autofree char *rombar = NULL;
  g_autofree char *aliasStr = NULL;
  
  virDomainDeviceInfoClear(info);

@@ -6673,18 +6671,17 @@ virDomainDeviceInfoParseXML(virDomainXMLOptionPtr 
xmlopt,
  }
  
  if (rom) {

-if ((romenabled = virXMLPropString(rom, "enabled")) &&
-((info->romenabled = virTristateBoolTypeFromString(romenabled)) <= 
0)) {
-virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("unknown rom enabled value '%s'"), romenabled);
+virTristateBool romenabled = VIR_TRISTATE_BOOL_ABSENT;
+virTristateSwitch rombar = VIR_TRISTATE_SWITCH_ABSENT;


Okay, so the reason these variables are here is because 
virXMLPropYesNo()/virXMLPropOnOff() expects 
virTristateBool/virTristateSwitch variable, but info->romenabled and 
info->rombar are ints. They had to be, because previously we stored 
virTristateXXXTypeFromString() retval - which is type of int - directly 
into them. But I guess after these patches are merged we can finally 
switch them to the proper type.


Same applies to the next patch (I haven't looked further). It's 
perfectly okay to do it in a follow up patch.



+
+if (virXMLPropYesNo(rom, "enabled", ) < 0)
  goto cleanup;
-}
-if ((rombar = virXMLPropString(rom, "bar")) &&
-((info->rombar = virTristateSwitchTypeFromString(rombar)) <= 0)) {
-virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("unknown rom bar value '%s'"), rombar);
+
+if (virXMLPropOnOff(rom, "bar", ) < 0)
  goto cleanup;
-}
+
+info->romenabled = romenabled;
+info->rombar = rombar;
  info->romfile = virXMLPropString(rom, "file");
  
  if (info->romenabled == VIR_TRISTATE_BOOL_NO &&





Michal

Re: [PATCH v2 0/2] XML non-virtio video device validation

[Michal Privoznik]

On 3/10/21 5:42 PM, Kristina Hanicova wrote:


Kristina Hanicova (2):
   move virDomainCheckVirtioOptionsAreAbsent a few lines forward
   XML validate that non-virtio video devices have none virtio options

  src/conf/domain_validate.c | 60 --
  1 file changed, 32 insertions(+), 28 deletions(-)



Reviewed-by: Michal Privoznik 

and pushed.

Michal

Re: [PATCH 0/2] refactoring conf for virDomainDeviceInfoParseXML to use XPath

[Michal Privoznik]

On 3/12/21 1:44 PM, Kristina Hanicova wrote:


Kristina Hanicova (2):
   conf: Propagate xmlXPathContextPtr into virDomainDeviceInfoParseXML()
   refactoring virDomainDeviceInfoParseXML to use XPath

  src/conf/domain_conf.c | 116 +
  1 file changed, 49 insertions(+), 67 deletions(-)



Reviewed-by: Michal Privoznik 

and pushed.

Michal

Re: [PATCH] vm : forbid to start a removing vm

[Michal Privoznik]

On 3/11/21 2:13 AM, Hogan Wang wrote:

From: Zhuang Shengen 

When a vm is doing migration phase confirm, and then start it concurrently,
it will lead to the vm out of libvirtd control.

Cause Analysis:
1. thread1 migrate vm out.
2. thread2 start the migrating vm.
3. thread1 remove vm from domain list after migrate success.


I guess I'm puzzled here. Thread1 has started migration or finished too? 
How can thread2 start the VM during migration? Is that an offline migration?


To me it seems like there's a problem with what jobs are allowed during 
migration, or their order. Looking at qemuMigrationSrcConfirm() which is 
the function that does final step of migration (at the source), so it 
ends migration job, then it starts MODIFY job (which potentially unlocks 
the domain object) and only then it sets vm->removing = true and removes 
it from the list.


What I am trying to say is that because the vm object can be briefly 
unlocked thus another thread might lock it successfully, proceed to 
BeginJob(), where it unlocks the object again only to let the first 
thread to remove it from the list.


It's not only virDomainCreateWithFlags() that's affected - basically any 
other API that grabs a job. Even though, plenty of them require domain 
to be running, therefore they exit early with "domain not running" error 
message.


I wonder whether something among these lines fixes the problem for you 
(compile tested only, but it shows the idea):


diff --git i/src/qemu/qemu_migration.c w/src/qemu/qemu_migration.c
index 79dcb4a15d..4b1bb77155 100644
--- i/src/qemu/qemu_migration.c
+++ w/src/qemu/qemu_migration.c
@@ -3468,6 +3468,7 @@ qemuMigrationSrcConfirm(virQEMUDriverPtr driver,
 qemuMigrationJobPhase phase;
 g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
 int ret = -1;
+bool haveJob;

 if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_OUT))
 goto cleanup;
@@ -3485,15 +3486,21 @@ qemuMigrationSrcConfirm(virQEMUDriverPtr driver,
cookiein, cookieinlen,
flags, cancelled);

-qemuMigrationJobFinish(driver, vm);
+haveJob = qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) >= 0;
+
 if (!virDomainObjIsActive(vm)) {
 if (!cancelled && ret == 0 && flags & 
VIR_MIGRATE_UNDEFINE_SOURCE) {

 virDomainDeleteConfig(cfg->configDir, cfg->autostartDir, vm);
 vm->persistent = 0;
 }
-qemuDomainRemoveInactiveJob(driver, vm);
+qemuDomainRemoveInactive(driver, vm);
 }

+if (haveJob)
+qemuDomainObjEndJob(driver, vm);
+
+qemuMigrationJobFinish(driver, vm);
+
  cleanup:
 virDomainObjEndAPI();
 return ret;




4. thread2 acquired the vm job success and start the vm.
5. cannot find the vm any more by 'virsh list' command. Actually,
the started vm is not exist in the domain list.

Solution:
Check the vm->removing state before start.


Signed-off-by: Zhuang Shengen 
Reviewed-by: Hogan Wang 
---
  src/qemu/qemu_driver.c | 6 ++
  1 file changed, 6 insertions(+)

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index d1a3659774..a5dfea94cb 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -6637,6 +6637,12 @@ qemuDomainCreateWithFlags(virDomainPtr dom, unsigned int 
flags)
  goto endjob;
  }
  
+if (vm->removing) {

+virReportError(VIR_ERR_OPERATION_INVALID,
+   "%s", _("domain is already removing"));


s/removing/being removed/


+goto endjob;
+}
+
  if (qemuDomainObjStart(dom->conn, driver, vm, flags,
 QEMU_ASYNC_JOB_START) < 0)
  goto endjob;



Michal

Re: [PATCH v2] bhyve: add support

[Michal Privoznik]

On 3/16/21 1:20 PM, Roman Bogorodskiy wrote:

Implement "" support for bhyve driver.
As there are not really lot of options, try to find
"BHYVE_UEFI.fd" firmware which is installed by the
sysutils/uefi-edk2-bhyve FreeBSD port.

If not found, just use the first found firmware
in the firmwares directory (which is configurable via
config file).

Signed-off-by: Roman Bogorodskiy 
---
Changes from v1:

  - Fixed various leaks,
  - Re-implemented testing using opendir() mock.

  po/POTFILES.in|  1 +
  src/bhyve/bhyve_domain.c  |  5 +
  src/bhyve/bhyve_firmware.c| 91 +++
  src/bhyve/bhyve_firmware.h| 30 ++
  src/bhyve/bhyve_process.c | 15 +++
  src/bhyve/bhyve_process.h |  5 +
  src/bhyve/bhyve_utils.h   |  2 +
  src/bhyve/meson.build |  1 +
  tests/bhyvefirmwaredata/empty/.keepme |  0
  .../three_firmwares/BHYVE_UEFI.fd |  0
  .../three_firmwares/BHYVE_UEFI_CSM.fd |  0
  .../three_firmwares/refind_x64.efi|  0
  .../bhyvexml2argv-firmware-efi.args   | 11 +++
  .../bhyvexml2argv-firmware-efi.ldargs |  1 +
  .../bhyvexml2argv-firmware-efi.xml| 22 +
  tests/bhyvexml2argvmock.c | 33 +++
  tests/bhyvexml2argvtest.c | 52 ---
  17 files changed, 257 insertions(+), 12 deletions(-)
  create mode 100644 src/bhyve/bhyve_firmware.c
  create mode 100644 src/bhyve/bhyve_firmware.h
  create mode 100644 tests/bhyvefirmwaredata/empty/.keepme
  create mode 100644 tests/bhyvefirmwaredata/three_firmwares/BHYVE_UEFI.fd
  create mode 100644 tests/bhyvefirmwaredata/three_firmwares/BHYVE_UEFI_CSM.fd
  create mode 100644 tests/bhyvefirmwaredata/three_firmwares/refind_x64.efi
  create mode 100644 tests/bhyvexml2argvdata/bhyvexml2argv-firmware-efi.args
  create mode 100644 tests/bhyvexml2argvdata/bhyvexml2argv-firmware-efi.ldargs
  create mode 100644 tests/bhyvexml2argvdata/bhyvexml2argv-firmware-efi.xml


Reviewed-by: Michal Privoznik 

Michal

[libvirt PATCH 7/9] conf: use switch in virDomainDefParseBootOptions

[Pavel Hrdina]

The original code used a lot of conditions and was not that obvious
when each XML bits are parsed.

Signed-off-by: Pavel Hrdina 
---
 src/conf/domain_conf.c | 42 ++
 1 file changed, 26 insertions(+), 16 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 14a2c818d6..7729333897 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -19702,33 +19702,43 @@ virDomainDefParseBootOptions(virDomainDefPtr def,
  *   - An init script (exe)
  */
 
-if (def->os.type == VIR_DOMAIN_OSTYPE_EXE) {
-if (virDomainDefParseBootInitOptions(def, ctxt) < 0)
-return -1;
-}
-
-if (def->os.type == VIR_DOMAIN_OSTYPE_XEN ||
-def->os.type == VIR_DOMAIN_OSTYPE_XENPVH ||
-def->os.type == VIR_DOMAIN_OSTYPE_HVM ||
-def->os.type == VIR_DOMAIN_OSTYPE_UML) {
-
+switch ((virDomainOSType) def->os.type) {
+case VIR_DOMAIN_OSTYPE_HVM:
 virDomainDefParseBootKernelOptions(def, ctxt);
 
-if (def->os.type == VIR_DOMAIN_OSTYPE_HVM) {
-if (virDomainDefParseBootFirmwareOptions(def, ctxt) < 0)
-return -1;
-}
+if (virDomainDefParseBootFirmwareOptions(def, ctxt) < 0)
+return -1;
 
 if (virDomainDefParseBootLoaderOptions(def, ctxt) < 0)
 return -1;
-}
 
-if (def->os.type == VIR_DOMAIN_OSTYPE_HVM) {
 if (virDomainDefParseBootAcpiOptions(def, ctxt) < 0)
 return -1;
 
 if (virDomainDefParseBootXML(ctxt, def) < 0)
 return -1;
+
+break;
+
+case VIR_DOMAIN_OSTYPE_XEN:
+case VIR_DOMAIN_OSTYPE_XENPVH:
+case VIR_DOMAIN_OSTYPE_UML:
+virDomainDefParseBootKernelOptions(def, ctxt);
+
+if (virDomainDefParseBootLoaderOptions(def, ctxt) < 0)
+return -1;
+
+break;
+
+case VIR_DOMAIN_OSTYPE_EXE:
+if (virDomainDefParseBootInitOptions(def, ctxt) < 0)
+return -1;
+
+break;
+
+case VIR_DOMAIN_OSTYPE_LINUX:
+case VIR_DOMAIN_OSTYPE_LAST:
+break;
 }
 
 return 0;
-- 
2.30.2

[libvirt PATCH 5/9] conf: introduce virDomainDefParseBootLoaderOptions

[Pavel Hrdina]

Extract the code to it's own function.

Signed-off-by: Pavel Hrdina 
---
 src/conf/domain_conf.c | 42 +++---
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 03985b6687..f6e8d5180a 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -19620,6 +19620,31 @@ virDomainDefParseBootFirmwareOptions(virDomainDefPtr 
def,
 }
 
 
+static int
+virDomainDefParseBootLoaderOptions(virDomainDefPtr def,
+   xmlXPathContextPtr ctxt)
+{
+xmlNodePtr loader_node = virXPathNode("./os/loader[1]", ctxt);
+const bool fwAutoSelect = def->os.firmware != 
VIR_DOMAIN_OS_DEF_FIRMWARE_NONE;
+
+if (!loader_node)
+return 0;
+
+def->os.loader = g_new0(virDomainLoaderDef, 1);
+
+if (virDomainLoaderDefParseXML(loader_node,
+   def->os.loader,
+   fwAutoSelect) < 0)
+return -1;
+
+def->os.loader->nvram = virXPathString("string(./os/nvram[1])", ctxt);
+if (!fwAutoSelect)
+def->os.loader->templt = 
virXPathString("string(./os/nvram[1]/@template)", ctxt);
+
+return 0;
+}
+
+
 static int
 virDomainDefParseBootOptions(virDomainDefPtr def,
  xmlXPathContextPtr ctxt)
@@ -19646,7 +19671,6 @@ virDomainDefParseBootOptions(virDomainDefPtr def,
 def->os.type == VIR_DOMAIN_OSTYPE_XENPVH ||
 def->os.type == VIR_DOMAIN_OSTYPE_HVM ||
 def->os.type == VIR_DOMAIN_OSTYPE_UML) {
-xmlNodePtr loader_node;
 
 virDomainDefParseBootKernelOptions(def, ctxt);
 
@@ -19655,20 +19679,8 @@ virDomainDefParseBootOptions(virDomainDefPtr def,
 return -1;
 }
 
-if ((loader_node = virXPathNode("./os/loader[1]", ctxt))) {
-const bool fwAutoSelect = def->os.firmware != 
VIR_DOMAIN_OS_DEF_FIRMWARE_NONE;
-
-def->os.loader = g_new0(virDomainLoaderDef, 1);
-
-if (virDomainLoaderDefParseXML(loader_node,
-   def->os.loader,
-   fwAutoSelect) < 0)
-return -1;
-
-def->os.loader->nvram = virXPathString("string(./os/nvram[1])", 
ctxt);
-if (!fwAutoSelect)
-def->os.loader->templt = 
virXPathString("string(./os/nvram[1]/@template)", ctxt);
-}
+if (virDomainDefParseBootLoaderOptions(def, ctxt) < 0)
+return -1;
 }
 
 if (def->os.type == VIR_DOMAIN_OSTYPE_HVM) {
-- 
2.30.2

[libvirt PATCH 4/9] conf: introduce virDomainDefParseBootFirmwareOptions

[Pavel Hrdina]

Extract the code to it's own function.

Signed-off-by: Pavel Hrdina 
---
 src/conf/domain_conf.c | 39 +++
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 4876fe61bb..03985b6687 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -19595,6 +19595,31 @@ virDomainDefParseBootKernelOptions(virDomainDefPtr def,
 }
 
 
+static int
+virDomainDefParseBootFirmwareOptions(virDomainDefPtr def,
+ xmlXPathContextPtr ctxt)
+{
+g_autofree char *firmware = virXPathString("string(./os/@firmware)", ctxt);
+int fw = 0;
+
+if (!firmware)
+return 0;
+
+fw = virDomainOsDefFirmwareTypeFromString(firmware);
+
+if (fw <= 0) {
+virReportError(VIR_ERR_XML_ERROR,
+   _("unknown firmware value %s"),
+   firmware);
+return -1;
+}
+
+def->os.firmware = fw;
+
+return 0;
+}
+
+
 static int
 virDomainDefParseBootOptions(virDomainDefPtr def,
  xmlXPathContextPtr ctxt)
@@ -19621,23 +19646,13 @@ virDomainDefParseBootOptions(virDomainDefPtr def,
 def->os.type == VIR_DOMAIN_OSTYPE_XENPVH ||
 def->os.type == VIR_DOMAIN_OSTYPE_HVM ||
 def->os.type == VIR_DOMAIN_OSTYPE_UML) {
-g_autofree char *firmware = NULL;
 xmlNodePtr loader_node;
 
 virDomainDefParseBootKernelOptions(def, ctxt);
 
-if (def->os.type == VIR_DOMAIN_OSTYPE_HVM &&
-(firmware = virXPathString("string(./os/@firmware)", ctxt))) {
-int fw = virDomainOsDefFirmwareTypeFromString(firmware);
-
-if (fw <= 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("unknown firmware value %s"),
-   firmware);
+if (def->os.type == VIR_DOMAIN_OSTYPE_HVM) {
+if (virDomainDefParseBootFirmwareOptions(def, ctxt) < 0)
 return -1;
-}
-
-def->os.firmware = fw;
 }
 
 if ((loader_node = virXPathNode("./os/loader[1]", ctxt))) {
-- 
2.30.2

[libvirt PATCH 6/9] conf: introduce virDomainDefParseBootAcpiOptions

[Pavel Hrdina]

Extract the code to it's own function.

Signed-off-by: Pavel Hrdina 
---
 src/conf/domain_conf.c | 79 --
 1 file changed, 45 insertions(+), 34 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index f6e8d5180a..14a2c818d6 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -19645,14 +19645,54 @@ virDomainDefParseBootLoaderOptions(virDomainDefPtr 
def,
 }
 
 
+static int
+virDomainDefParseBootAcpiOptions(virDomainDefPtr def,
+ xmlXPathContextPtr ctxt)
+{
+int n;
+g_autofree xmlNodePtr *nodes = NULL;
+g_autofree char *tmp = NULL;
+
+if ((n = virXPathNodeSet("./os/acpi/table", ctxt, )) < 0)
+return -1;
+
+if (n > 1) {
+virReportError(VIR_ERR_XML_ERROR, "%s",
+   _("Only one acpi table is supported"));
+return -1;
+}
+
+if (n == 1) {
+tmp = virXMLPropString(nodes[0], "type");
+
+if (!tmp) {
+virReportError(VIR_ERR_XML_ERROR, "%s",
+   _("Missing acpi table type"));
+return -1;
+}
+
+if (STREQ_NULLABLE(tmp, "slic")) {
+VIR_FREE(tmp);
+if (!(tmp = virXMLNodeContentString(nodes[0])))
+return -1;
+
+def->os.slic_table = virFileSanitizePath(tmp);
+} else {
+virReportError(VIR_ERR_XML_ERROR,
+   _("Unknown acpi table type: %s"),
+   tmp);
+return -1;
+}
+}
+
+return 0;
+}
+
+
 static int
 virDomainDefParseBootOptions(virDomainDefPtr def,
  xmlXPathContextPtr ctxt)
 {
-int n;
-g_autofree xmlNodePtr *nodes = NULL;
-g_autofree char *tmp = NULL;
-
 /*
  * Booting options for different OS types
  *
@@ -19684,38 +19724,9 @@ virDomainDefParseBootOptions(virDomainDefPtr def,
 }
 
 if (def->os.type == VIR_DOMAIN_OSTYPE_HVM) {
-if ((n = virXPathNodeSet("./os/acpi/table", ctxt, )) < 0)
+if (virDomainDefParseBootAcpiOptions(def, ctxt) < 0)
 return -1;
 
-if (n > 1) {
-virReportError(VIR_ERR_XML_ERROR, "%s",
-   _("Only one acpi table is supported"));
-return -1;
-}
-
-if (n == 1) {
-tmp = virXMLPropString(nodes[0], "type");
-
-if (!tmp) {
-virReportError(VIR_ERR_XML_ERROR, "%s",
-   _("Missing acpi table type"));
-return -1;
-}
-
-if (STREQ_NULLABLE(tmp, "slic")) {
-VIR_FREE(tmp);
-if (!(tmp = virXMLNodeContentString(nodes[0])))
-return -1;
-
-def->os.slic_table = virFileSanitizePath(tmp);
-} else {
-virReportError(VIR_ERR_XML_ERROR,
-   _("Unknown acpi table type: %s"),
-   tmp);
-return -1;
-}
-}
-
 if (virDomainDefParseBootXML(ctxt, def) < 0)
 return -1;
 }
-- 
2.30.2

[libvirt PATCH 9/9] qemu: implement support for firmware auto-selection feature filtering

[Pavel Hrdina]

Signed-off-by: Pavel Hrdina 
---
 src/qemu/qemu_firmware.c  | 40 +++
 ...re-efi-no-enrolled-keys.x86_64-latest.args | 49 ++
 .../os-firmware-efi-no-enrolled-keys.xml  | 25 ++
 tests/qemuxml2argvtest.c  |  1 +
 ...are-efi-no-enrolled-keys.x86_64-latest.xml | 50 +++
 tests/qemuxml2xmltest.c   |  1 +
 6 files changed, 166 insertions(+)
 create mode 100644 
tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args
 create mode 100644 tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.xml
 create mode 100644 
tests/qemuxml2xmloutdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.xml

diff --git a/src/qemu/qemu_firmware.c b/src/qemu/qemu_firmware.c
index d3198e2d45..f6f371f51f 100644
--- a/src/qemu/qemu_firmware.c
+++ b/src/qemu/qemu_firmware.c
@@ -930,6 +930,10 @@ qemuFirmwareMatchDomain(const virDomainDef *def,
 bool supportsS4 = false;
 bool requiresSMM = false;
 bool supportsSEV = false;
+bool supportsSecureBoot = false;
+bool hasEnrolledKeys = false;
+int reqSecureBoot;
+int reqEnrolledKeys;
 
 want = qemuFirmwareOSInterfaceTypeFromOsDefFirmware(def->os.firmware);
 
@@ -979,7 +983,13 @@ qemuFirmwareMatchDomain(const virDomainDef *def,
 break;
 
 case QEMU_FIRMWARE_FEATURE_SECURE_BOOT:
+supportsSecureBoot = true;
+break;
+
 case QEMU_FIRMWARE_FEATURE_ENROLLED_KEYS:
+hasEnrolledKeys = true;
+break;
+
 case QEMU_FIRMWARE_FEATURE_VERBOSE_DYNAMIC:
 case QEMU_FIRMWARE_FEATURE_VERBOSE_STATIC:
 case QEMU_FIRMWARE_FEATURE_NONE:
@@ -1000,6 +1010,36 @@ qemuFirmwareMatchDomain(const virDomainDef *def,
 return false;
 }
 
+if (def->os.firmwareFeatures) {
+reqSecureBoot = 
def->os.firmwareFeatures[VIR_DOMAIN_OS_DEF_FIRMWARE_FEATURE_SECURE_BOOT];
+if (reqSecureBoot != VIR_TRISTATE_BOOL_ABSENT) {
+if (reqSecureBoot == VIR_TRISTATE_BOOL_YES && !supportsSecureBoot) 
{
+VIR_DEBUG("User requested Secure Boot, firmware '%s' doesn't 
support it",
+  path);
+return false;
+}
+
+if (reqSecureBoot == VIR_TRISTATE_BOOL_NO && supportsSecureBoot) {
+VIR_DEBUG("User refused Secure Boot, firmware '%s' supports 
it", path);
+return false;
+}
+}
+
+reqEnrolledKeys = 
def->os.firmwareFeatures[VIR_DOMAIN_OS_DEF_FIRMWARE_FEATURE_ENROLLED_KEYS];
+if (reqEnrolledKeys != VIR_TRISTATE_BOOL_ABSENT) {
+if (reqEnrolledKeys == VIR_TRISTATE_BOOL_YES && !hasEnrolledKeys) {
+VIR_DEBUG("User requested Enrolled keys, firmware '%s' doesn't 
support it",
+  path);
+return false;
+}
+
+if (reqEnrolledKeys == VIR_TRISTATE_BOOL_NO && hasEnrolledKeys) {
+VIR_DEBUG("User refused Enrolled keys, firmware '%s' supports 
it", path);
+return false;
+}
+}
+}
+
 if (def->os.loader &&
 def->os.loader->secure == VIR_TRISTATE_BOOL_YES &&
 !requiresSMM) {
diff --git 
a/tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args 
b/tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args
new file mode 100644
index 00..561a905e78
--- /dev/null
+++ b/tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args
@@ -0,0 +1,49 @@
+LC_ALL=C \
+PATH=/bin \
+HOME=/tmp/lib/domain--1-fedora \
+USER=test \
+LOGNAME=test \
+XDG_DATA_HOME=/tmp/lib/domain--1-fedora/.local/share \
+XDG_CACHE_HOME=/tmp/lib/domain--1-fedora/.cache \
+XDG_CONFIG_HOME=/tmp/lib/domain--1-fedora/.config \
+/usr/bin/qemu-system-x86_64 \
+-name guest=fedora,debug-threads=on \
+-S \
+-object secret,id=masterKey0,format=raw,\
+file=/tmp/lib/domain--1-fedora/master-key.aes \
+-blockdev '{"driver":"file","filename":"/usr/share/OVMF/OVMF_CODE.fd",\
+"node-name":"libvirt-pflash0-storage","auto-read-only":true,\
+"discard":"unmap"}' \
+-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,\
+"driver":"raw","file":"libvirt-pflash0-storage"}' \
+-blockdev '{"driver":"file",\
+"filename":"/var/lib/libvirt/qemu/nvram/fedora_VARS.fd",\
+"node-name":"libvirt-pflash1-storage","auto-read-only":true,\
+"discard":"unmap"}' \
+-blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,\
+"driver":"raw","file":"libvirt-pflash1-storage"}' \
+-machine pc-q35-4.0,accel=kvm,usb=off,dump-guest-core=off,\
+pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,\
+memory-backend=pc.ram \
+-cpu qemu64 \
+-m 8 \
+-object memory-backend-ram,id=pc.ram,size=8388608 \
+-overcommit mem-lock=off \
+-smp 1,sockets=1,cores=1,threads=1 \
+-uuid 63840878-0deb-4095-97e6-fc444d9bc9fa \
+-display none \
+-no-user-config \
+-nodefaults \
+-chardev 

[libvirt PATCH 8/9] conf: introduce support for firmware auto-selection feature filtering

[Pavel Hrdina]

When the firmware auto-selection was introduced it always picked first
usable firmware based on the JSON descriptions on the host. It is
possible to add/remove/change the JSON files but it will always be for
the whole host.

This patch introduces support for configuring the auto-selection per VM
by adding users an option to limit what features they would like to have
available in the firmware.

Signed-off-by: Pavel Hrdina 
---
 docs/formatdomain.rst | 31 +++
 docs/schemas/domaincommon.rng | 23 +
 src/conf/domain_conf.c| 83 ++-
 src/conf/domain_conf.h| 10 +++
 .../os-firmware-efi-invalid-type.xml  | 28 +++
 ...os-firmware-invalid-type.x86_64-latest.err |  1 +
 .../os-firmware-invalid-type.xml  | 28 +++
 tests/qemuxml2argvtest.c  |  1 +
 ...aarch64-os-firmware-efi.aarch64-latest.xml |  1 +
 .../os-firmware-bios.x86_64-latest.xml|  1 +
 .../os-firmware-efi-secboot.x86_64-latest.xml |  1 +
 .../os-firmware-efi.x86_64-latest.xml |  1 +
 tests/vmx2xmldata/vmx2xml-firmware-efi.xml|  1 +
 13 files changed, 207 insertions(+), 3 deletions(-)
 create mode 100644 tests/qemuxml2argvdata/os-firmware-efi-invalid-type.xml
 create mode 100644 
tests/qemuxml2argvdata/os-firmware-invalid-type.x86_64-latest.err
 create mode 100644 tests/qemuxml2argvdata/os-firmware-invalid-type.xml

diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
index c101d5a1f1..dd063b0794 100644
--- a/docs/formatdomain.rst
+++ b/docs/formatdomain.rst
@@ -155,6 +155,37 @@ harddisk, cdrom, network) determining where to obtain/find 
the boot image.
the host native arch will be chosen. For the ``test``, ``ESX`` and 
``VMWare``
hypervisor drivers, however, the ``i686`` arch will always be chosen even on
an ``x86_64`` host. :since:`Since 0.0.1`
+``firmware``
+   :since:`Since 7.2.0 QEMU/KVM only`
+
+   When used together with ``firmware`` attribute of ``os`` element the 
``type``
+   attribute must have the same value.
+
+   List of mandatory attributes:
+
+   - ``type`` (accepted values are ``bios`` and ``efi``) same as the 
``firmware``
+ attribute of ``os`` element.
+
+   When using firmware auto-selection there are different features enabled in
+   the firmwares. The list of features can be used to limit what firmware 
should
+   be automatically selected for the VM. The list of features can be specified
+   using zero or more ``feature`` elements. Libvirt will take into 
consideration
+   only the listed features and ignore the rest when selecting the firmware.
+
+   ``feature``
+  The list of mandatory attributes:
+
+  - ``enabled`` (accepted values are ``yes`` and ``no``) is used to tell 
libvirt
+if the feature must be enabled or not in the automatically selected 
firmware
+
+  - ``name`` the name of the feature, the list of the features:
+
+- ``enrolled-keys`` whether the selected nvram template has default
+  certificate enrolled. Firmware with Secure Boot feature but without
+  enrolled keys will successfully boot non-signed binaries as well.
+  Valid only for firmwares with Secure Boot feature.
+
+- ``secure-boot`` whether the firmware implements UEFI Secure boot 
feature.
 ``loader``
The optional ``loader`` tag refers to a firmware blob, which is specified by
absolute path, used to assist the domain creation process. It is used by Xen
diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng
index e6db2f5b74..1dbfc68f18 100644
--- a/docs/schemas/domaincommon.rng
+++ b/docs/schemas/domaincommon.rng
@@ -276,6 +276,29 @@
   
 
 
+
+  
+
+  
+bios
+efi
+  
+
+
+  
+
+  
+
+
+  
+enrolled-keys
+secure-boot
+  
+
+  
+
+  
+
 
   
 
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 7729333897..dcfe5c0d03 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -1318,6 +1318,12 @@ VIR_ENUM_IMPL(virDomainOsDefFirmware,
   "efi",
 );
 
+VIR_ENUM_IMPL(virDomainOsDefFirmwareFeature,
+  VIR_DOMAIN_OS_DEF_FIRMWARE_FEATURE_LAST,
+  "enrolled-keys",
+  "secure-boot",
+);
+
 VIR_ENUM_IMPL(virDomainCFPC,
   VIR_DOMAIN_CFPC_LAST,
   "none",
@@ -19600,22 +19606,67 @@ virDomainDefParseBootFirmwareOptions(virDomainDefPtr 
def,
  xmlXPathContextPtr ctxt)
 {
 g_autofree char *firmware = virXPathString("string(./os/@firmware)", ctxt);
+g_autofree char *type = 

[libvirt PATCH 3/9] conf: introduce virDomainDefParseBootKernelOptions

[Pavel Hrdina]

Extract the code to it's own function.

Signed-off-by: Pavel Hrdina 
---
 src/conf/domain_conf.c | 18 +-
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 31b908d8fe..4876fe61bb 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -19583,6 +19583,18 @@ virDomainDefParseBootInitOptions(virDomainDefPtr def,
 }
 
 
+static void
+virDomainDefParseBootKernelOptions(virDomainDefPtr def,
+   xmlXPathContextPtr ctxt)
+{
+def->os.kernel = virXPathString("string(./os/kernel[1])", ctxt);
+def->os.initrd = virXPathString("string(./os/initrd[1])", ctxt);
+def->os.cmdline = virXPathString("string(./os/cmdline[1])", ctxt);
+def->os.dtb = virXPathString("string(./os/dtb[1])", ctxt);
+def->os.root = virXPathString("string(./os/root[1])", ctxt);
+}
+
+
 static int
 virDomainDefParseBootOptions(virDomainDefPtr def,
  xmlXPathContextPtr ctxt)
@@ -19612,11 +19624,7 @@ virDomainDefParseBootOptions(virDomainDefPtr def,
 g_autofree char *firmware = NULL;
 xmlNodePtr loader_node;
 
-def->os.kernel = virXPathString("string(./os/kernel[1])", ctxt);
-def->os.initrd = virXPathString("string(./os/initrd[1])", ctxt);
-def->os.cmdline = virXPathString("string(./os/cmdline[1])", ctxt);
-def->os.dtb = virXPathString("string(./os/dtb[1])", ctxt);
-def->os.root = virXPathString("string(./os/root[1])", ctxt);
+virDomainDefParseBootKernelOptions(def, ctxt);
 
 if (def->os.type == VIR_DOMAIN_OSTYPE_HVM &&
 (firmware = virXPathString("string(./os/@firmware)", ctxt))) {
-- 
2.30.2

[libvirt PATCH 2/9] conf: introduce virDomainDefParseBootInitOptions

[Pavel Hrdina]

Extract the code to it's own function.

Signed-off-by: Pavel Hrdina 
---
 src/conf/domain_conf.c | 109 +++--
 1 file changed, 61 insertions(+), 48 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 47756ff0be..31b908d8fe 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -19523,12 +19523,70 @@ virDomainVcpuParse(virDomainDefPtr def,
 }
 
 
+static int
+virDomainDefParseBootInitOptions(virDomainDefPtr def,
+ xmlXPathContextPtr ctxt)
+{
+char *name = NULL;
+size_t i;
+int n;
+g_autofree xmlNodePtr *nodes = NULL;
+
+def->os.init = virXPathString("string(./os/init[1])", ctxt);
+def->os.cmdline = virXPathString("string(./os/cmdline[1])", ctxt);
+def->os.initdir = virXPathString("string(./os/initdir[1])", ctxt);
+def->os.inituser = virXPathString("string(./os/inituser[1])", ctxt);
+def->os.initgroup = virXPathString("string(./os/initgroup[1])", ctxt);
+
+if ((n = virXPathNodeSet("./os/initarg", ctxt, )) < 0)
+return -1;
+
+def->os.initargv = g_new0(char *, n+1);
+for (i = 0; i < n; i++) {
+if (!nodes[i]->children ||
+!nodes[i]->children->content) {
+virReportError(VIR_ERR_XML_ERROR, "%s",
+   _("No data supplied for  element"));
+return -1;
+}
+def->os.initargv[i] = g_strdup((const char 
*)nodes[i]->children->content);
+}
+def->os.initargv[n] = NULL;
+VIR_FREE(nodes);
+
+if ((n = virXPathNodeSet("./os/initenv", ctxt, )) < 0)
+return -1;
+
+def->os.initenv = g_new0(virDomainOSEnvPtr, n+1);
+for (i = 0; i < n; i++) {
+if (!(name = virXMLPropString(nodes[i], "name"))) {
+virReportError(VIR_ERR_XML_ERROR, "%s",
+   _("No name supplied for  element"));
+return -1;
+}
+
+if (!nodes[i]->children ||
+!nodes[i]->children->content) {
+virReportError(VIR_ERR_XML_ERROR,
+   _("No value supplied for  
element"),
+   name);
+return -1;
+}
+
+def->os.initenv[i] = g_new0(virDomainOSEnv, 1);
+def->os.initenv[i]->name = name;
+def->os.initenv[i]->value = g_strdup((const char 
*)nodes[i]->children->content);
+}
+def->os.initenv[n] = NULL;
+
+return 0;
+}
+
+
 static int
 virDomainDefParseBootOptions(virDomainDefPtr def,
  xmlXPathContextPtr ctxt)
 {
-char *name = NULL;
-size_t i;
 int n;
 g_autofree xmlNodePtr *nodes = NULL;
 g_autofree char *tmp = NULL;
@@ -19543,53 +19601,8 @@ virDomainDefParseBootOptions(virDomainDefPtr def,
  */
 
 if (def->os.type == VIR_DOMAIN_OSTYPE_EXE) {
-def->os.init = virXPathString("string(./os/init[1])", ctxt);
-def->os.cmdline = virXPathString("string(./os/cmdline[1])", ctxt);
-def->os.initdir = virXPathString("string(./os/initdir[1])", ctxt);
-def->os.inituser = virXPathString("string(./os/inituser[1])", ctxt);
-def->os.initgroup = virXPathString("string(./os/initgroup[1])", ctxt);
-
-if ((n = virXPathNodeSet("./os/initarg", ctxt, )) < 0)
+if (virDomainDefParseBootInitOptions(def, ctxt) < 0)
 return -1;
-
-def->os.initargv = g_new0(char *, n+1);
-for (i = 0; i < n; i++) {
-if (!nodes[i]->children ||
-!nodes[i]->children->content) {
-virReportError(VIR_ERR_XML_ERROR, "%s",
-   _("No data supplied for  element"));
-return -1;
-}
-def->os.initargv[i] = g_strdup((const char 
*)nodes[i]->children->content);
-}
-def->os.initargv[n] = NULL;
-VIR_FREE(nodes);
-
-if ((n = virXPathNodeSet("./os/initenv", ctxt, )) < 0)
-return -1;
-
-def->os.initenv = g_new0(virDomainOSEnvPtr, n+1);
-for (i = 0; i < n; i++) {
-if (!(name = virXMLPropString(nodes[i], "name"))) {
-virReportError(VIR_ERR_XML_ERROR, "%s",
-_("No name supplied for  element"));
-return -1;
-}
-
-if (!nodes[i]->children ||
-!nodes[i]->children->content) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("No value supplied for  
element"),
-   name);
-return -1;
-}
-
-def->os.initenv[i] = g_new0(virDomainOSEnv, 1);
-def->os.initenv[i]->name = name;
-def->os.initenv[i]->value = g_strdup((const char 
*)nodes[i]->children->content);
-}
-def->os.initenv[n] = NULL;
-VIR_FREE(nodes);
 }
 
 if (def->os.type == VIR_DOMAIN_OSTYPE_XEN ||
-- 
2.30.2

[libvirt PATCH 0/9] improve firmware auto-selection

[Pavel Hrdina]

Cleanup the parser code and implement firmware feature filtering support
to allow users to modify firmware auto-selection behavior per VM.

More details in PATCH 08.

Pavel Hrdina (9):
  docs: improve description of secure attribute for loader element
  conf: introduce virDomainDefParseBootInitOptions
  conf: introduce virDomainDefParseBootKernelOptions
  conf: introduce virDomainDefParseBootFirmwareOptions
  conf: introduce virDomainDefParseBootLoaderOptions
  conf: introduce virDomainDefParseBootAcpiOptions
  conf: use switch in virDomainDefParseBootOptions
  conf: introduce support for firmware auto-selection feature filtering
  qemu: implement support for firmware auto-selection feature filtering

 docs/formatdomain.rst |  35 +-
 docs/schemas/domaincommon.rng |  23 +
 src/conf/domain_conf.c| 396 --
 src/conf/domain_conf.h|  10 +
 src/qemu/qemu_firmware.c  |  40 ++
 .../os-firmware-efi-invalid-type.xml  |  28 ++
 ...re-efi-no-enrolled-keys.x86_64-latest.args |  49 +++
 .../os-firmware-efi-no-enrolled-keys.xml  |  25 ++
 ...os-firmware-invalid-type.x86_64-latest.err |   1 +
 .../os-firmware-invalid-type.xml  |  28 ++
 tests/qemuxml2argvtest.c  |   2 +
 ...aarch64-os-firmware-efi.aarch64-latest.xml |   1 +
 .../os-firmware-bios.x86_64-latest.xml|   1 +
 ...are-efi-no-enrolled-keys.x86_64-latest.xml |  50 +++
 .../os-firmware-efi-secboot.x86_64-latest.xml |   1 +
 .../os-firmware-efi.x86_64-latest.xml |   1 +
 tests/qemuxml2xmltest.c   |   1 +
 tests/vmx2xmldata/vmx2xml-firmware-efi.xml|   1 +
 18 files changed, 567 insertions(+), 126 deletions(-)
 create mode 100644 tests/qemuxml2argvdata/os-firmware-efi-invalid-type.xml
 create mode 100644 
tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.args
 create mode 100644 tests/qemuxml2argvdata/os-firmware-efi-no-enrolled-keys.xml
 create mode 100644 
tests/qemuxml2argvdata/os-firmware-invalid-type.x86_64-latest.err
 create mode 100644 tests/qemuxml2argvdata/os-firmware-invalid-type.xml
 create mode 100644 
tests/qemuxml2xmloutdata/os-firmware-efi-no-enrolled-keys.x86_64-latest.xml

-- 
2.30.2

[libvirt PATCH 1/9] docs: improve description of secure attribute for loader element

[Pavel Hrdina]

The original text was not explaining what this attribute actually
controls and could have been interpreted as a control switch for the
Secure boot feature in firmwares.

Signed-off-by: Pavel Hrdina 
---
 docs/formatdomain.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/formatdomain.rst b/docs/formatdomain.rst
index a2ea2690a5..c101d5a1f1 100644
--- a/docs/formatdomain.rst
+++ b/docs/formatdomain.rst
@@ -167,7 +167,9 @@ harddisk, cdrom, network) determining where to obtain/find 
the boot image.
in the guest memory the file should be mapped. For instance, if the loader
path points to an UEFI image, ``type`` should be ``pflash``. Moreover, some
firmwares may implement the Secure boot feature. Attribute ``secure`` can be
-   used then to control it. :since:`Since 2.1.0`
+   used to tell the hypervisor that the firmware implements Secure Boot 
Feature.
+   It cannot be used to enable or disable the feature itself in the firmware.
+   :since:`Since 2.1.0`
 ``nvram``
Some UEFI firmwares may want to use a non-volatile memory to store some
variables. In the host, this is represented as a file and the absolute path
-- 
2.30.2

Re: [libvirt PATCH] nodedev: Don't crash when exiting before init is done

[Michal Privoznik]

On 3/16/21 11:27 PM, Jonathon Jongsma wrote:

If libvirtd is terminated before the node driver finishes
initialization, it can crash with a backtrace similar to the following:

 Stack trace of thread 1922933:
 #0  0x7f8515178774 g_hash_table_find (libglib-2.0.so.0)
 #1  0x7f851593ea98 virHashSearch (libvirt.so.0)
 #2  0x7f8515a1dd83 virNodeDeviceObjListSearch (libvirt.so.0)
 #3  0x7f84cceb40a1 udevAddOneDevice (libvirt_driver_nodedev.so)
 #4  0x7f84cceb5fae nodeStateInitializeEnumerate 
(libvirt_driver_nodedev.so)
 #5  0x7f85159840cb virThreadHelper (libvirt.so.0)
 #6  0x7f8511c7d14a start_thread (libpthread.so.0)
 #7  0x7f851442bdb3 __clone (libc.so.6)

 Stack trace of thread 1922863:
 #0  0x7f851442651d syscall (libc.so.6)
 #1  0x7f85159842d4 virThreadSelfID (libvirt.so.0)
 #2  0x7f851594e240 virLogFormatString (libvirt.so.0)
 #3  0x7f851596635d vir_object_finalize (libvirt.so.0)
 #4  0x7f8514efe8e9 g_object_unref (libgobject-2.0.so.0)
 #5  0x7f85159667f8 virObjectUnref (libvirt.so.0)
 #6  0x7f851517755f g_hash_table_remove_all_nodes.part.0 
(libglib-2.0.so.0)
 #7  0x7f8515177e62 g_hash_table_unref (libglib-2.0.so.0)
 #8  0x7f851596637e vir_object_finalize (libvirt.so.0)
 #9  0x7f8514efe8e9 g_object_unref (libgobject-2.0.so.0)
 #10 0x7f85159667f8 virObjectUnref (libvirt.so.0)
 #11 0x7f84cceb2b42 nodeStateCleanup (libvirt_driver_nodedev.so)
 #12 0x7f8515b37950 virStateCleanup (libvirt.so.0)
 #13 0x5648085348e8 main (libvirtd)
 #14 0x7f8514352493 __libc_start_main (libc.so.6)
 #15 0x5648085350fe _start (libvirtd)

This is because the initial population of the device list is done in a
separate initialization thread. If we attempt to exit libvirtd before
this init thread has completed, we'll try to free the device list while
accessing it from the other thread. In order to guarantee that this
init thread is not accessing the device list when we're cleaning up the
nodedev driver, make it joinable and wait for it to finish before
proceding with the cleanup. This is similar to how we handle the udev
event handler thread.

The separate initialization thread was added in commit
9f0ae0b1.

https://bugzilla.redhat.com/show_bug.cgi?id=1933590


Since this bug was closed as a duplicate of 1836865, I'm replacing this 
line. Hope you don't mind.




Signed-off-by: Jonathon Jongsma 
---
  src/node_device/node_device_udev.c | 7 +--
  1 file changed, 5 insertions(+), 2 deletions(-)


Reviewed-by: Michal Privoznik 

And pushed.

Michal

Re: [PATCH] virxml: Fix possible memory leak in virXMLNodeContentString()

[Michal Privoznik]

On 3/17/21 6:01 PM, Kristina Hanicova wrote:

Previously, if xml node passed to the virXMLNodeContentString()
was not of type XML_ELEMENT_NODE, @ret could have caused a memory
leak because xmlNodeGetContent() works for other types of nodes
as well.

Signed-off-by: Kristina Hanicova 
---
  src/util/virxml.c | 4 +++-
  1 file changed, 3 insertions(+), 1 deletion(-)



Reviewed-by: Michal Privoznik 

Nice catch! Pushed.

Michal

Re: [libvirt PATCH v3 6/6] ci: helper: Convert private methods to Python common naming practice

[Andrea Bolognani]

On Thu, 2021-03-18 at 09:09 +0100, Erik Skultety wrote:
> As documented at [1], the common practice wrt to private

s/wrt to/with respect to/

>  def parse(self):
>  return self.parser.parse_args()

Shouldn't this change be applied to Parser.parser...

>  def run(self):
>  self.args.func(self)

... and Application.args too?

Everything else looks good.

-- 
Andrea Bolognani / Red Hat / Virtualization

Re: [libvirt PATCH v3 5/6] ci: util: Add a registry checker for stale images

[Andrea Bolognani]

On Thu, 2021-03-18 at 09:09 +0100, Erik Skultety wrote:
> +def check_stale_images(self):
> +if self.args.check_stale != "yes" or self.args.quiet:
> +return

I would prefer it if this check were to be performed in the
action_refresh() method to decide whether or not check_stale_images()
should be called.

> +print(f"""
> +The following images are stale and can be purged from the registry:
> +{stale_details}
> +
> +You can remove the above images over the API with the following code snippet:

"You can delete the images listed above using this shell snippet:"

> +$ for image_id in {stale_ids}; do \\
> +  curl --request DELETE --header "PRIVATE-TOKEN: " \\
> +  {registry_uri}/$image_id \\
> +  done

This will not result in a working shell snippet being displayed. What
you want is

  f"""
 $ for image_id in {stale_ids}; do
   curl --request DELETE --header "PRIVATE-TOKEN: " \\
{registry_uri}/$image_id;
   done
  """

> +
> +You can generate a personal access token here:
> +{gitlab_uri}/-/profile/personal_access_tokens""")

Empty line before the URL for readability.

I still think that we should use textwrap.dedent(), as it makes the
script much more readable at the cost of a single additional
string.replace() call:

  if stale_images:
  spacing = "\n" + 4 * " "
  stale_fmt = [f"{k} (ID: {v})" for k, v in stale_images.items()]
  stale_details = spacing.join(stale_fmt)
  stale_ids = ' '.join([str(id) for id in stale_images.values()])
  registry_uri = util.get_registry_uri(namespace, gitlab_uri)

  msg = textwrap.dedent(f"""
  The following images are stale and can be purged from the registry:

  STALE_DETAILS

  You can delete the images listed above using this shell snippet:

  $ for image_id in {stale_ids}; do
curl --request DELETE --header "PRIVATE-TOKEN: 
" \\
 {registry_uri}/$image_id; \\
done

  You can generate a personal access token here:

  {gitlab_uri}/-/profile/personal_access_tokens
  """)
  print(msg.replace("STALE_DETAILS", stale_details))

>  def action_refresh(self):
> +# refresh Dockerfiles and vars files
>  self.refresh_containers()
>  self.refresh_cirrus()
>  
> +# check for stale images
> +self.check_stale_images()

The method names are really quite self-explanatory, so the comments
you're introducing don't add much IMHO... I'd say leave them out.

> +def get_registry_stale_images(registry_uri: str,
> +  supported_distros: List[str]) -> Dict[str, 
> int]:
> +"""
> +Check the GitLab image registry for images that we no longer support and
> +which should be deleted.
> +
> +:param uri: URI pointing to a GitLab instance's image registry
> +:param supported_distros: list of hosts supported by lcitool
> +:return: dictionary formatted as: {: 
> }
> +"""
> +
> +images = get_registry_images(registry_uri)
> +
> +# extract distro names from the list of registry images
> +registry_distros = [get_image_distro(i["name"]) for i in images]
> +
> +# - compare the distros powering the images in GitLab registry with
> +#   the list of host available from lcitool
> +# - @unsupported is a set containing the distro names which we no longer
> +#   support; we need to map these back to registry image names
> +unsupported = set(registry_distros) - set(supported_distros)
> +if unsupported:
> +stale_images = {}
> +for distro in unsupported:
> +for img in images:
> +# gitlab images are named like "ci--?"
> +if distro in img["name"]:
> +stale_images[img["name"]] = img["id"]
> +
> +return stale_images

As far as I can tell, this can be achieved in a much more
straightforward way with

  def get_registry_stale_images(registry_uri, supported_distros):

  images = get_registry_images(registry_uri)
  stale_images = {}

  for img in images:
  if get_image_distro(img["name"]) not in supported_distros:
  stale_images[img["name"]] = img["id"]

  return stale_images

At least from a quick test, the results appear to be the same. Am I
missing something?

-- 
Andrea Bolognani / Red Hat / Virtualization

Re: [PATCH] NEWS: Document domain memory dirty rate APIs

[Michal Privoznik]

On 3/18/21 10:00 AM, Hao Wang wrote:

Signed-off-by: Hao Wang 
---
  NEWS.rst | 11 +++
  1 file changed, 11 insertions(+)

diff --git a/NEWS.rst b/NEWS.rst
index bd40373a80..df35044ef0 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -13,6 +13,17 @@ v7.2.0 (unreleased)
  
  * **New features**
  
+  * qemu: Implement domain memory dirty rate calculation API

+
+New API (``virDomainStartDirtyRateCalc()``) and virsh command
+(``domdirtyrate-calc``) are added to start calculating a live domain's
+memory dirty rate.


No need for those brackets.


+
+  * qemu: Support reporting memory dirty rate stats
+
+The memory dirty rate stats can be obtained through ``virsh domstats
+--dirtyrate`` via the virConnectGetAllDomainStats API.
+
  * **Improvements**
  
  * **Bug fixes**




Reviewed-by: Michal Privoznik 

and pushed. Thanks for handling this.

Michal

Re: [libvirt PATCH v2 4/6] ci: Introduce a util module

[Andrea Bolognani]

On Wed, 2021-03-17 at 18:40 +0100, Erik Skultety wrote:
> One more thing, what is the Python 3. acceptance policy for scripts? I
> mean we expect the scripts to work with 3.X as we declare on libvirt.org, but
> f-strings for example are 3.6+ and 3.9 essentially makes the typing module
> deprecated since it integrated most of the hints to standard collections [1],
> IOW instead of writing "List[Dict[type, type]]" you can now use the standard
> containers for that as well: "list[dict[type, type]]" without importing
> anything, so basically the list/dict builtin can serve as both container
> constructors as well as static type hints.

All target platforms, including CentOS 7 and Debian 10, have 3.6+ so
it's okay to use f-strings.

-- 
Andrea Bolognani / Red Hat / Virtualization

[PULL 13/13] block: remove support for using "file" driver with block/char devices

[Daniel P . Berrangé]

The 'host_device' and 'host_cdrom' drivers must be used instead.

Reviewed-by: Eric Blake 
Signed-off-by: Daniel P. Berrangé 
---
 block/file-posix.c   | 17 ++---
 docs/system/deprecated.rst   |  7 ---
 docs/system/removed-features.rst |  7 +++
 tests/qemu-iotests/226.out   | 10 +-
 4 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/block/file-posix.c b/block/file-posix.c
index 05079b40ca..20e14f8e96 100644
--- a/block/file-posix.c
+++ b/block/file-posix.c
@@ -719,15 +719,9 @@ static int raw_open_common(BlockDriverState *bs, QDict 
*options,
 }
 
 if (!device) {
-if (S_ISBLK(st.st_mode)) {
-warn_report("Opening a block device as a file using the '%s' "
-"driver is deprecated", bs->drv->format_name);
-} else if (S_ISCHR(st.st_mode)) {
-warn_report("Opening a character device as a file using the '%s' "
-"driver is deprecated", bs->drv->format_name);
-} else if (!S_ISREG(st.st_mode)) {
-error_setg(errp, "A regular file was expected by the '%s' driver, "
-   "but something else was given", bs->drv->format_name);
+if (!S_ISREG(st.st_mode)) {
+error_setg(errp, "'%s' driver requires '%s' to be a regular file",
+   bs->drv->format_name, bs->filename);
 ret = -EINVAL;
 goto fail;
 } else {
@@ -736,8 +730,9 @@ static int raw_open_common(BlockDriverState *bs, QDict 
*options,
 }
 } else {
 if (!(S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode))) {
-error_setg(errp, "'%s' driver expects either "
-   "a character or block device", bs->drv->format_name);
+error_setg(errp, "'%s' driver requires '%s' to be either "
+   "a character or block device",
+   bs->drv->format_name, bs->filename);
 ret = -EINVAL;
 goto fail;
 }
diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index c7e80fa731..d5e80d66eb 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -21,13 +21,6 @@ deprecated.
 System emulator command line arguments
 --
 
-``-drive file=json:{...{'driver':'file'}}`` (since 3.0)
-'''
-
-The 'file' driver for drives is no longer appropriate for character or host
-devices and will only accept regular files (S_IFREG). The correct driver
-for these file types is 'host_cdrom' or 'host_device' as appropriate.
-
 ``QEMU_AUDIO_`` environment variables and ``-audio-help`` (since 4.0)
 '
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index d164d3f290..bff16b7b05 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -57,6 +57,13 @@ by the ``tls-authz`` and ``sasl-authz`` options.
 The ``pretty=on|off`` switch has no effect for HMP monitors and
 its use is rejected.
 
+``-drive file=json:{...{'driver':'file'}}`` (removed 6.0)
+'
+
+The 'file' driver for drives is no longer appropriate for character or host
+devices and will only accept regular files (S_IFREG). The correct driver
+for these file types is 'host_cdrom' or 'host_device' as appropriate.
+
 QEMU Machine Protocol (QMP) commands
 
 
diff --git a/tests/qemu-iotests/226.out b/tests/qemu-iotests/226.out
index 42be973ff2..55504d29c4 100644
--- a/tests/qemu-iotests/226.out
+++ b/tests/qemu-iotests/226.out
@@ -3,23 +3,23 @@ QA output created by 226
 === Testing with driver:file ===
 
 == Testing RO ==
-qemu-io: can't open: A regular file was expected by the 'file' driver, but 
something else was given
-qemu-io: warning: Opening a character device as a file using the 'file' driver 
is deprecated
+qemu-io: can't open: 'file' driver requires 'TEST_DIR/t.IMGFMT' to be a 
regular file
+qemu-io: can't open: 'file' driver requires '/dev/null' to be a regular file
 == Testing RW ==
 qemu-io: can't open: Could not open 'TEST_DIR/t.IMGFMT': Is a directory
-qemu-io: warning: Opening a character device as a file using the 'file' driver 
is deprecated
+qemu-io: can't open: 'file' driver requires '/dev/null' to be a regular file
 
 === Testing with driver:host_device ===
 
 == Testing RO ==
-qemu-io: can't open: 'host_device' driver expects either a character or block 
device
+qemu-io: can't open: 'host_device' driver requires 'TEST_DIR/t.IMGFMT' to be 
either a character or block device
 == Testing RW ==
 qemu-io: can't open: Could not open 'TEST_DIR/t.IMGFMT': Is a directory
 
 === Testing with driver:host_cdrom ===
 
 == Testing RO ==
-qemu-io: can't open: 'host_cdrom' driver expects either a character or block 
device
+qemu-io: can't open: 

[PULL 12/13] block: remove 'dirty-bitmaps' field from 'BlockInfo' struct

[Daniel P . Berrangé]

The same data is available in the 'BlockDeviceInfo' struct.

Reviewed-by: Vladimir Sementsov-Ogievskiy 
Signed-off-by: Daniel P. Berrangé 
---
 block/qapi.c|  5 -
 docs/system/deprecated.rst  | 13 -
 docs/system/removed-features.rst|  9 +
 qapi/block-core.json| 11 +--
 tests/qemu-iotests/194  |  4 ++--
 tests/qemu-iotests/236  |  2 +-
 tests/qemu-iotests/246  |  3 ++-
 tests/qemu-iotests/254  |  2 +-
 tests/qemu-iotests/260  |  5 +++--
 .../tests/migrate-bitmaps-postcopy-test |  6 --
 10 files changed, 23 insertions(+), 37 deletions(-)

diff --git a/block/qapi.c b/block/qapi.c
index 3acc118c44..943e7b15ad 100644
--- a/block/qapi.c
+++ b/block/qapi.c
@@ -383,11 +383,6 @@ static void bdrv_query_info(BlockBackend *blk, BlockInfo 
**p_info,
 info->io_status = blk_iostatus(blk);
 }
 
-if (bs && !QLIST_EMPTY(>dirty_bitmaps)) {
-info->has_dirty_bitmaps = true;
-info->dirty_bitmaps = bdrv_query_dirty_bitmaps(bs);
-}
-
 if (bs && bs->drv) {
 info->has_inserted = true;
 info->inserted = bdrv_block_device_info(blk, bs, false, errp);
diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index eadba0f288..c7e80fa731 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -198,19 +198,6 @@ Use arguments ``base-node`` and ``top-node`` instead.
 
 Specify the properties for the object as top-level arguments instead.
 
-``query-block`` result field ``dirty-bitmaps`` (Since 4.2)
-''
-
-The ``dirty-bitmaps`` field of the ``BlockInfo`` structure, returned by
-the query-block command is itself now deprecated. The ``dirty-bitmaps``
-field of the ``BlockDeviceInfo`` struct should be used instead, which is the
-type of the ``inserted`` field in query-block replies, as well as the
-type of array items in query-named-block-nodes.
-
-Since the ``dirty-bitmaps`` field is optionally present in both the old and
-new locations, clients must use introspection to learn where to anticipate
-the field if/when it does appear in command output.
-
 ``nbd-server-add`` and ``nbd-server-remove`` (since 5.2)
 
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index 47fdfe6f72..d164d3f290 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -127,6 +127,15 @@ The ``status`` field of the ``BlockDirtyInfo`` structure, 
returned by
 these commands is removed. Two new boolean fields, ``recording`` and
 ``busy`` effectively replace it.
 
+``query-block`` result field ``dirty-bitmaps`` (removed in 6.0)
+'''
+
+The ``dirty-bitmaps`` field of the ``BlockInfo`` structure, returned by
+the query-block command is itself now removed. The ``dirty-bitmaps``
+field of the ``BlockDeviceInfo`` struct should be used instead, which is the
+type of the ``inserted`` field in query-block replies, as well as the
+type of array items in query-named-block-nodes.
+
 Human Monitor Protocol (HMP) commands
 -
 
diff --git a/qapi/block-core.json b/qapi/block-core.json
index 2a0c345c2c..0399449e13 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -539,9 +539,6 @@
 # @tray_open: True if the device's tray is open
 # (only present if it has a tray)
 #
-# @dirty-bitmaps: dirty bitmaps information (only present if the
-# driver has one or more dirty bitmaps) (Since 2.0)
-#
 # @io-status: @BlockDeviceIoStatus. Only present if the device
 # supports it and the VM is configured to stop on errors
 # (supported device models: virtio-blk, IDE, SCSI except
@@ -550,18 +547,12 @@
 # @inserted: @BlockDeviceInfo describing the device if media is
 #present
 #
-# Features:
-# @deprecated: Member @dirty-bitmaps is deprecated.  Use @inserted
-#  member @dirty-bitmaps instead.
-#
 # Since:  0.14
 ##
 { 'struct': 'BlockInfo',
   'data': {'device': 'str', '*qdev': 'str', 'type': 'str', 'removable': 'bool',
'locked': 'bool', '*inserted': 'BlockDeviceInfo',
-   '*tray_open': 'bool', '*io-status': 'BlockDeviceIoStatus',
-   '*dirty-bitmaps': { 'type': ['BlockDirtyInfo'],
-   'features': [ 'deprecated' ] } } }
+   '*tray_open': 'bool', '*io-status': 'BlockDeviceIoStatus' } }
 
 ##
 # @BlockMeasureInfo:
diff --git a/tests/qemu-iotests/194 b/tests/qemu-iotests/194
index 3889266afa..e44b8df728 100755
--- a/tests/qemu-iotests/194
+++ b/tests/qemu-iotests/194
@@ -95,7 +95,7 @@ with 

[PULL 11/13] block: remove dirty bitmaps 'status' field

[Daniel P . Berrangé]

The same information is available via the 'recording' and 'busy' fields.

Reviewed-by: Vladimir Sementsov-Ogievskiy 
Signed-off-by: Daniel P. Berrangé 
---
 block/dirty-bitmap.c |  38 
 docs/system/deprecated.rst   |   7 -
 docs/system/removed-features.rst |   7 +
 include/block/dirty-bitmap.h |   1 -
 qapi/block-core.json |  45 
 tests/qemu-iotests/124   |   4 -
 tests/qemu-iotests/194.out   |   4 +-
 tests/qemu-iotests/236.out   |  42 ++--
 tests/qemu-iotests/246.out   |  66 ++
 tests/qemu-iotests/254.out   |   9 +-
 tests/qemu-iotests/257.out   | 378 +++
 11 files changed, 174 insertions(+), 427 deletions(-)

diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
index a0eaa28785..68d295d6e3 100644
--- a/block/dirty-bitmap.c
+++ b/block/dirty-bitmap.c
@@ -166,43 +166,6 @@ bool bdrv_dirty_bitmap_enabled(BdrvDirtyBitmap *bitmap)
 return !bitmap->disabled;
 }
 
-/**
- * bdrv_dirty_bitmap_status: This API is now deprecated.
- * Called with BQL taken.
- *
- * A BdrvDirtyBitmap can be in four possible user-visible states:
- * (1) Active:   successor is NULL, and disabled is false: full r/w mode
- * (2) Disabled: successor is NULL, and disabled is true: qualified r/w mode,
- *   guest writes are dropped, but monitor writes are possible,
- *   through commands like merge and clear.
- * (3) Frozen:   successor is not NULL.
- *   A frozen bitmap cannot be renamed, deleted, cleared, set,
- *   enabled, merged to, etc. A frozen bitmap can only abdicate()
- *   or reclaim().
- *   In this state, the anonymous successor bitmap may be either
- *   Active and recording writes from the guest (e.g. backup jobs),
- *   or it can be Disabled and not recording writes.
- * (4) Locked:   Whether Active or Disabled, the user cannot modify this bitmap
- *   in any way from the monitor.
- * (5) Inconsistent: This is a persistent bitmap whose "in use" bit is set, and
- *   is unusable by QEMU. It can be deleted to remove it from
- *   the qcow2.
- */
-DirtyBitmapStatus bdrv_dirty_bitmap_status(BdrvDirtyBitmap *bitmap)
-{
-if (bdrv_dirty_bitmap_inconsistent(bitmap)) {
-return DIRTY_BITMAP_STATUS_INCONSISTENT;
-} else if (bdrv_dirty_bitmap_has_successor(bitmap)) {
-return DIRTY_BITMAP_STATUS_FROZEN;
-} else if (bdrv_dirty_bitmap_busy(bitmap)) {
-return DIRTY_BITMAP_STATUS_LOCKED;
-} else if (!bdrv_dirty_bitmap_enabled(bitmap)) {
-return DIRTY_BITMAP_STATUS_DISABLED;
-} else {
-return DIRTY_BITMAP_STATUS_ACTIVE;
-}
-}
-
 /* Called with BQL taken.  */
 static bool bdrv_dirty_bitmap_recording(BdrvDirtyBitmap *bitmap)
 {
@@ -582,7 +545,6 @@ BlockDirtyInfoList 
*bdrv_query_dirty_bitmaps(BlockDriverState *bs)
 info->granularity = bdrv_dirty_bitmap_granularity(bm);
 info->has_name = !!bm->name;
 info->name = g_strdup(bm->name);
-info->status = bdrv_dirty_bitmap_status(bm);
 info->recording = bdrv_dirty_bitmap_recording(bm);
 info->busy = bdrv_dirty_bitmap_busy(bm);
 info->persistent = bm->persistent;
diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index d6051ef6b4..eadba0f288 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -198,13 +198,6 @@ Use arguments ``base-node`` and ``top-node`` instead.
 
 Specify the properties for the object as top-level arguments instead.
 
-``query-named-block-nodes`` and ``query-block`` result dirty-bitmaps[i].status 
(since 4.0)
-''
-
-The ``status`` field of the ``BlockDirtyInfo`` structure, returned by
-these commands is deprecated. Two new boolean fields, ``recording`` and
-``busy`` effectively replace it.
-
 ``query-block`` result field ``dirty-bitmaps`` (Since 4.2)
 ''
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index eeff82b5ec..47fdfe6f72 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -120,6 +120,13 @@ Removed with no replacement.
 
 Removed with no replacement.
 
+``query-named-block-nodes`` and ``query-block`` result dirty-bitmaps[i].status 
(removed in 6.0)
+'''
+
+The ``status`` field of the ``BlockDirtyInfo`` structure, returned by
+these commands is removed. Two new boolean fields, ``recording`` and
+``busy`` effectively replace it.
+
 Human Monitor Protocol (HMP) commands
 -
 
diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
index f581cf9fd7..40950ae3d5 100644
--- a/include/block/dirty-bitmap.h
+++ 

[PULL 10/13] block: remove 'encryption_key_missing' flag from QAPI

[Daniel P . Berrangé]

This has been hardcoded to "false" since 2.10.0, since secrets required
to unlock block devices are now always provided up front instead of using
interactive prompts.

Reviewed-by: Eric Blake 
Reviewed-by: Thomas Huth 
Signed-off-by: Daniel P. Berrangé 
---
 block/qapi.c |  1 -
 docs/system/deprecated.rst   | 10 ---
 docs/system/removed-features.rst | 10 +++
 qapi/block-core.json |  8 --
 tests/qemu-iotests/184.out   |  6 ++--
 tests/qemu-iotests/191.out   | 48 +++-
 tests/qemu-iotests/273.out   | 15 --
 7 files changed, 33 insertions(+), 65 deletions(-)

diff --git a/block/qapi.c b/block/qapi.c
index 84a0aadc09..3acc118c44 100644
--- a/block/qapi.c
+++ b/block/qapi.c
@@ -62,7 +62,6 @@ BlockDeviceInfo *bdrv_block_device_info(BlockBackend *blk,
 info->ro = bs->read_only;
 info->drv= g_strdup(bs->drv->format_name);
 info->encrypted  = bs->encrypted;
-info->encryption_key_missing = false;
 
 info->cache = g_new(BlockdevCacheInfo, 1);
 *info->cache = (BlockdevCacheInfo) {
diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index ced6adf752..d6051ef6b4 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -183,16 +183,6 @@ Use argument ``id`` instead.
 
 Use argument ``id`` instead.
 
-``query-named-block-nodes`` result ``encryption_key_missing`` (since 2.10.0)
-
-
-Always false.
-
-``query-block`` result ``inserted.encryption_key_missing`` (since 2.10.0)
-'
-
-Always false.
-
 ``blockdev-add`` empty string argument ``backing`` (since 2.10.0)
 '
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index 2d6ff6aaf6..eeff82b5ec 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -110,6 +110,16 @@ chardev client socket with ``wait`` option (removed in 6.0)
 Character devices creating sockets in client mode should not specify
 the 'wait' field, which is only applicable to sockets in server mode
 
+``query-named-block-nodes`` result ``encryption_key_missing`` (removed in 6.0)
+''
+
+Removed with no replacement.
+
+``query-block`` result ``inserted.encryption_key_missing`` (removed in 6.0)
+'''
+
+Removed with no replacement.
+
 Human Monitor Protocol (HMP) commands
 -
 
diff --git a/qapi/block-core.json b/qapi/block-core.json
index 9f555d5c1d..d256b7b776 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -319,8 +319,6 @@
 #
 # @encrypted: true if the backing device is encrypted
 #
-# @encryption_key_missing: always false
-#
 # @detect_zeroes: detect and optimize zero writes (Since 2.1)
 #
 # @bps: total throughput limit in bytes per second is specified
@@ -385,10 +383,6 @@
 # @dirty-bitmaps: dirty bitmaps information (only present if node
 # has one or more dirty bitmaps) (Since 4.2)
 #
-# Features:
-# @deprecated: Member @encryption_key_missing is deprecated.  It is
-#  always false.
-#
 # Since: 0.14
 #
 ##
@@ -396,8 +390,6 @@
   'data': { 'file': 'str', '*node-name': 'str', 'ro': 'bool', 'drv': 'str',
 '*backing_file': 'str', 'backing_file_depth': 'int',
 'encrypted': 'bool',
-'encryption_key_missing': { 'type': 'bool',
-'features': [ 'deprecated' ] },
 'detect_zeroes': 'BlockdevDetectZeroesOptions',
 'bps': 'int', 'bps_rd': 'int', 'bps_wr': 'int',
 'iops': 'int', 'iops_rd': 'int', 'iops_wr': 'int',
diff --git a/tests/qemu-iotests/184.out b/tests/qemu-iotests/184.out
index 87c73070e3..77e5489d65 100644
--- a/tests/qemu-iotests/184.out
+++ b/tests/qemu-iotests/184.out
@@ -54,8 +54,7 @@ Testing:
 "direct": false,
 "writeback": true
 },
-"file": "json:{\"throttle-group\": \"group0\", \"driver\": 
\"throttle\", \"file\": {\"driver\": \"null-co\"}}",
-"encryption_key_missing": false
+"file": "json:{\"throttle-group\": \"group0\", \"driver\": 
\"throttle\", \"file\": {\"driver\": \"null-co\"}}"
 },
 {
 "iops_rd": 0,
@@ -82,8 +81,7 @@ Testing:
 "direct": false,
 "writeback": true
 },
-"file": "null-co://",
-"encryption_key_missing": false
+"file": "null-co://"
 }
 ]
 }
diff --git a/tests/qemu-iotests/191.out b/tests/qemu-iotests/191.out
index 022021efab..ea88777374 100644
--- a/tests/qemu-iotests/191.out
+++ 

[PULL 09/13] hw/scsi: remove 'scsi-disk' device

[Daniel P . Berrangé]

The 'scsi-hd' and 'scsi-cd' devices provide suitable alternatives.

Reviewed-by: Thomas Huth 
Signed-off-by: Daniel P. Berrangé 
---
 docs/system/deprecated.rst   |  9 -
 docs/system/removed-features.rst |  6 
 hw/i386/pc.c |  1 -
 hw/scsi/scsi-disk.c  | 62 
 scripts/device-crash-test|  1 -
 tests/qemu-iotests/051   |  2 --
 tests/qemu-iotests/051.pc.out| 10 --
 7 files changed, 6 insertions(+), 85 deletions(-)

diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index f512e05734..ced6adf752 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -275,15 +275,6 @@ The ``I7200`` guest CPU relies on the nanoMIPS ISA, which 
is deprecated
 (the ISA has never been upstreamed to a compiler toolchain). Therefore
 this CPU is also deprecated.
 
-System emulator devices

-
-``scsi-disk`` (since 4.2)
-'
-
-The 'scsi-disk' device is deprecated. Users should use 'scsi-hd' or
-'scsi-cd' as appropriate to get a SCSI hard disk or CD-ROM as needed.
-
 System emulator machines
 
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index 794fc3f367..2d6ff6aaf6 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -233,6 +233,12 @@ System emulator devices
 The 'ide-drive' device has been removed. Users should use 'ide-hd' or
 'ide-cd' as appropriate to get an IDE hard disk or CD-ROM as needed.
 
+``scsi-disk`` (removed in 6.0)
+''
+
+The 'scsi-disk' device has been removed. Users should use 'scsi-hd' or
+'scsi-cd' as appropriate to get a SCSI hard disk or CD-ROM as needed.
+
 Related binaries
 
 
diff --git a/hw/i386/pc.c b/hw/i386/pc.c
index 804913bb7e..35e1770950 100644
--- a/hw/i386/pc.c
+++ b/hw/i386/pc.c
@@ -338,7 +338,6 @@ GlobalProperty pc_compat_1_4[] = {
 PC_CPU_MODEL_IDS("1.4.0")
 { "scsi-hd", "discard_granularity", "0" },
 { "scsi-cd", "discard_granularity", "0" },
-{ "scsi-disk", "discard_granularity", "0" },
 { "ide-hd", "discard_granularity", "0" },
 { "ide-cd", "discard_granularity", "0" },
 { "virtio-blk-pci", "discard_granularity", "0" },
diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c
index 2eaea7e637..3580e7ee61 100644
--- a/hw/scsi/scsi-disk.c
+++ b/hw/scsi/scsi-disk.c
@@ -2476,28 +2476,6 @@ static void scsi_cd_realize(SCSIDevice *dev, Error 
**errp)
 aio_context_release(ctx);
 }
 
-static void scsi_disk_realize(SCSIDevice *dev, Error **errp)
-{
-DriveInfo *dinfo;
-Error *local_err = NULL;
-
-warn_report("'scsi-disk' is deprecated, "
-"please use 'scsi-hd' or 'scsi-cd' instead");
-
-if (!dev->conf.blk) {
-scsi_realize(dev, _err);
-assert(local_err);
-error_propagate(errp, local_err);
-return;
-}
-
-dinfo = blk_legacy_dinfo(dev->conf.blk);
-if (dinfo && dinfo->media_cd) {
-scsi_cd_realize(dev, errp);
-} else {
-scsi_hd_realize(dev, errp);
-}
-}
 
 static const SCSIReqOps scsi_disk_emulate_reqops = {
 .size = sizeof(SCSIDiskReq),
@@ -3161,45 +3139,6 @@ static const TypeInfo scsi_block_info = {
 };
 #endif
 
-static Property scsi_disk_properties[] = {
-DEFINE_SCSI_DISK_PROPERTIES(),
-DEFINE_PROP_BIT("removable", SCSIDiskState, features,
-SCSI_DISK_F_REMOVABLE, false),
-DEFINE_PROP_BIT("dpofua", SCSIDiskState, features,
-SCSI_DISK_F_DPOFUA, false),
-DEFINE_PROP_UINT64("wwn", SCSIDiskState, qdev.wwn, 0),
-DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, qdev.port_wwn, 0),
-DEFINE_PROP_UINT16("port_index", SCSIDiskState, port_index, 0),
-DEFINE_PROP_UINT64("max_unmap_size", SCSIDiskState, max_unmap_size,
-   DEFAULT_MAX_UNMAP_SIZE),
-DEFINE_PROP_UINT64("max_io_size", SCSIDiskState, max_io_size,
-   DEFAULT_MAX_IO_SIZE),
-DEFINE_PROP_INT32("scsi_version", SCSIDiskState, qdev.default_scsi_version,
-  5),
-DEFINE_PROP_END_OF_LIST(),
-};
-
-static void scsi_disk_class_initfn(ObjectClass *klass, void *data)
-{
-DeviceClass *dc = DEVICE_CLASS(klass);
-SCSIDeviceClass *sc = SCSI_DEVICE_CLASS(klass);
-
-sc->realize  = scsi_disk_realize;
-sc->alloc_req= scsi_new_request;
-sc->unit_attention_reported = scsi_disk_unit_attention_reported;
-dc->fw_name = "disk";
-dc->desc = "virtual SCSI disk or CD-ROM (legacy)";
-dc->reset = scsi_disk_reset;
-device_class_set_props(dc, scsi_disk_properties);
-dc->vmsd  = _scsi_disk_state;
-}
-
-static const TypeInfo scsi_disk_info = {
-.name  = "scsi-disk",
-.parent= TYPE_SCSI_DISK_BASE,
-.class_init= scsi_disk_class_initfn,
-};
-
 static void scsi_disk_register_types(void)
 {
 

[PULL 08/13] hw/ide: remove 'ide-drive' device

[Daniel P . Berrangé]

The 'ide-hd' and 'ide-cd' devices provide suitable alternatives.

Reviewed-by: Thomas Huth 
Signed-off-by: Daniel P. Berrangé 
---
 docs/qdev-device-use.txt |  2 +-
 docs/system/deprecated.rst   |  6 -
 docs/system/removed-features.rst |  8 +++
 hw/i386/pc.c |  1 -
 hw/ide/qdev.c| 38 
 hw/ppc/mac_newworld.c| 13 ---
 hw/ppc/mac_oldworld.c| 13 ---
 hw/sparc64/sun4u.c   | 15 -
 scripts/device-crash-test|  1 -
 softmmu/vl.c |  1 -
 tests/qemu-iotests/051   |  2 --
 tests/qemu-iotests/051.pc.out| 10 -
 12 files changed, 9 insertions(+), 101 deletions(-)

diff --git a/docs/qdev-device-use.txt b/docs/qdev-device-use.txt
index 245cdf29c7..2408889334 100644
--- a/docs/qdev-device-use.txt
+++ b/docs/qdev-device-use.txt
@@ -388,7 +388,7 @@ type.
 some DEVNAMEs:
 
 default device  suppressing DEVNAMEs
-CD-ROM  ide-cd, ide-drive, ide-hd, scsi-cd, scsi-hd
+CD-ROM  ide-cd, ide-hd, scsi-cd, scsi-hd
 floppy  floppy, isa-fdc
 parallelisa-parallel
 serial  isa-serial
diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index eed35fd004..f512e05734 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -278,12 +278,6 @@ this CPU is also deprecated.
 System emulator devices
 ---
 
-``ide-drive`` (since 4.2)
-'
-
-The 'ide-drive' device is deprecated. Users should use 'ide-hd' or
-'ide-cd' as appropriate to get an IDE hard disk or CD-ROM as needed.
-
 ``scsi-disk`` (since 4.2)
 '
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index 33b8c08f9b..794fc3f367 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -224,6 +224,14 @@ the upstream Linux kernel in 2018, and it has also been 
dropped from glibc, so
 there is no new Linux development taking place with this architecture. For
 running the old binaries, you can use older versions of QEMU.
 
+System emulator devices
+---
+
+``ide-drive`` (removed in 6.0)
+''
+
+The 'ide-drive' device has been removed. Users should use 'ide-hd' or
+'ide-cd' as appropriate to get an IDE hard disk or CD-ROM as needed.
 
 Related binaries
 
diff --git a/hw/i386/pc.c b/hw/i386/pc.c
index 410db9ef96..804913bb7e 100644
--- a/hw/i386/pc.c
+++ b/hw/i386/pc.c
@@ -341,7 +341,6 @@ GlobalProperty pc_compat_1_4[] = {
 { "scsi-disk", "discard_granularity", "0" },
 { "ide-hd", "discard_granularity", "0" },
 { "ide-cd", "discard_granularity", "0" },
-{ "ide-drive", "discard_granularity", "0" },
 { "virtio-blk-pci", "discard_granularity", "0" },
 /* DEV_NVECTORS_UNSPECIFIED as a uint32_t string: */
 { "virtio-serial-pci", "vectors", "0x" },
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index 8cd19fa5e9..e70ebc83a0 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -283,20 +283,6 @@ static void ide_cd_realize(IDEDevice *dev, Error **errp)
 ide_dev_initfn(dev, IDE_CD, errp);
 }
 
-static void ide_drive_realize(IDEDevice *dev, Error **errp)
-{
-DriveInfo *dinfo = NULL;
-
-warn_report("'ide-drive' is deprecated, "
-"please use 'ide-hd' or 'ide-cd' instead");
-
-if (dev->conf.blk) {
-dinfo = blk_legacy_dinfo(dev->conf.blk);
-}
-
-ide_dev_initfn(dev, dinfo && dinfo->media_cd ? IDE_CD : IDE_HD, errp);
-}
-
 #define DEFINE_IDE_DEV_PROPERTIES() \
 DEFINE_BLOCK_PROPERTIES(IDEDrive, dev.conf),\
 DEFINE_BLOCK_ERROR_PROPERTIES(IDEDrive, dev.conf),  \
@@ -355,29 +341,6 @@ static const TypeInfo ide_cd_info = {
 .class_init= ide_cd_class_init,
 };
 
-static Property ide_drive_properties[] = {
-DEFINE_IDE_DEV_PROPERTIES(),
-DEFINE_PROP_END_OF_LIST(),
-};
-
-static void ide_drive_class_init(ObjectClass *klass, void *data)
-{
-DeviceClass *dc = DEVICE_CLASS(klass);
-IDEDeviceClass *k = IDE_DEVICE_CLASS(klass);
-
-k->realize  = ide_drive_realize;
-dc->fw_name = "drive";
-dc->desc= "virtual IDE disk or CD-ROM (legacy)";
-device_class_set_props(dc, ide_drive_properties);
-}
-
-static const TypeInfo ide_drive_info = {
-.name  = "ide-drive",
-.parent= TYPE_IDE_DEVICE,
-.instance_size = sizeof(IDEDrive),
-.class_init= ide_drive_class_init,
-};
-
 static void ide_device_class_init(ObjectClass *klass, void *data)
 {
 DeviceClass *k = DEVICE_CLASS(klass);
@@ -402,7 +365,6 @@ static void ide_register_types(void)
 type_register_static(_bus_info);
 type_register_static(_hd_info);
 type_register_static(_cd_info);
-type_register_static(_drive_info);
 type_register_static(_device_type_info);
 }
 
diff --git 

[PULL 07/13] chardev: reject use of 'wait' flag for socket client chardevs

[Daniel P . Berrangé]

This only makes sense conceptually when used with listener chardevs.

Reviewed-by: Marc-André Lureau 
Signed-off-by: Daniel P. Berrangé 
---
 chardev/char-socket.c| 12 
 docs/system/deprecated.rst   |  6 --
 docs/system/removed-features.rst |  6 ++
 3 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/chardev/char-socket.c b/chardev/char-socket.c
index c8bced76b7..f618bdec28 100644
--- a/chardev/char-socket.c
+++ b/chardev/char-socket.c
@@ -1339,14 +1339,10 @@ static bool qmp_chardev_validate_socket(ChardevSocket 
*sock,
 return false;
 }
 if (sock->has_wait) {
-warn_report("'wait' option is deprecated with "
-"socket in client connect mode");
-if (sock->wait) {
-error_setg(errp, "%s",
-   "'wait' option is incompatible with "
-   "socket in client connect mode");
-return false;
-}
+error_setg(errp, "%s",
+   "'wait' option is incompatible with "
+   "socket in client connect mode");
+return false;
 }
 }
 
diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index 30c5969e22..eed35fd004 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -228,12 +228,6 @@ Since the ``dirty-bitmaps`` field is optionally present in 
both the old and
 new locations, clients must use introspection to learn where to anticipate
 the field if/when it does appear in command output.
 
-chardev client socket with ``wait`` option (since 4.0)
-''
-
-Character devices creating sockets in client mode should not specify
-the 'wait' field, which is only applicable to sockets in server mode
-
 ``nbd-server-add`` and ``nbd-server-remove`` (since 5.2)
 
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index ce1087c6a7..33b8c08f9b 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -104,6 +104,12 @@ The ``query-cpus`` command is replaced by the 
``query-cpus-fast`` command.
 The ``arch`` output member of the ``query-cpus-fast`` command is
 replaced by the ``target`` output member.
 
+chardev client socket with ``wait`` option (removed in 6.0)
+'''
+
+Character devices creating sockets in client mode should not specify
+the 'wait' field, which is only applicable to sockets in server mode
+
 Human Monitor Protocol (HMP) commands
 -
 
-- 
2.30.2

[PULL 06/13] machine: remove 'arch' field from 'query-cpus-fast' QMP command

[Daniel P . Berrangé]

Reviewed-by: Thomas Huth 
Signed-off-by: Daniel P. Berrangé 
---
 docs/system/deprecated.rst   |  6 -
 docs/system/removed-features.rst |  6 +
 hw/core/machine-qmp-cmds.c   | 41 
 qapi/machine.json| 22 -
 4 files changed, 6 insertions(+), 69 deletions(-)

diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index 9a62afa6de..30c5969e22 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -228,12 +228,6 @@ Since the ``dirty-bitmaps`` field is optionally present in 
both the old and
 new locations, clients must use introspection to learn where to anticipate
 the field if/when it does appear in command output.
 
-``query-cpus-fast`` ``arch`` output member (since 3.0.0)
-
-
-The ``arch`` output member of the ``query-cpus-fast`` command is
-replaced by the ``target`` output member.
-
 chardev client socket with ``wait`` option (since 4.0)
 ''
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index d7d86b3143..ce1087c6a7 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -98,6 +98,12 @@ Use ``migrate_set_parameter`` instead.
 
 The ``query-cpus`` command is replaced by the ``query-cpus-fast`` command.
 
+``query-cpus-fast`` ``arch`` output member (removed in 6.0)
+'''
+
+The ``arch`` output member of the ``query-cpus-fast`` command is
+replaced by the ``target`` output member.
+
 Human Monitor Protocol (HMP) commands
 -
 
diff --git a/hw/core/machine-qmp-cmds.c b/hw/core/machine-qmp-cmds.c
index af60cd969d..68a942595a 100644
--- a/hw/core/machine-qmp-cmds.c
+++ b/hw/core/machine-qmp-cmds.c
@@ -24,46 +24,6 @@
 #include "sysemu/runstate.h"
 #include "sysemu/sysemu.h"
 
-static CpuInfoArch sysemu_target_to_cpuinfo_arch(SysEmuTarget target)
-{
-/*
- * The @SysEmuTarget -> @CpuInfoArch mapping below is based on the
- * TARGET_ARCH -> TARGET_BASE_ARCH mapping in the "configure" script.
- */
-switch (target) {
-case SYS_EMU_TARGET_I386:
-case SYS_EMU_TARGET_X86_64:
-return CPU_INFO_ARCH_X86;
-
-case SYS_EMU_TARGET_PPC:
-case SYS_EMU_TARGET_PPC64:
-return CPU_INFO_ARCH_PPC;
-
-case SYS_EMU_TARGET_SPARC:
-case SYS_EMU_TARGET_SPARC64:
-return CPU_INFO_ARCH_SPARC;
-
-case SYS_EMU_TARGET_MIPS:
-case SYS_EMU_TARGET_MIPSEL:
-case SYS_EMU_TARGET_MIPS64:
-case SYS_EMU_TARGET_MIPS64EL:
-return CPU_INFO_ARCH_MIPS;
-
-case SYS_EMU_TARGET_TRICORE:
-return CPU_INFO_ARCH_TRICORE;
-
-case SYS_EMU_TARGET_S390X:
-return CPU_INFO_ARCH_S390;
-
-case SYS_EMU_TARGET_RISCV32:
-case SYS_EMU_TARGET_RISCV64:
-return CPU_INFO_ARCH_RISCV;
-
-default:
-return CPU_INFO_ARCH_OTHER;
-}
-}
-
 static void cpustate_to_cpuinfo_s390(CpuInfoS390 *info, const CPUState *cpu)
 {
 #ifdef TARGET_S390X
@@ -104,7 +64,6 @@ CpuInfoFastList *qmp_query_cpus_fast(Error **errp)
 value->props = props;
 }
 
-value->arch = sysemu_target_to_cpuinfo_arch(target);
 value->target = target;
 if (target == SYS_EMU_TARGET_S390X) {
 cpustate_to_cpuinfo_s390(>u.s390x, cpu);
diff --git a/qapi/machine.json b/qapi/machine.json
index 9811927504..c0c52aef10 100644
--- a/qapi/machine.json
+++ b/qapi/machine.json
@@ -34,21 +34,6 @@
  'sh4eb', 'sparc', 'sparc64', 'tricore', 'unicore32',
  'x86_64', 'xtensa', 'xtensaeb' ] }
 
-##
-# @CpuInfoArch:
-#
-# An enumeration of cpu types that enable additional information during
-# @query-cpus-fast.
-#
-# @s390: since 2.12
-#
-# @riscv: since 2.12
-#
-# Since: 2.6
-##
-{ 'enum': 'CpuInfoArch',
-  'data': ['x86', 'sparc', 'ppc', 'mips', 'tricore', 's390', 'riscv', 'other' 
] }
-
 ##
 # @CpuS390State:
 #
@@ -86,14 +71,9 @@
 # @props: properties describing to which node/socket/core/thread
 # virtual CPU belongs to, provided if supported by board
 #
-# @arch: base architecture of the cpu
-#
 # @target: the QEMU system emulation target, which determines which
 #  additional fields will be listed (since 3.0)
 #
-# Features:
-# @deprecated: Member @arch is deprecated.  Use @target instead.
-#
 # Since: 2.12
 #
 ##
@@ -102,8 +82,6 @@
   'qom-path' : 'str',
   'thread-id': 'int',
   '*props'   : 'CpuInstanceProperties',
-  'arch' : { 'type': 'CpuInfoArch',
- 'features': [ 'deprecated' ] },
   'target'   : 'SysEmuTarget' },
   'discriminator' : 'target',
   'data'  : { 's390x': 'CpuInfoS390' } }
-- 
2.30.2

[PULL 05/13] machine: remove 'query-cpus' QMP command

[Daniel P . Berrangé]

The newer 'query-cpus-fast' command avoids side effects on the guest
execution. Note that some of the field names are different in the
'query-cpus-fast' command.

Reviewed-by: Wainer dos Santos Moschetta 
Tested-by: Wainer dos Santos Moschetta 
Signed-off-by: Daniel P. Berrangé 
---
 docs/system/deprecated.rst |   5 -
 docs/system/removed-features.rst   |   5 +
 hw/core/machine-hmp-cmds.c |   8 +-
 hw/core/machine-qmp-cmds.c |  79 --
 qapi/machine.json  | 161 +
 tests/acceptance/pc_cpu_hotplug_props.py   |   2 +-
 tests/acceptance/x86_cpu_model_versions.py |   2 +-
 tests/migration/guestperf/engine.py|   2 +-
 tests/qtest/numa-test.c|   6 +-
 tests/qtest/qmp-test.c |   6 +-
 tests/qtest/test-x86-cpuid-compat.c|   4 +-
 11 files changed, 22 insertions(+), 258 deletions(-)

diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index c92387180c..9a62afa6de 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -228,11 +228,6 @@ Since the ``dirty-bitmaps`` field is optionally present in 
both the old and
 new locations, clients must use introspection to learn where to anticipate
 the field if/when it does appear in command output.
 
-``query-cpus`` (since 2.12.0)
-'
-
-The ``query-cpus`` command is replaced by the ``query-cpus-fast`` command.
-
 ``query-cpus-fast`` ``arch`` output member (since 3.0.0)
 
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index d7f46a6ebc..d7d86b3143 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -93,6 +93,11 @@ Use ``migrate_set_parameter`` and ``info 
migrate_parameters`` instead.
 
 Use ``migrate_set_parameter`` instead.
 
+``query-cpus`` (removed in 6.0)
+'''
+
+The ``query-cpus`` command is replaced by the ``query-cpus-fast`` command.
+
 Human Monitor Protocol (HMP) commands
 -
 
diff --git a/hw/core/machine-hmp-cmds.c b/hw/core/machine-hmp-cmds.c
index 6357be9c6b..58248cffa3 100644
--- a/hw/core/machine-hmp-cmds.c
+++ b/hw/core/machine-hmp-cmds.c
@@ -130,7 +130,7 @@ void hmp_info_numa(Monitor *mon, const QDict *qdict)
 {
 int i, nb_numa_nodes;
 NumaNodeMem *node_mem;
-CpuInfoList *cpu_list, *cpu;
+CpuInfoFastList *cpu_list, *cpu;
 MachineState *ms = MACHINE(qdev_get_machine());
 
 nb_numa_nodes = ms->numa_state ? ms->numa_state->num_nodes : 0;
@@ -139,7 +139,7 @@ void hmp_info_numa(Monitor *mon, const QDict *qdict)
 return;
 }
 
-cpu_list = qmp_query_cpus(_abort);
+cpu_list = qmp_query_cpus_fast(_abort);
 node_mem = g_new0(NumaNodeMem, nb_numa_nodes);
 
 query_numa_node_mem(node_mem, ms);
@@ -148,7 +148,7 @@ void hmp_info_numa(Monitor *mon, const QDict *qdict)
 for (cpu = cpu_list; cpu; cpu = cpu->next) {
 if (cpu->value->has_props && cpu->value->props->has_node_id &&
 cpu->value->props->node_id == i) {
-monitor_printf(mon, " %" PRIi64, cpu->value->CPU);
+monitor_printf(mon, " %" PRIi64, cpu->value->cpu_index);
 }
 }
 monitor_printf(mon, "\n");
@@ -157,6 +157,6 @@ void hmp_info_numa(Monitor *mon, const QDict *qdict)
 monitor_printf(mon, "node %d plugged: %" PRId64 " MB\n", i,
node_mem[i].node_plugged_mem >> 20);
 }
-qapi_free_CpuInfoList(cpu_list);
+qapi_free_CpuInfoFastList(cpu_list);
 g_free(node_mem);
 }
diff --git a/hw/core/machine-qmp-cmds.c b/hw/core/machine-qmp-cmds.c
index 44e979e503..af60cd969d 100644
--- a/hw/core/machine-qmp-cmds.c
+++ b/hw/core/machine-qmp-cmds.c
@@ -24,85 +24,6 @@
 #include "sysemu/runstate.h"
 #include "sysemu/sysemu.h"
 
-CpuInfoList *qmp_query_cpus(Error **errp)
-{
-MachineState *ms = MACHINE(qdev_get_machine());
-MachineClass *mc = MACHINE_GET_CLASS(ms);
-CpuInfoList *head = NULL, **tail = 
-CPUState *cpu;
-
-CPU_FOREACH(cpu) {
-CpuInfo *value;
-#if defined(TARGET_I386)
-X86CPU *x86_cpu = X86_CPU(cpu);
-CPUX86State *env = _cpu->env;
-#elif defined(TARGET_PPC)
-PowerPCCPU *ppc_cpu = POWERPC_CPU(cpu);
-CPUPPCState *env = _cpu->env;
-#elif defined(TARGET_SPARC)
-SPARCCPU *sparc_cpu = SPARC_CPU(cpu);
-CPUSPARCState *env = _cpu->env;
-#elif defined(TARGET_RISCV)
-RISCVCPU *riscv_cpu = RISCV_CPU(cpu);
-CPURISCVState *env = _cpu->env;
-#elif defined(TARGET_MIPS)
-MIPSCPU *mips_cpu = MIPS_CPU(cpu);
-CPUMIPSState *env = _cpu->env;
-#elif defined(TARGET_TRICORE)
-TriCoreCPU *tricore_cpu = TRICORE_CPU(cpu);
-CPUTriCoreState *env = _cpu->env;
-#elif defined(TARGET_S390X)
-S390CPU *s390_cpu = S390_CPU(cpu);
-

[PULL 04/13] migrate: remove QMP/HMP commands for speed, downtime and cache size

[Daniel P . Berrangé]

The generic 'migrate_set_parameters' command handle all types of param.

Only the QMP commands were documented in the deprecations page, but the
rationale for deprecating applies equally to HMP, and the replacements
exist. Furthermore the HMP commands are just shims to the QMP commands,
so removing the latter breaks the former unless they get re-implemented.

Reviewed-by: Dr. David Alan Gilbert 
Signed-off-by: Daniel P. Berrangé 
---
 docs/devel/migration.rst|  2 +-
 docs/rdma.txt   |  2 +-
 docs/system/deprecated.rst  | 10 ---
 docs/system/removed-features.rst| 20 ++
 docs/xbzrle.txt |  5 --
 hmp-commands-info.hx| 13 
 hmp-commands.hx | 45 -
 include/monitor/hmp.h   |  4 --
 migration/migration.c   | 45 -
 migration/ram.c |  2 +-
 monitor/hmp-cmds.c  | 34 --
 qapi/migration.json | 98 -
 tests/migration/guestperf/engine.py | 16 ++---
 tests/qemu-iotests/181  |  2 +-
 tests/qtest/migration-test.c| 48 --
 tests/qtest/test-hmp.c  |  6 +-
 tests/qtest/vhost-user-test.c   |  8 +--
 17 files changed, 40 insertions(+), 320 deletions(-)

diff --git a/docs/devel/migration.rst b/docs/devel/migration.rst
index ad381b89b2..19c3d4f3ea 100644
--- a/docs/devel/migration.rst
+++ b/docs/devel/migration.rst
@@ -641,7 +641,7 @@ time per vCPU.
 
 .. note::
   During the postcopy phase, the bandwidth limits set using
-  ``migrate_set_speed`` is ignored (to avoid delaying requested pages that
+  ``migrate_set_parameter`` is ignored (to avoid delaying requested pages that
   the destination is waiting for).
 
 Postcopy device transfer
diff --git a/docs/rdma.txt b/docs/rdma.txt
index 49dc9f8bca..2b4cdea1d8 100644
--- a/docs/rdma.txt
+++ b/docs/rdma.txt
@@ -89,7 +89,7 @@ RUNNING:
 First, set the migration speed to match your hardware's capabilities:
 
 QEMU Monitor Command:
-$ migrate_set_speed 40g # or whatever is the MAX of your RDMA device
+$ migrate_set_parameter max_bandwidth 40g # or whatever is the MAX of your 
RDMA device
 
 Next, on the destination machine, add the following to the QEMU command line:
 
diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index bffc7e4d59..c92387180c 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -183,11 +183,6 @@ Use argument ``id`` instead.
 
 Use argument ``id`` instead.
 
-``migrate_set_downtime`` and ``migrate_set_speed`` (since 2.8.0)
-
-
-Use ``migrate-set-parameters`` instead.
-
 ``query-named-block-nodes`` result ``encryption_key_missing`` (since 2.10.0)
 
 
@@ -203,11 +198,6 @@ Always false.
 
 Use argument value ``null`` instead.
 
-``migrate-set-cache-size`` and ``query-migrate-cache-size`` (since 2.11.0)
-''
-
-Use ``migrate-set-parameters`` and ``query-migrate-parameters`` instead.
-
 ``block-commit`` arguments ``base`` and ``top`` (since 3.1.0)
 '
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index e932d023a2..d7f46a6ebc 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -83,6 +83,16 @@ Use ``blockdev-change-medium`` or ``change-vnc-password`` 
instead.
 The ``query-events`` command has been superseded by the more powerful
 and accurate ``query-qmp-schema`` command.
 
+``migrate_set_cache_size`` and ``query-migrate-cache-size`` (removed in 6.0)
+
+
+Use ``migrate_set_parameter`` and ``info migrate_parameters`` instead.
+
+``migrate_set_downtime`` and ``migrate_set_speed`` (removed in 6.0)
+'''
+
+Use ``migrate_set_parameter`` instead.
+
 Human Monitor Protocol (HMP) commands
 -
 
@@ -111,6 +121,16 @@ The ``acl_show``, ``acl_reset``, ``acl_policy``, 
``acl_add``, and
 ``acl_remove`` commands were removed with no replacement. Authorization
 for VNC should be performed using the pluggable QAuthZ objects.
 
+``migrate-set-cache-size`` and ``info migrate-cache-size`` (removed in 6.0)
+'''
+
+Use ``migrate-set-parameters`` and ``info migrate-parameters`` instead.
+
+``migrate_set_downtime`` and ``migrate_set_speed`` (removed in 6.0)
+'''
+
+Use ``migrate-set-parameters`` instead.
+
 Guest Emulator ISAs
 ---
 
diff --git a/docs/xbzrle.txt b/docs/xbzrle.txt
index 6bd1828f34..bcb3f0c901 

[PULL 02/13] monitor: raise error when 'pretty' option is used with HMP

[Daniel P . Berrangé]

This is only semantically useful for QMP.

Reviewed-by: Dr. David Alan Gilbert 
Signed-off-by: Daniel P. Berrangé 
---
 docs/system/deprecated.rst   | 7 ---
 docs/system/removed-features.rst | 6 ++
 monitor/monitor.c| 4 ++--
 qemu-options.hx  | 5 +++--
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index 0e1fdf5c14..8f4d756e35 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -53,13 +53,6 @@ needs two devices (``-device intel-hda -device hda-duplex``) 
and
 ``pcspk`` which can be activated using ``-machine
 pcspk-audiodev=``.
 
-``-mon ...,control=readline,pretty=on|off`` (since 4.1)
-'''
-
-The ``pretty=on|off`` switch has no effect for HMP monitors, but is
-silently ignored. Using the switch with HMP monitors will become an
-error in the future.
-
 RISC-V ``-bios`` (since 5.1)
 
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index aa612c2846..372aadceae 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -51,6 +51,12 @@ host controller or ``-usb`` if necessary).
 The ``acl`` option to the ``-vnc`` argument has been replaced
 by the ``tls-authz`` and ``sasl-authz`` options.
 
+``-mon ...,control=readline,pretty=on|off`` (removed in 6.0)
+
+
+The ``pretty=on|off`` switch has no effect for HMP monitors and
+its use is rejected.
+
 QEMU Machine Protocol (QMP) commands
 
 
diff --git a/monitor/monitor.c b/monitor/monitor.c
index 640496e562..636bcc81c5 100644
--- a/monitor/monitor.c
+++ b/monitor/monitor.c
@@ -716,8 +716,8 @@ int monitor_init(MonitorOptions *opts, bool allow_hmp, 
Error **errp)
 return -1;
 }
 if (opts->pretty) {
-warn_report("'pretty' is deprecated for HMP monitors, it has no "
-"effect and will be removed in future versions");
+error_setg(errp, "'pretty' is not compatible with HMP monitors");
+return -1;
 }
 monitor_init_hmp(chr, true, _err);
 break;
diff --git a/qemu-options.hx b/qemu-options.hx
index 9128dec510..671b310ab8 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -3740,8 +3740,9 @@ DEF("mon", HAS_ARG, QEMU_OPTION_mon, \
 "-mon [chardev=]name[,mode=readline|control][,pretty[=on|off]]\n", 
QEMU_ARCH_ALL)
 SRST
 ``-mon [chardev=]name[,mode=readline|control][,pretty[=on|off]]``
-Setup monitor on chardev name. ``pretty`` turns on JSON pretty
-printing easing human reading and debugging.
+Setup monitor on chardev name. ``pretty`` is only valid when
+``mode=control``, turning on JSON pretty printing to ease
+human reading and debugging.
 ERST
 
 DEF("debugcon", HAS_ARG, QEMU_OPTION_debugcon, \
-- 
2.30.2

[PULL 03/13] monitor: remove 'query-events' QMP command

[Daniel P . Berrangé]

The code comment suggests removing QAPIEvent_(str|lookup) symbols too,
however, these are both auto-generated as standard for any enum in
QAPI. As such it they'll exist whether we use them or not.

Reviewed-by: Eric Blake 
Reviewed-by: Thomas Huth 
Signed-off-by: Daniel P. Berrangé 
---
 docs/system/deprecated.rst   |  6 -
 docs/system/removed-features.rst |  6 +
 monitor/qmp-cmds-control.c   | 24 -
 qapi/control.json| 45 
 4 files changed, 6 insertions(+), 75 deletions(-)

diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index 8f4d756e35..bffc7e4d59 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -249,12 +249,6 @@ The ``query-cpus`` command is replaced by the 
``query-cpus-fast`` command.
 The ``arch`` output member of the ``query-cpus-fast`` command is
 replaced by the ``target`` output member.
 
-``query-events`` (since 4.0)
-
-
-The ``query-events`` command has been superseded by the more powerful
-and accurate ``query-qmp-schema`` command.
-
 chardev client socket with ``wait`` option (since 4.0)
 ''
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index 372aadceae..e932d023a2 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -77,6 +77,12 @@ documentation of ``query-hotpluggable-cpus`` for additional 
details.
 
 Use ``blockdev-change-medium`` or ``change-vnc-password`` instead.
 
+``query-events`` (removed in 6.0)
+'
+
+The ``query-events`` command has been superseded by the more powerful
+and accurate ``query-qmp-schema`` command.
+
 Human Monitor Protocol (HMP) commands
 -
 
diff --git a/monitor/qmp-cmds-control.c b/monitor/qmp-cmds-control.c
index 509ae870bd..513b547233 100644
--- a/monitor/qmp-cmds-control.c
+++ b/monitor/qmp-cmds-control.c
@@ -130,30 +130,6 @@ CommandInfoList *qmp_query_commands(Error **errp)
 return list;
 }
 
-EventInfoList *qmp_query_events(Error **errp)
-{
-/*
- * TODO This deprecated command is the only user of
- * QAPIEvent_str() and QAPIEvent_lookup[].  When the command goes,
- * they should go, too.
- */
-EventInfoList *ev_list = NULL;
-QAPIEvent e;
-
-for (e = 0 ; e < QAPI_EVENT__MAX ; e++) {
-const char *event_name = QAPIEvent_str(e);
-EventInfo *info;
-
-assert(event_name != NULL);
-info = g_malloc0(sizeof(*info));
-info->name = g_strdup(event_name);
-
-QAPI_LIST_PREPEND(ev_list, info);
-}
-
-return ev_list;
-}
-
 /*
  * Minor hack: generated marshalling suppressed for this command
  * ('gen': false in the schema) so we can parse the JSON string
diff --git a/qapi/control.json b/qapi/control.json
index 2615d5170b..71a838d49e 100644
--- a/qapi/control.json
+++ b/qapi/control.json
@@ -159,51 +159,6 @@
 { 'command': 'query-commands', 'returns': ['CommandInfo'],
   'allow-preconfig': true }
 
-##
-# @EventInfo:
-#
-# Information about a QMP event
-#
-# @name: The event name
-#
-# Since: 1.2
-##
-{ 'struct': 'EventInfo', 'data': {'name': 'str'} }
-
-##
-# @query-events:
-#
-# Return information on QMP events.
-#
-# Features:
-# @deprecated: This command is deprecated, because its output doesn't
-#  reflect compile-time configuration.  Use 'query-qmp-schema'
-#  instead.
-#
-# Returns: A list of @EventInfo.
-#
-# Since: 1.2
-#
-# Example:
-#
-# -> { "execute": "query-events" }
-# <- {
-#  "return": [
-#  {
-# "name":"SHUTDOWN"
-#  },
-#  {
-# "name":"RESET"
-#  }
-#   ]
-#}
-#
-# Note: This example has been shortened as the real response is too long.
-#
-##
-{ 'command': 'query-events', 'returns': ['EventInfo'],
-  'features': [ 'deprecated' ] }
-
 ##
 # @quit:
 #
-- 
2.30.2

[PULL 01/13] ui, monitor: remove deprecated VNC ACL option and HMP commands

[Daniel P . Berrangé]

The VNC ACL concept has been replaced by the pluggable "authz" framework
which does not use monitor commands.

Reviewed-by: Thomas Huth 
Reviewed-by: Dr. David Alan Gilbert 
Signed-off-by: Daniel P. Berrangé 
---
 docs/system/deprecated.rst   |  16 ---
 docs/system/removed-features.rst |  12 ++
 hmp-commands.hx  |  76 -
 monitor/misc.c   | 187 ---
 ui/vnc.c |  38 ---
 5 files changed, 12 insertions(+), 317 deletions(-)

diff --git a/docs/system/deprecated.rst b/docs/system/deprecated.rst
index 1db30f8f27..0e1fdf5c14 100644
--- a/docs/system/deprecated.rst
+++ b/docs/system/deprecated.rst
@@ -28,12 +28,6 @@ The 'file' driver for drives is no longer appropriate for 
character or host
 devices and will only accept regular files (S_IFREG). The correct driver
 for these file types is 'host_cdrom' or 'host_device' as appropriate.
 
-``-vnc acl`` (since 4.0.0)
-''
-
-The ``acl`` option to the ``-vnc`` argument has been replaced
-by the ``tls-authz`` and ``sasl-authz`` options.
-
 ``QEMU_AUDIO_`` environment variables and ``-audio-help`` (since 4.0)
 '
 
@@ -281,16 +275,6 @@ Use the more generic commands ``block-export-add`` and 
``block-export-del``
 instead.  As part of this deprecation, where ``nbd-server-add`` used a
 single ``bitmap``, the new ``block-export-add`` uses a list of ``bitmaps``.
 
-Human Monitor Protocol (HMP) commands
--
-
-``acl_show``, ``acl_reset``, ``acl_policy``, ``acl_add``, ``acl_remove`` 
(since 4.0.0)
-''
-
-The ``acl_show``, ``acl_reset``, ``acl_policy``, ``acl_add``, and
-``acl_remove`` commands are deprecated with no replacement. Authorization
-for VNC should be performed using the pluggable QAuthZ objects.
-
 System emulator CPUS
 
 
diff --git a/docs/system/removed-features.rst b/docs/system/removed-features.rst
index 82e7fcc517..aa612c2846 100644
--- a/docs/system/removed-features.rst
+++ b/docs/system/removed-features.rst
@@ -45,6 +45,11 @@ This option lacked the possibility to specify an audio 
backend device.
 Use ``-device usb-audio`` now instead (and specify a corresponding USB
 host controller or ``-usb`` if necessary).
 
+``-vnc acl`` (removed in 6.0)
+'
+
+The ``acl`` option to the ``-vnc`` argument has been replaced
+by the ``tls-authz`` and ``sasl-authz`` options.
 
 QEMU Machine Protocol (QMP) commands
 
@@ -87,6 +92,13 @@ documentation of ``query-hotpluggable-cpus`` for additional 
details.
 No replacement.  The ``change vnc password`` and ``change DEVICE MEDIUM``
 commands are not affected.
 
+``acl_show``, ``acl_reset``, ``acl_policy``, ``acl_add``, ``acl_remove`` 
(removed in 6.0)
+'
+
+The ``acl_show``, ``acl_reset``, ``acl_policy``, ``acl_add``, and
+``acl_remove`` commands were removed with no replacement. Authorization
+for VNC should be performed using the pluggable QAuthZ objects.
+
 Guest Emulator ISAs
 ---
 
diff --git a/hmp-commands.hx b/hmp-commands.hx
index d4001f9c5d..b500b8526d 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -1433,82 +1433,6 @@ SRST
   Change watchdog action.
 ERST
 
-{
-.name   = "acl_show",
-.args_type  = "aclname:s",
-.params = "aclname",
-.help   = "list rules in the access control list",
-.cmd= hmp_acl_show,
-},
-
-SRST
-``acl_show`` *aclname*
-  List all the matching rules in the access control list, and the default
-  policy. There are currently two named access control lists,
-  *vnc.x509dname* and *vnc.username* matching on the x509 client
-  certificate distinguished name, and SASL username respectively.
-ERST
-
-{
-.name   = "acl_policy",
-.args_type  = "aclname:s,policy:s",
-.params = "aclname allow|deny",
-.help   = "set default access control list policy",
-.cmd= hmp_acl_policy,
-},
-
-SRST
-``acl_policy`` *aclname* ``allow|deny``
-  Set the default access control list policy, used in the event that
-  none of the explicit rules match. The default policy at startup is
-  always ``deny``.
-ERST
-
-{
-.name   = "acl_add",
-.args_type  = "aclname:s,match:s,policy:s,index:i?",
-.params = "aclname match allow|deny [index]",
-.help   = "add a match rule to the access control list",
-.cmd= hmp_acl_add,
-},
-
-SRST
-``acl_add`` *aclname* *match* ``allow|deny`` [*index*]
-  Add a match rule to the access control list, allowing or denying access.
-  The match will normally be an exact username or x509 distinguished name,
-  but can 

[PULL v2 00/13] misc patches removing deprecated features

[Daniel P . Berrangé]

The following changes since commit 571d413b5da6bc6f1c2aaca8484717642255ddb0:

  Merge remote-tracking branch 'remotes/mcayland/tags/qemu-openbios-20210316' 
into staging (2021-03-17 21:02:37 +)

are available in the Git repository at:

  https://gitlab.com/berrange/qemu tags/dep-many-pull-request

for you to fetch changes up to 8d17adf34f501ded65a106572740760f0a75577c:

  block: remove support for using "file" driver with block/char devices 
(2021-03-18 09:22:55 +)


Remove many old deprecated features

The following features have been deprecated for well over the 2
release cycle we promise

  ``-drive file=json:{...{'driver':'file'}}`` (since 3.0)
  ``-vnc acl`` (since 4.0.0)
  ``-mon ...,control=readline,pretty=on|off`` (since 4.1)
  ``migrate_set_downtime`` and ``migrate_set_speed`` (since 2.8.0)
  ``query-named-block-nodes`` result ``encryption_key_missing`` (since 2.10.0)
  ``query-block`` result ``inserted.encryption_key_missing`` (since 2.10.0)
  ``migrate-set-cache-size`` and ``query-migrate-cache-size`` (since 2.11.0)
  ``query-named-block-nodes`` and ``query-block`` result 
dirty-bitmaps[i].status (since 4.0)
  ``query-cpus`` (since 2.12.0)
  ``query-cpus-fast`` ``arch`` output member (since 3.0.0)
  ``query-events`` (since 4.0)
  chardev client socket with ``wait`` option (since 4.0)
  ``acl_show``, ``acl_reset``, ``acl_policy``, ``acl_add``, ``acl_remove`` 
(since 4.0.0)
  ``ide-drive`` (since 4.2)
  ``scsi-disk`` (since 4.2)



Daniel P. Berrangé (13):
  ui, monitor: remove deprecated VNC ACL option and HMP commands
  monitor: raise error when 'pretty' option is used with HMP
  monitor: remove 'query-events' QMP command
  migrate: remove QMP/HMP commands for speed, downtime and cache size
  machine: remove 'query-cpus' QMP command
  machine: remove 'arch' field from 'query-cpus-fast' QMP command
  chardev: reject use of 'wait' flag for socket client chardevs
  hw/ide: remove 'ide-drive' device
  hw/scsi: remove 'scsi-disk' device
  block: remove 'encryption_key_missing' flag from QAPI
  block: remove dirty bitmaps 'status' field
  block: remove 'dirty-bitmaps' field from 'BlockInfo' struct
  block: remove support for using "file" driver with block/char devices

 block/dirty-bitmap.c  |  38 --
 block/file-posix.c|  17 +-
 block/qapi.c  |   6 -
 chardev/char-socket.c |  12 +-
 docs/devel/migration.rst  |   2 +-
 docs/qdev-device-use.txt  |   2 +-
 docs/rdma.txt |   2 +-
 docs/system/deprecated.rst| 108 -
 docs/system/removed-features.rst  | 108 +
 docs/xbzrle.txt   |   5 -
 hmp-commands-info.hx  |  13 -
 hmp-commands.hx   | 121 --
 hw/core/machine-hmp-cmds.c|   8 +-
 hw/core/machine-qmp-cmds.c| 120 --
 hw/i386/pc.c  |   2 -
 hw/ide/qdev.c |  38 --
 hw/ppc/mac_newworld.c |  13 -
 hw/ppc/mac_oldworld.c |  13 -
 hw/scsi/scsi-disk.c   |  62 ---
 hw/sparc64/sun4u.c|  15 -
 include/block/dirty-bitmap.h  |   1 -
 include/monitor/hmp.h |   4 -
 migration/migration.c |  45 ---
 migration/ram.c   |   2 +-
 monitor/hmp-cmds.c|  34 --
 monitor/misc.c| 187 -
 monitor/monitor.c |   4 +-
 monitor/qmp-cmds-control.c|  24 --
 qapi/block-core.json  |  64 +--
 qapi/control.json |  45 ---
 qapi/machine.json | 181 +
 qapi/migration.json   |  98 -
 qemu-options.hx   |   5 +-
 scripts/device-crash-test |   2 -
 softmmu/vl.c  |   1 -
 tests/acceptance/pc_cpu_hotplug_props.py  |   2 +-
 tests/acceptance/x86_cpu_model_versions.py|   2 +-
 tests/migration/guestperf/engine.py   |  18 +-
 tests/qemu-iotests/051|   4 -
 tests/qemu-iotests/051.pc.out |  20 -
 tests/qemu-iotests/124|   4 -
 tests/qemu-iotests/181|   2 +-
 tests/qemu-iotests/184.out|   6 +-
 tests/qemu-iotests/191.out|  48 +--
 tests/qemu-iotests/194|   4 +-
 tests/qemu-iotests/194.out|   4 +-
 tests/qemu-iotests/226.out|  

[PATCH] NEWS: Document domain memory dirty rate APIs

[Hao Wang]

Signed-off-by: Hao Wang 
---
 NEWS.rst | 11 +++
 1 file changed, 11 insertions(+)

diff --git a/NEWS.rst b/NEWS.rst
index bd40373a80..df35044ef0 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -13,6 +13,17 @@ v7.2.0 (unreleased)
 
 * **New features**
 
+  * qemu: Implement domain memory dirty rate calculation API
+
+New API (``virDomainStartDirtyRateCalc()``) and virsh command
+(``domdirtyrate-calc``) are added to start calculating a live domain's
+memory dirty rate.
+
+  * qemu: Support reporting memory dirty rate stats
+
+The memory dirty rate stats can be obtained through ``virsh domstats
+--dirtyrate`` via the virConnectGetAllDomainStats API.
+
 * **Improvements**
 
 * **Bug fixes**
-- 
2.23.0

Re: [PATCH v7 0/7] migration/dirtyrate: Introduce APIs for getting domain memory dirty rate

[Hao Wang]

Sure. I'll post NEWS.rst information soon, and really appreciate your help!

Best Regards,
Hao

On 2021/3/18 15:49, Michal Privoznik wrote:
> On 3/16/21 1:32 PM, Hao Wang wrote:
>> Hao Wang (7):
>>    migration/dirtyrate: Introduce virDomainStartDirtyRateCalc API
>>    migration/dirtyrate: Implement qemuDomainStartDirtyRateCalc
>>    migration/dirtyrate: Introduce domdirtyrate-calc virsh api
>>    migration/dirtyrate: Introduce virDomainDirtyRateStatus enum
>>    migration/dirtyrate: Implement qemuMonitorQueryDirtyRate
>>    migration/dirtyrate: Extend dirtyrate statistics for domGetStats
>>    migration/dirtyrate: Introduce command 'virsh domstats --dirtyrate'
>>
>>   docs/manpages/virsh.rst  |  33 +-
>>   include/libvirt/libvirt-domain.h |  23 +++
>>   src/driver-hypervisor.h  |   6 ++
>>   src/libvirt-domain.c |  59 +
>>   src/libvirt_public.syms  |   5 ++
>>   src/qemu/qemu_driver.c   | 110 +++
>>   src/qemu/qemu_monitor.c  |  24 +++
>>   src/qemu/qemu_monitor.h  |  19 +
>>   src/qemu/qemu_monitor_json.c | 101 
>>   src/qemu/qemu_monitor_json.h |   8 +++
>>   src/remote/remote_driver.c   |   1 +
>>   src/remote/remote_protocol.x |  14 +++-
>>   src/remote_protocol-structs  |   6 ++
>>   tools/virsh-domain-monitor.c |   7 ++
>>   tools/virsh-domain.c |  58 
>>   15 files changed, 471 insertions(+), 3 deletions(-)
>>
> 
> Looks good to me.
> 
> Reviewed-by: Michal Privoznik 
> 
> Will push shortly. Can you please post a follow up patch that adds NEWS.rst 
> entry? I think this deserves to be mention in release notes.
> 
> Michal
> 
> .

Re: [libvirt PATCH v2 6/6] ci: util: Add a registry checker for stale images

[Erik Skultety]

...

> Another thing I'm not a fan of is having the message start at column
> zero instead of being aligned along with the rest of the code...
> textwrap.dedent() could help here, although it'd be a little annoying
> to use because we want the image details to be indented and we can't
> know the correct indent in advance, so we'd have to do a two-step
> process along the lines of
> 
>   msg = textwrap.dedent(f"""
>   The following images are stale and can be purged from the registry:
> 
>   STALE_DETAILS
> 
>   ...
>   """)
>   print(msg.replace("STALE_DETAILS", stale_details))
> 
> A bit more cumbersome, but still preferable when it comes to
> readability IMHO.


So, I figured it would be more convenient to send a new revision rather than
continue the discussion here, so basically apart from ^this last request for
using textwrap.dedent I incorporated all your comments and we can discuss the
text wrapping further in there.

[v3] https://listman.redhat.com/archives/libvir-list/2021-March/msg00892.html

Regards,
Erik

[libvirt PATCH v3 1/6] ci: helper: Use the ArgumentDefaultsHelpFormatter help formatter

[Erik Skultety]

This help formatter class reports the defaults we use for options
taking an argument.

Signed-off-by: Erik Skultety 
Reviewed-by: Andrea Bolognani 
---
 ci/helper | 5 +
 1 file changed, 5 insertions(+)

diff --git a/ci/helper b/ci/helper
index 73a3f729da..8f34f4b59d 100755
--- a/ci/helper
+++ b/ci/helper
@@ -79,6 +79,7 @@ class Parser:
 "build",
 help="run a build in a container",
 parents=[containerparser, mesonparser],
+formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 buildparser.set_defaults(func=Application.action_build)
 
@@ -87,6 +88,7 @@ class Parser:
 "test",
 help="run a build in a container (including tests)",
 parents=[containerparser, mesonparser],
+formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 testparser.set_defaults(func=Application.action_test)
 
@@ -95,6 +97,7 @@ class Parser:
 "shell",
 help="start a shell in a container",
 parents=[containerparser],
+formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 shellparser.set_defaults(func=Application.action_shell)
 
@@ -102,6 +105,7 @@ class Parser:
 listimagesparser = subparsers.add_parser(
 "list-images",
 help="list known container images",
+formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 listimagesparser.set_defaults(func=Application.action_list_images)
 
@@ -110,6 +114,7 @@ class Parser:
 "refresh",
 help="refresh data generated with lcitool",
 parents=[lcitoolparser],
+formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 refreshparser.set_defaults(func=Application.action_refresh)
 
-- 
2.29.2

[libvirt PATCH v3 4/6] ci: helper: Rewrite image listing to Python

[Erik Skultety]

The corresponding Bash script is dropped.
After this patch's rewrite, the Makefile's original image listing
target remains intact only to notify the user to use the Python helper
instead.

Signed-off-by: Erik Skultety 
Reviewed-by: Andrea Bolognani 
---
 ci/Makefile   | 12 
 ci/helper | 38 +-
 ci/list-images.sh | 14 --
 3 files changed, 37 insertions(+), 27 deletions(-)
 delete mode 100644 ci/list-images.sh

diff --git a/ci/Makefile b/ci/Makefile
index 72f5bda942..02ce0df7ee 100644
--- a/ci/Makefile
+++ b/ci/Makefile
@@ -211,17 +211,6 @@ ci-build@%:
 ci-test@%:
$(MAKE) -C $(CI_ROOTDIR) ci-build@$* CI_NINJA_ARGS=test
 
-ci-list-images:
-   @echo
-   @echo "Available x86 container images:"
-   @echo
-   @sh list-images.sh "$(CI_IMAGE_PREFIX)" | grep -v cross
-   @echo
-   @echo "Available cross-compiler container images:"
-   @echo
-   @sh list-images.sh "$(CI_IMAGE_PREFIX)" | grep cross
-   @echo
-
 ci-help:
@echo
@echo
@@ -241,7 +230,6 @@ ci-help:
@echo "ci-build@\$$IMAGE - run a default 'ninja' build"
@echo "ci-test@\$$IMAGE  - run a 'ninja test'"
@echo "ci-shell@\$$IMAGE - run an interactive shell"
-   @echo "ci-list-images  - list available images"
@echo "ci-help - show this help message"
@echo
@echo "Available make variables:"
diff --git a/ci/helper b/ci/helper
index 53baef3cb1..31cf72fbdf 100755
--- a/ci/helper
+++ b/ci/helper
@@ -11,6 +11,8 @@ import shutil
 import subprocess
 import sys
 
+import util
+
 
 class Parser:
 def __init__(self):
@@ -66,6 +68,20 @@ class Parser:
 help="path to lcitool binary",
 )
 
+# Options that are common to actions communicating with a GitLab
+# instance
+gitlabparser = argparse.ArgumentParser(add_help=False)
+gitlabparser.add_argument(
+"--namespace",
+default="libvirt/libvirt",
+help="GitLab project namespace"
+)
+gitlabparser.add_argument(
+"--gitlab-uri",
+default="https://gitlab.com;,
+help="base GitLab URI"
+)
+
 # Main parser
 self.parser = argparse.ArgumentParser()
 subparsers = self.parser.add_subparsers(
@@ -105,6 +121,7 @@ class Parser:
 listimagesparser = subparsers.add_parser(
 "list-images",
 help="list known container images",
+parents=[gitlabparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 listimagesparser.set_defaults(func=Application.action_list_images)
@@ -249,7 +266,26 @@ class Application:
 self.make_run(f"ci-shell@{self.args.target}")
 
 def action_list_images(self):
-self.make_run(f"ci-list-images")
+registry_uri = util.get_registry_uri(self.args.namespace,
+ self.args.gitlab_uri)
+images = util.get_registry_images(registry_uri)
+
+# skip the "ci-" prefix each of our container images' name has
+name_prefix = "ci-"
+names = [i["name"][len(name_prefix):] for i in images]
+names.sort()
+
+native = [name for name in names if "-cross-" not in name]
+cross = [name for name in names if "-cross-" in name]
+
+spacing = 4 * " "
+print("Available x86 container images:\n")
+print(spacing + ("\n" + spacing).join(native))
+
+if cross:
+print()
+print("Available cross-compiler container images:\n")
+print(spacing + ("\n" + spacing).join(cross))
 
 def action_refresh(self):
 self.refresh_containers()
diff --git a/ci/list-images.sh b/ci/list-images.sh
deleted file mode 100644
index b85b132253..00
--- a/ci/list-images.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-prefix="${1##registry.gitlab.com/}"
-
-PROJECT_ID=192693
-
-all_repos() {
-  curl -s 
"https://gitlab.com/api/v4/projects/$PROJECT_ID/registry/repositories?per_page=100;
 \
-| tr , '\n' | grep '"path":' | sed 's,"path":",,g;s,"$,,g'
-}
-
-all_repos | grep "^$prefix" | sed "s,^$prefix,,g" | while read repo; do
-echo "$repo"
-done | sort -u
-- 
2.29.2

[libvirt PATCH v3 3/6] ci: Introduce a util module

[Erik Skultety]

With the gradual rewrite of the Makefile to the 'helper' script will
require helper functions that would better live in a separate util
module.

Signed-off-by: Erik Skultety 
Reviewed-by: Andrea Bolognani 
---
 ci/util.py | 40 
 1 file changed, 40 insertions(+)
 create mode 100644 ci/util.py

diff --git a/ci/util.py b/ci/util.py
new file mode 100644
index 00..f9f8320276
--- /dev/null
+++ b/ci/util.py
@@ -0,0 +1,40 @@
+import json
+import urllib.request
+import urllib.parse
+
+from typing import Dict, List
+
+
+def get_registry_uri(namespace: str,
+ gitlab_uri: str = "https://gitlab.com;) -> str:
+"""
+Construct a v4 API URI pointing the namespaced project's image registry.
+
+:param namespace: GitLab project namespace, e.g. "libvirt/libvirt"
+:param gitlab_uri: GitLab base URI, can be a private deployment
+:param api_version: GitLab REST API version number
+:return: URI pointing to a namespaced project's image registry
+"""
+
+# this converts something like "libvirt/libvirt" to "libvirt%2Flibvirt"
+namespace_urlenc = urllib.parse.quote_plus(namespace)
+
+project_uri = f"{gitlab_uri}/api/v4/projects/{namespace_urlenc}"
+
+uri = project_uri + "/registry/repositories"
+return uri
+
+
+def get_registry_images(uri: str) -> List[Dict]:
+"""
+List all container images that are currently available in the given GitLab
+project.
+
+:param uri: URI pointing to a GitLab instance's image registry
+:return: list of container image names
+"""
+
+r = urllib.request.urlopen(uri + "?per_page=100")
+
+# read the HTTP response and load the JSON part of it
+return json.loads(r.read().decode())
-- 
2.29.2

[libvirt PATCH v3 6/6] ci: helper: Convert private methods to Python common naming practice

[Erik Skultety]

As documented at [1], the common practice wrt to private
attributes/methods naming is to prefix them with an underscore.

[1] https://docs.python.org/3/tutorial/classes.html#private-variables

Signed-off-by: Erik Skultety 
---
 ci/helper | 68 +++
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/ci/helper b/ci/helper
index b5255db835..3d2e0397f5 100755
--- a/ci/helper
+++ b/ci/helper
@@ -97,7 +97,7 @@ class Parser:
 parents=[containerparser, mesonparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
-buildparser.set_defaults(func=Application.action_build)
+buildparser.set_defaults(func=Application._action_build)
 
 # test action
 testparser = subparsers.add_parser(
@@ -106,7 +106,7 @@ class Parser:
 parents=[containerparser, mesonparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
-testparser.set_defaults(func=Application.action_test)
+testparser.set_defaults(func=Application._action_test)
 
 # shell action
 shellparser = subparsers.add_parser(
@@ -115,7 +115,7 @@ class Parser:
 parents=[containerparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
-shellparser.set_defaults(func=Application.action_shell)
+shellparser.set_defaults(func=Application._action_shell)
 
 # list-images action
 listimagesparser = subparsers.add_parser(
@@ -124,7 +124,7 @@ class Parser:
 parents=[gitlabparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
-listimagesparser.set_defaults(func=Application.action_list_images)
+listimagesparser.set_defaults(func=Application._action_list_images)
 
 # refresh action
 refreshparser = subparsers.add_parser(
@@ -146,7 +146,7 @@ class Parser:
 default="yes",
 help="check for existence of stale images on the GitLab instance"
 )
-refreshparser.set_defaults(func=Application.action_refresh)
+refreshparser.set_defaults(func=Application._action_refresh)
 
 def parse(self):
 return self.parser.parse_args()
@@ -161,7 +161,7 @@ class Application:
 if not shutil.which(self.args.lcitool):
 sys.exit("error: 'lcitool' not installed")
 
-def make_run(self, target):
+def _make_run(self, target):
 args = [
 "-C",
 self.basedir,
@@ -185,15 +185,15 @@ class Application:
 if pty.spawn(["make"] + args) != 0:
 sys.exit("error: 'make' failed")
 
-def lcitool_run(self, args):
+def _lcitool_run(self, args):
 output = subprocess.check_output([self.args.lcitool] + args)
 return output.decode("utf-8")
 
-def lcitool_get_hosts(self):
-output = self.lcitool_run(["hosts"])
+def _lcitool_get_hosts(self):
+output = self._lcitool_run(["hosts"])
 return output.splitlines()
 
-def generate_dockerfile(self, host, cross=None):
+def _generate_dockerfile(self, host, cross=None):
 args = ["dockerfile", host, "libvirt"]
 outdir = self.basedir.joinpath("containers")
 outfile = f"ci-{host}.Dockerfile"
@@ -206,11 +206,11 @@ class Application:
 if not self.args.quiet:
 print(outpath)
 
-output = self.lcitool_run(args)
+output = self._lcitool_run(args)
 with open(outpath, "w") as f:
 f.write(output)
 
-def generate_vars(self, host):
+def _generate_vars(self, host):
 args = ["variables", host, "libvirt"]
 outdir = self.basedir.joinpath("cirrus")
 outfile = f"{host}.vars"
@@ -219,11 +219,11 @@ class Application:
 if not self.args.quiet:
 print(outpath)
 
-output = self.lcitool_run(args)
+output = self._lcitool_run(args)
 with open(outpath, "w") as f:
 f.write(output)
 
-def refresh_containers(self):
+def _refresh_containers(self):
 debian_cross = [
 "aarch64",
 "armv6l",
@@ -240,39 +240,39 @@ class Application:
 "mingw64",
 ]
 
-for host in self.lcitool_get_hosts():
+for host in self._lcitool_get_hosts():
 if host.startswith("freebsd-") or host.startswith("macos-"):
 continue
 
-self.generate_dockerfile(host)
+self._generate_dockerfile(host)
 
 if host == "fedora-rawhide":
 for cross in fedora_cross:
-self.generate_dockerfile(host, cross)
+self._generate_dockerfile(host, cross)
 
 if host.startswith("debian-"):
 for cross in debian_cross:
 if host == "debian-sid" and cross == "mips":
 continue
-

[libvirt PATCH v3 0/6] ci: helper: Rewrite image listing in Python and add stale Docker image checker

[Erik Skultety]

since v2:
- dropped original patch 1 renaming ci-help to help in Makefile
- changed the --quiet cmdline option to be tied to the refresh cmd only
- urrlib imports fixed
- improved on Python type hinting
- improved util functions docstrings
- replaced all tabs in formatted strings with 4 spaces
- moved the list_stale_images method to util as requested
- renamed variables in list_stale_images so that it's more clear at first
  glance what they actually hold
- added a hint on where GitLab personal tokens should be generated
- added a patch converting private method naming to reflect the common
  Python practice (I did it in a separate patch so that if this is not
  accepted I only need to ditch a single patch with no further changes)

v1: https://listman.redhat.com/archives/libvir-list/2021-February/msg00641.ht=
ml
v2: https://listman.redhat.com/archives/libvir-list/2021-March/msg00789.html

EXAMPLE OUTPUT OF THE REGISTRY CHECKER:

The following images can be purged from the registry:

ci-debian-9: (ID: 1154661)
ci-debian-9-cross-aarch64: (ID: 1154667)
ci-debian-9-cross-mipsel: (ID: 1154669)
ci-debian-9-cross-armv7l: (ID: 1154671)
ci-debian-9-cross-armv6l: (ID: 1154676)
ci-debian-9-cross-mips: (ID: 1154678)
ci-debian-9-cross-ppc64le: (ID: 1154682)
ci-debian-9-cross-s390x: (ID: 1154683)
ci-debian-9-cross-mips64el: (ID: 1154686)
ci-fedora-31: (ID: 1154687)
ci-opensuse-151: (ID: 1154724)

You can remove the above images over the API with the following code snippet:

$ for image_id in 1154661 1154667 1154669 1154671 1154676 1154678 1154682=
 \
  1154683 1154686 1154687 1154724; do \
  curl --request DELETE --header "PRIVATE-TOKEN: " \
  https://gitlab.com/api/v4/projects/libvirt%2Flibvirt/registry/repos=
itories/$image_id \
  done

You can generate a personal access token here:
https://gitlab.com/-/profile/personal_access_tokens

Erik Skultety (6):
  ci: helper: Use the ArgumentDefaultsHelpFormatter help formatter
  ci: helper: Introduce --quiet
  ci: Introduce a util module
  ci: helper: Rewrite image listing to Python
  ci: util: Add a registry checker for stale images
  ci: helper: Convert private methods to Python common naming practice

 ci/Makefile   |  12 
 ci/helper | 162 --
 ci/list-images.sh |  14 
 ci/util.py|  92 ++
 4 files changed, 219 insertions(+), 61 deletions(-)
 delete mode 100644 ci/list-images.sh
 create mode 100644 ci/util.py

--=20
2.29.2

[libvirt PATCH v3 2/6] ci: helper: Introduce --quiet

[Erik Skultety]

Offer an option to silence all output to stdout, e.g. when generating
dockerfiles/varsfiles.

Signed-off-by: Erik Skultety 
Reviewed-by: Andrea Bolognani 
---
 ci/helper | 12 ++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/ci/helper b/ci/helper
index 8f34f4b59d..53baef3cb1 100755
--- a/ci/helper
+++ b/ci/helper
@@ -116,6 +116,12 @@ class Parser:
 parents=[lcitoolparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
+refreshparser.add_argument(
+"--quiet",
+action="store_true",
+default=False,
+help="refresh data silently"
+)
 refreshparser.set_defaults(func=Application.action_refresh)
 
 def parse(self):
@@ -173,7 +179,8 @@ class Application:
 outfile = f"ci-{host}-cross-{cross}.Dockerfile"
 
 outpath = outdir.joinpath(outfile)
-print(outpath)
+if not self.args.quiet:
+print(outpath)
 
 output = self.lcitool_run(args)
 with open(outpath, "w") as f:
@@ -185,7 +192,8 @@ class Application:
 outfile = f"{host}.vars"
 
 outpath = outdir.joinpath(outfile)
-print(outpath)
+if not self.args.quiet:
+print(outpath)
 
 output = self.lcitool_run(args)
 with open(outpath, "w") as f:
-- 
2.29.2

[libvirt PATCH v3 5/6] ci: util: Add a registry checker for stale images

[Erik Skultety]

This function checks whether there are any stale Docker images in the
registry that can be purged. Since we're pulling available container
images from our GitLab registry with the 'list-images' action, it
could happen that we'd list old (already unsupported) images and make
them available for the user to consume and run a build in them.
Naturally, the build will most likely fail leaving the user confused.

Signed-off-by: Erik Skultety 
---
 ci/helper  | 45 -
 ci/util.py | 52 
 2 files changed, 96 insertions(+), 1 deletion(-)

diff --git a/ci/helper b/ci/helper
index 31cf72fbdf..b5255db835 100755
--- a/ci/helper
+++ b/ci/helper
@@ -130,7 +130,7 @@ class Parser:
 refreshparser = subparsers.add_parser(
 "refresh",
 help="refresh data generated with lcitool",
-parents=[lcitoolparser],
+parents=[lcitoolparser, gitlabparser],
 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 )
 refreshparser.add_argument(
@@ -139,6 +139,13 @@ class Parser:
 default=False,
 help="refresh data silently"
 )
+refreshparser.add_argument(
+"--check-stale",
+action="store",
+choices=["yes", "no"],
+default="yes",
+help="check for existence of stale images on the GitLab instance"
+)
 refreshparser.set_defaults(func=Application.action_refresh)
 
 def parse(self):
@@ -287,10 +294,46 @@ class Application:
 print("Available cross-compiler container images:\n")
 print(spacing + ("\n" + spacing).join(cross))
 
+def check_stale_images(self):
+if self.args.check_stale != "yes" or self.args.quiet:
+return
+
+namespace = self.args.namespace
+gitlab_uri = self.args.gitlab_uri
+registry_uri = util.get_registry_uri(namespace, gitlab_uri)
+lcitool_hosts = self.lcitool_get_hosts()
+
+stale_images = util.get_registry_stale_images(registry_uri,
+  lcitool_hosts)
+if stale_images:
+spacing = "\n" + 4 * " "
+stale_fmt = [f"{k} (ID: {v})" for k, v in stale_images.items()]
+stale_details = spacing + spacing.join(stale_fmt)
+stale_ids = ' '.join([str(id) for id in stale_images.values()])
+registry_uri = util.get_registry_uri(namespace, gitlab_uri)
+
+print(f"""
+The following images are stale and can be purged from the registry:
+{stale_details}
+
+You can remove the above images over the API with the following code snippet:
+
+$ for image_id in {stale_ids}; do \\
+  curl --request DELETE --header "PRIVATE-TOKEN: " \\
+  {registry_uri}/$image_id \\
+  done
+
+You can generate a personal access token here:
+{gitlab_uri}/-/profile/personal_access_tokens""")
+
 def action_refresh(self):
+# refresh Dockerfiles and vars files
 self.refresh_containers()
 self.refresh_cirrus()
 
+# check for stale images
+self.check_stale_images()
+
 def run(self):
 self.args.func(self)
 
diff --git a/ci/util.py b/ci/util.py
index f9f8320276..d69c246872 100644
--- a/ci/util.py
+++ b/ci/util.py
@@ -38,3 +38,55 @@ def get_registry_images(uri: str) -> List[Dict]:
 
 # read the HTTP response and load the JSON part of it
 return json.loads(r.read().decode())
+
+
+def get_image_distro(image_name: str) -> str:
+"""
+Extract the name of the distro in the GitLab image registry name, e.g.
+ci-debian-9-cross-mipsel --> debian-9
+
+:param image_name: name of the GitLab registry image
+:return: distro name as a string
+"""
+name_prefix = "ci-"
+name_suffix = "-cross-"
+
+distro = image_name[len(name_prefix):]
+
+index = distro.find(name_suffix)
+if index > 0:
+distro = distro[:index]
+
+return distro
+
+
+def get_registry_stale_images(registry_uri: str,
+  supported_distros: List[str]) -> Dict[str, int]:
+"""
+Check the GitLab image registry for images that we no longer support and
+which should be deleted.
+
+:param uri: URI pointing to a GitLab instance's image registry
+:param supported_distros: list of hosts supported by lcitool
+:return: dictionary formatted as: {: }
+"""
+
+images = get_registry_images(registry_uri)
+
+# extract distro names from the list of registry images
+registry_distros = [get_image_distro(i["name"]) for i in images]
+
+# - compare the distros powering the images in GitLab registry with
+#   the list of host available from lcitool
+# - @unsupported is a set containing the distro names which we no longer
+#   support; we need to map these back to registry image names
+unsupported = set(registry_distros) - 

[libvirt PATCH 35/38] network_conf: Use virXMLProp(OnOff|YesNo) in virNetworkForwardNatDefParseXML

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/network_conf.c | 15 ++-
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/src/conf/network_conf.c b/src/conf/network_conf.c
index 4cf4aa4840..93972a544a 100644
--- a/src/conf/network_conf.c
+++ b/src/conf/network_conf.c
@@ -1321,7 +1321,6 @@ virNetworkForwardNatDefParseXML(const char *networkName,
 g_autofree xmlNodePtr *natPortNodes = NULL;
 g_autofree char *addrStart = NULL;
 g_autofree char *addrEnd = NULL;
-g_autofree char *ipv6 = NULL;
 VIR_XPATH_NODE_AUTORESTORE(ctxt)
 
 ctxt->node = node;
@@ -1333,18 +1332,8 @@ virNetworkForwardNatDefParseXML(const char *networkName,
 return -1;
 }
 
-ipv6 = virXMLPropString(node, "ipv6");
-if (ipv6) {
-int natIPv6;
-if ((natIPv6 = virTristateBoolTypeFromString(ipv6)) <= 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("Invalid ipv6 setting '%s' "
- "in network '%s' NAT"),
-   ipv6, networkName);
-return -1;
-}
-def->natIPv6 = natIPv6;
-}
+if (virXMLPropYesNo(node, "ipv6", >natIPv6) < 0)
+return -1;
 
 /* addresses for SNAT */
 nNatAddrs = virXPathNodeSet("./address", ctxt, );
-- 
2.26.2

[libvirt PATCH 37/38] storage_adapter_conf: Use virXMLProp(OnOff|YesNo) in virStorageAdapterParseXMLFCHost

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/storage_adapter_conf.c | 17 +
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/conf/storage_adapter_conf.c b/src/conf/storage_adapter_conf.c
index 77ecb8d5f2..96e6c733d0 100644
--- a/src/conf/storage_adapter_conf.c
+++ b/src/conf/storage_adapter_conf.c
@@ -64,26 +64,19 @@ static int
 virStorageAdapterParseXMLFCHost(xmlNodePtr node,
 virStorageAdapterFCHostPtr fchost)
 {
-char *managed = NULL;
+virTristateBool managed = VIR_TRISTATE_BOOL_ABSENT;
 
-fchost->parent = virXMLPropString(node, "parent");
-if ((managed = virXMLPropString(node, "managed"))) {
-if ((fchost->managed = virTristateBoolTypeFromString(managed)) < 0) {
-virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("unknown fc_host managed setting '%s'"),
-   managed);
-VIR_FREE(managed);
-return -1;
-}
-}
+if (virXMLPropYesNo(node, "managed", ) < 0)
+return -1;
 
+fchost->parent = virXMLPropString(node, "parent");
+fchost->managed = managed;
 fchost->parent_wwnn = virXMLPropString(node, "parent_wwnn");
 fchost->parent_wwpn = virXMLPropString(node, "parent_wwpn");
 fchost->parent_fabric_wwn = virXMLPropString(node, "parent_fabric_wwn");
 fchost->wwpn = virXMLPropString(node, "wwpn");
 fchost->wwnn = virXMLPropString(node, "wwnn");
 
-VIR_FREE(managed);
 return 0;
 }
 
-- 
2.26.2

[libvirt PATCH 38/38] storage_conf: Use virXMLProp(OnOff|YesNo) in virStoragePoolDefParseSource

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/storage_conf.c | 16 +---
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
index 2e07c81f8a..7dc528b103 100644
--- a/src/conf/storage_conf.c
+++ b/src/conf/storage_conf.c
@@ -605,7 +605,7 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
 goto cleanup;
 
 for (i = 0; i < nsource; i++) {
-g_autofree char *partsep = NULL;
+virTristateBool partsep = VIR_TRISTATE_BOOL_ABSENT;
 virStoragePoolSourceDevice dev = { .path = NULL };
 dev.path = virXMLPropString(nodeset[i], "path");
 
@@ -615,17 +615,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
 goto cleanup;
 }
 
-partsep = virXMLPropString(nodeset[i], "part_separator");
-if (partsep) {
-dev.part_separator = virTristateBoolTypeFromString(partsep);
-if (dev.part_separator <= 0) {
-virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("invalid part_separator setting '%s'"),
-   partsep);
-virStoragePoolSourceDeviceClear();
-goto cleanup;
-}
+if (virXMLPropYesNo(nodeset[i], "part_separator", ) < 0) {
+virStoragePoolSourceDeviceClear();
+goto cleanup;
 }
+dev.part_separator = partsep;
 
 if (VIR_APPEND_ELEMENT(source->devices, source->ndevice, dev) < 0) {
 virStoragePoolSourceDeviceClear();
-- 
2.26.2

[libvirt PATCH 24/38] domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainShmemDefParseXML

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/domain_conf.c | 13 ++---
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 0c4f7ab879..a1aefdf0bf 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -14413,17 +14413,8 @@ virDomainShmemDefParseXML(virDomainXMLOptionPtr xmlopt,
 }
 VIR_FREE(tmp);
 
-if ((tmp = virXMLPropString(msi, "ioeventfd"))) {
-int val;
-
-if ((val = virTristateSwitchTypeFromString(tmp)) <= 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("invalid msi ioeventfd setting for shmem: 
'%s'"),
-   tmp);
-goto cleanup;
-}
-def->msi.ioeventfd = val;
-}
+if (virXMLPropOnOff(msi, "ioeventfd", >msi.ioeventfd) < 0)
+goto cleanup;
 }
 
 /* msi option is only relevant with a server */
-- 
2.26.2

[libvirt PATCH 29/38] domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainFeaturesDefParse

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/domain_conf.c | 138 ++---
 1 file changed, 48 insertions(+), 90 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 39f171ba74..4f020714bc 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -18292,6 +18292,7 @@ virDomainFeaturesDefParse(virDomainDefPtr def,
 
 for (i = 0; i < n; i++) {
 g_autofree char *tmp = NULL;
+virTristateSwitch triSwitch = VIR_TRISTATE_SWITCH_ABSENT;
 int val = virDomainFeatureTypeFromString((const char *)nodes[i]->name);
 if (val < 0) {
 virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
@@ -18301,16 +18302,9 @@ virDomainFeaturesDefParse(virDomainDefPtr def,
 
 switch ((virDomainFeature) val) {
 case VIR_DOMAIN_FEATURE_APIC:
-if ((tmp = virXPathString("string(./features/apic/@eoi)", ctxt))) {
-int eoi;
-if ((eoi = virTristateSwitchTypeFromString(tmp)) <= 0) {
-virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("unknown value for attribute eoi: '%s'"),
-   tmp);
-return -1;
-}
-def->apic_eoi = eoi;
-}
+if (virXMLPropOnOff(nodes[i], "eoi", ) < 0)
+return -1;
+def->apic_eoi = triSwitch;
 G_GNUC_FALLTHROUGH;
 case VIR_DOMAIN_FEATURE_ACPI:
 case VIR_DOMAIN_FEATURE_PAE:
@@ -18342,16 +18336,10 @@ virDomainFeaturesDefParse(virDomainDefPtr def,
 case VIR_DOMAIN_FEATURE_PVSPINLOCK:
 case VIR_DOMAIN_FEATURE_VMPORT:
 case VIR_DOMAIN_FEATURE_SMM:
-if ((tmp = virXMLPropString(nodes[i], "state"))) {
-if ((def->features[val] = 
virTristateSwitchTypeFromString(tmp)) == -1) {
-virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("unknown state attribute '%s' of feature 
'%s'"),
-   tmp, virDomainFeatureTypeToString(val));
-return -1;
-}
-} else {
+if (virXMLPropOnOff(nodes[i], "state", ) < 0)
+return -1;
+if ((def->features[val] = triSwitch) == VIR_TRISTATE_SWITCH_ABSENT)
 def->features[val] = VIR_TRISTATE_SWITCH_ON;
-}
 break;
 
 case VIR_DOMAIN_FEATURE_GIC:
@@ -18459,15 +18447,11 @@ virDomainFeaturesDefParse(virDomainDefPtr def,
 case VIR_DOMAIN_FEATURE_HTM:
 case VIR_DOMAIN_FEATURE_NESTED_HV:
 case VIR_DOMAIN_FEATURE_CCF_ASSIST:
-if (!(tmp = virXMLPropString(nodes[i], "state"))) {
-virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("missing state attribute '%s' of feature 
'%s'"),
-   tmp, virDomainFeatureTypeToString(val));
+if (virXMLPropOnOff(nodes[i], "state", ) < 0)
 return -1;
-}
-if ((def->features[val] = virTristateSwitchTypeFromString(tmp)) < 
0) {
+if ((def->features[val] = triSwitch) == 
VIR_TRISTATE_SWITCH_ABSENT) {
 virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("unknown state attribute '%s' of feature 
'%s'"),
+   _("missing state attribute '%s' of feature 
'%s'"),
tmp, virDomainFeatureTypeToString(val));
 return -1;
 }
@@ -18482,13 +18466,12 @@ virDomainFeaturesDefParse(virDomainDefPtr def,
 
 if (def->features[VIR_DOMAIN_FEATURE_HYPERV] == VIR_TRISTATE_SWITCH_ON) {
 int feature;
-int value;
+virTristateSwitch state = VIR_TRISTATE_SWITCH_ABSENT;
 xmlNodePtr node = ctxt->node;
 if ((n = virXPathNodeSet("./features/hyperv/*", ctxt, )) < 0)
 return -1;
 
 for (i = 0; i < n; i++) {
-g_autofree char *tmp = NULL;
 feature = virDomainHypervTypeFromString((const char 
*)nodes[i]->name);
 if (feature < 0) {
 virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
@@ -18499,7 +18482,10 @@ virDomainFeaturesDefParse(virDomainDefPtr def,
 
 ctxt->node = nodes[i];
 
-if (!(tmp = virXMLPropString(nodes[i], "state"))) {
+if (virXMLPropOnOff(nodes[i], "state", ) < 0)
+return -1;
+
+if (state == VIR_TRISTATE_SWITCH_ABSENT) {
 virReportError(VIR_ERR_XML_ERROR,
_("missing 'state' attribute for "
  "HyperV Enlightenment feature '%s'"),
@@ -18507,15 +18493,7 @@ virDomainFeaturesDefParse(virDomainDefPtr def,
 return -1;
 }
 
-if ((value = virTristateSwitchTypeFromString(tmp)) < 0) {
-

[libvirt PATCH 23/38] domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainMemballoonDefParseXML

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/domain_conf.c | 18 ++
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index fe35a3960e..0c4f7ab879 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -14264,8 +14264,8 @@ virDomainMemballoonDefParseXML(virDomainXMLOptionPtr 
xmlopt,
 VIR_XPATH_NODE_AUTORESTORE(ctxt)
 unsigned int period = 0;
 g_autofree char *model = NULL;
-g_autofree char *freepage_reporting = NULL;
-g_autofree char *deflate = NULL;
+virTristateSwitch autodeflate = VIR_TRISTATE_SWITCH_ABSENT;
+virTristateSwitch free_page_reporting = VIR_TRISTATE_SWITCH_ABSENT;
 
 def = g_new0(virDomainMemballoonDef, 1);
 
@@ -14282,19 +14282,13 @@ virDomainMemballoonDefParseXML(virDomainXMLOptionPtr 
xmlopt,
 goto error;
 }
 
-if ((deflate = virXMLPropString(node, "autodeflate")) &&
-(def->autodeflate = virTristateSwitchTypeFromString(deflate)) <= 0) {
-virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("invalid autodeflate attribute value '%s'"), deflate);
+if (virXMLPropOnOff(node, "autodeflate", ) < 0)
 goto error;
-}
+def->autodeflate = autodeflate;
 
-if ((freepage_reporting = virXMLPropString(node, "freePageReporting")) &&
-(def->free_page_reporting = 
virTristateSwitchTypeFromString(freepage_reporting)) <= 0) {
-virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("invalid freePageReporting attribute value '%s'"), 
freepage_reporting);
+if (virXMLPropOnOff(node, "freePageReporting", _page_reporting) < 0)
 goto error;
-}
+def->free_page_reporting = free_page_reporting;
 
 ctxt->node = node;
 if (virXPathUInt("string(./stats/@period)", ctxt, ) < -1) {
-- 
2.26.2

[libvirt PATCH 36/38] numa_conf: Use virXMLProp(OnOff|YesNo) in virDomainNumaDefParseXML

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/numa_conf.c | 13 ++---
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c
index 64b93fd7d1..d16a44c955 100644
--- a/src/conf/numa_conf.c
+++ b/src/conf/numa_conf.c
@@ -1079,17 +1079,8 @@ virDomainNumaDefParseXML(virDomainNumaPtr def,
 VIR_FREE(tmp);
 }
 
-if ((tmp = virXMLPropString(nodes[i], "discard"))) {
-if ((rc = virTristateBoolTypeFromString(tmp)) <= 0) {
-virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("Invalid 'discard' attribute value '%s'"),
-   tmp);
-goto cleanup;
-}
-
-def->mem_nodes[cur_cell].discard = rc;
-VIR_FREE(tmp);
-}
+if (virXMLPropYesNo(nodes[i], "discard", 
>mem_nodes[cur_cell].discard) < 0)
+goto cleanup;
 
 /* Parse NUMA distances info */
 if (virDomainNumaDefNodeDistanceParseXML(def, ctxt, cur_cell) < 0)
-- 
2.26.2

[libvirt PATCH 08/38] domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainDiskSourceNVMeParse

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/domain_conf.c | 13 -
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 55920e7a7d..e3e38ae30d 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -8428,7 +8428,7 @@ virDomainDiskSourceNVMeParse(xmlNodePtr node,
 g_autoptr(virStorageSourceNVMeDef) nvme = NULL;
 g_autofree char *type = NULL;
 g_autofree char *namespc = NULL;
-g_autofree char *managed = NULL;
+virTristateBool managed = VIR_TRISTATE_BOOL_ABSENT;
 xmlNodePtr address;
 
 nvme = g_new0(virStorageSourceNVMeDef, 1);
@@ -8459,14 +8459,9 @@ virDomainDiskSourceNVMeParse(xmlNodePtr node,
 return -1;
 }
 
-if ((managed = virXMLPropString(node, "managed"))) {
-if ((nvme->managed = virTristateBoolTypeFromString(managed)) <= 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("malformed managed value '%s'"),
-   managed);
-return -1;
-}
-}
+if (virXMLPropYesNo(node, "managed", ) < 0)
+return -1;
+nvme->managed = managed;
 
 if (!(address = virXPathNode("./address", ctxt))) {
 virReportError(VIR_ERR_XML_ERROR, "%s",
-- 
2.26.2

[libvirt PATCH 34/38] device_conf: Use virXMLProp(OnOff|YesNo) in virPCIDeviceAddressParseXML

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/device_conf.c | 10 +++---
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/conf/device_conf.c b/src/conf/device_conf.c
index 714ac50762..ee248648bd 100644
--- a/src/conf/device_conf.c
+++ b/src/conf/device_conf.c
@@ -214,7 +214,7 @@ virPCIDeviceAddressParseXML(xmlNodePtr node,
 g_autofree char *bus  = virXMLPropString(node, "bus");
 g_autofree char *slot = virXMLPropString(node, "slot");
 g_autofree char *function = virXMLPropString(node, "function");
-g_autofree char *multi= virXMLPropString(node, "multifunction");
+virTristateSwitch multifunction = VIR_TRISTATE_SWITCH_ABSENT;
 
 memset(addr, 0, sizeof(*addr));
 
@@ -246,14 +246,10 @@ virPCIDeviceAddressParseXML(xmlNodePtr node,
 return -1;
 }
 
-if (multi &&
-((addr->multi = virTristateSwitchTypeFromString(multi)) <= 0)) {
-virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-   _("Unknown value '%s' for  'multifunction' 
attribute"),
-   multi);
+if (virXMLPropOnOff(node, "multifunction", ) < 0)
 return -1;
+addr->multi = multifunction;
 
-}
 if (!virPCIDeviceAddressIsEmpty(addr) && !virPCIDeviceAddressIsValid(addr, 
true))
 return -1;
 
-- 
2.26.2

[libvirt PATCH 22/38] domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainAudioDefParseXML

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/domain_conf.c | 14 ++
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 6559c3d2f8..fe35a3960e 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -14025,23 +14025,13 @@ virDomainAudioDefParseXML(virDomainXMLOptionPtr 
xmlopt G_GNUC_UNUSED,
 break;
 
 case VIR_DOMAIN_AUDIO_TYPE_OSS: {
-g_autofree char *tryMMap = virXMLPropString(node, "tryMMap");
-g_autofree char *exclusive = virXMLPropString(node, "exclusive");
 g_autofree char *dspPolicy = virXMLPropString(node, "dspPolicy");
 
-if (tryMMap && ((def->backend.oss.tryMMap =
- virTristateBoolTypeFromString(tryMMap)) <= 0)) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("unknown 'tryMMap' value '%s'"), tryMMap);
+if (virXMLPropYesNo(node, "tryMMap", >backend.oss.tryMMap) < 0)
 goto error;
-}
 
-if (exclusive && ((def->backend.oss.exclusive =
-   virTristateBoolTypeFromString(exclusive)) <= 0)) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("unknown 'exclusive' value '%s'"), exclusive);
+if (virXMLPropYesNo(node, "exclusive", >backend.oss.exclusive) < 
0)
 goto error;
-}
 
 if (dspPolicy) {
 if (virStrToLong_i(dspPolicy, NULL, 10,
-- 
2.26.2

[libvirt PATCH 33/38] backup_conf: Use virXMLProp(OnOff|YesNo) in virDomainBackupDefParse

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/backup_conf.c | 16 ++--
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/src/conf/backup_conf.c b/src/conf/backup_conf.c
index 8560f3bf2e..3a1863d2c3 100644
--- a/src/conf/backup_conf.c
+++ b/src/conf/backup_conf.c
@@ -234,8 +234,6 @@ virDomainBackupDefParse(xmlXPathContextPtr ctxt,
 def->incremental = virXPathString("string(./incremental)", ctxt);
 
 if ((node = virXPathNode("./server", ctxt))) {
-g_autofree char *tls = NULL;
-
 if (def->type != VIR_DOMAIN_BACKUP_TYPE_PULL) {
 virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("use of  requires pull mode backup"));
@@ -261,18 +259,8 @@ virDomainBackupDefParse(xmlXPathContextPtr ctxt,
 return NULL;
 }
 
-if ((tls = virXMLPropString(node, "tls"))) {
-int tmp;
-
-if ((tmp = virTristateBoolTypeFromString(tls)) <= 0) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("unknown value '%s' of 'tls' attribute"),\
-   tls);
-return NULL;
-}
-
-def->tls = tmp;
-}
+if (virXMLPropYesNo(node, "tls", >tls) < 0)
+return NULL;
 }
 
 if ((n = virXPathNodeSet("./disks/*", ctxt, )) < 0)
-- 
2.26.2

[libvirt PATCH 25/38] domain_conf: Use virXMLProp(OnOff|YesNo) in virDomainPerfEventDefParseXML

[Tim Wiederhake]

Signed-off-by: Tim Wiederhake 
---
 src/conf/domain_conf.c | 12 
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index a1aefdf0bf..c020bf9124 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -15496,7 +15496,7 @@ virDomainPerfEventDefParseXML(virDomainPerfDefPtr perf,
 {
 int event;
 g_autofree char *name = NULL;
-g_autofree char *enabled = NULL;
+virTristateBool enabled = VIR_TRISTATE_BOOL_ABSENT;
 
 if (!(name = virXMLPropString(node, "name"))) {
 virReportError(VIR_ERR_XML_ERROR, "%s", _("missing perf event name"));
@@ -15515,16 +15515,12 @@ virDomainPerfEventDefParseXML(virDomainPerfDefPtr 
perf,
 return -1;
 }
 
-if (!(enabled = virXMLPropString(node, "enabled"))) {
-virReportError(VIR_ERR_XML_ERROR,
-   _("missing state of perf event '%s'"), name);
+if (virXMLPropYesNo(node, "enabled", ) < 0)
 return -1;
-}
 
-if ((perf->events[event] = virTristateBoolTypeFromString(enabled)) < 0) {
+if ((perf->events[event] = enabled) == VIR_TRISTATE_BOOL_ABSENT) {
 virReportError(VIR_ERR_XML_ERROR,
-   _("invalid state '%s' of perf event '%s'"),
-   enabled, name);
+   _("missing state of perf event '%s'"), name);
 return -1;
 }
 
-- 
2.26.2