[PATCH RFC 10/32] python/qemu: Add flake8 to Pipfile

[John Snow]

Signed-off-by: John Snow 
---
 python/Pipfile  |  1 +
 python/Pipfile.lock | 31 ++-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/python/Pipfile b/python/Pipfile
index ddb2b5a518..e396e56e06 100644
--- a/python/Pipfile
+++ b/python/Pipfile
@@ -5,6 +5,7 @@ verify_ssl = true
 
 [dev-packages]
 pylint = "==2.5.0"
+flake8 = "*"
 
 [packages]
 
diff --git a/python/Pipfile.lock b/python/Pipfile.lock
index e6faa832e4..15cd8a918f 100644
--- a/python/Pipfile.lock
+++ b/python/Pipfile.lock
@@ -1,7 +1,7 @@
 {
 "_meta": {
 "hash": {
-"sha256": 
"7815dedfd7481b645389153dd45e9adb82c72956d0efc74d8f087497624b75e4"
+"sha256": 
"58115144ace0f646b5c62da260fb4867ac0a0e485de3f5b0a56c7854afa21f5b"
 },
 "pipfile-spec": 6,
 "requires": {
@@ -24,6 +24,21 @@
 ],
 "version": "==2.4.1"
 },
+"entrypoints": {
+"hashes": [
+
"sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19",
+
"sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"
+],
+"version": "==0.3"
+},
+"flake8": {
+"hashes": [
+
"sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb",
+
"sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca"
+],
+"index": "pypi",
+"version": "==3.7.9"
+},
 "isort": {
 "hashes": [
 
"sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
@@ -64,6 +79,20 @@
 ],
 "version": "==0.6.1"
 },
+"pycodestyle": {
+"hashes": [
+
"sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56",
+
"sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"
+],
+"version": "==2.5.0"
+},
+"pyflakes": {
+"hashes": [
+
"sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0",
+
"sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"
+],
+"version": "==2.1.1"
+},
 "pylint": {
 "hashes": [
 
"sha256:588e114e3f9a1630428c35b7dd1c82c1c93e1b0e78ee312ae4724c5e1a1e0245",
-- 
2.21.1

[PATCH RFC 13/32] python/qemu/lib: Adjust traceback typing

[John Snow]

mypy considers it incorrect to use `bool` to statically return false,
because it will assume that it could conceivably return True, and gives
different analysis in that case. Use a None return to achieve the same
effect, but make mypy happy.

Note: Pylint considers function signatures as code that might trip the
duplicate-code checker. I'd rather not disable this as it does not
trigger often in practice, so I'm disabling it as a one-off and filed a
change request; see https://github.com/PyCQA/pylint/issues/3619

Signed-off-by: John Snow 
---
 python/qemu/lib/machine.py |  8 ++--
 python/qemu/lib/qmp.py | 10 --
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/python/qemu/lib/machine.py b/python/qemu/lib/machine.py
index b2f0412197..2f94c851ed 100644
--- a/python/qemu/lib/machine.py
+++ b/python/qemu/lib/machine.py
@@ -24,6 +24,8 @@
 import shutil
 import socket
 import tempfile
+from typing import Optional, Type
+from types import TracebackType
 
 from . import qmp
 
@@ -127,9 +129,11 @@ def __init__(self, binary, args=None, wrapper=None, 
name=None,
 def __enter__(self):
 return self
 
-def __exit__(self, exc_type, exc_val, exc_tb):
+def __exit__(self,
+ exc_type: Optional[Type[BaseException]],
+ exc_val: Optional[BaseException],
+ exc_tb: Optional[TracebackType]) -> None:
 self.shutdown()
-return False
 
 def add_monitor_null(self):
 """
diff --git a/python/qemu/lib/qmp.py b/python/qemu/lib/qmp.py
index 73d49050ed..b91c9d5c1c 100644
--- a/python/qemu/lib/qmp.py
+++ b/python/qemu/lib/qmp.py
@@ -14,7 +14,9 @@
 from typing import (
 Optional,
 TextIO,
+Type,
 )
+from types import TracebackType
 
 
 class QMPError(Exception):
@@ -146,10 +148,14 @@ def __enter__(self):
 # Implement context manager enter function.
 return self
 
-def __exit__(self, exc_type, exc_value, exc_traceback):
+def __exit__(self,
+ # pylint: disable=duplicate-code
+ # see https://github.com/PyCQA/pylint/issues/3619
+ exc_type: Optional[Type[BaseException]],
+ exc_val: Optional[BaseException],
+ exc_tb: Optional[TracebackType]) -> None:
 # Implement context manager exit function.
 self.close()
-return False
 
 def connect(self, negotiate=True):
 """
-- 
2.21.1

[PATCH RFC 11/32] python/qemu/lib: remove Python2 style super() calls

[John Snow]

Use the Python3 style instead.

Signed-off-by: John Snow 
---
 python/qemu/lib/machine.py |  2 +-
 python/qemu/lib/qtest.py   | 15 +++
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/python/qemu/lib/machine.py b/python/qemu/lib/machine.py
index 4b260fa2cb..b2f0412197 100644
--- a/python/qemu/lib/machine.py
+++ b/python/qemu/lib/machine.py
@@ -55,7 +55,7 @@ def __init__(self, reply):
 desc = reply["error"]["desc"]
 except KeyError:
 desc = reply
-super(MonitorResponseError, self).__init__(desc)
+super().__init__(desc)
 self.reply = reply
 
 
diff --git a/python/qemu/lib/qtest.py b/python/qemu/lib/qtest.py
index 53d814c064..7943487c2b 100644
--- a/python/qemu/lib/qtest.py
+++ b/python/qemu/lib/qtest.py
@@ -101,29 +101,28 @@ def __init__(self, binary, args=None, name=None, 
test_dir="/var/tmp",
 name = "qemu-%d" % os.getpid()
 if sock_dir is None:
 sock_dir = test_dir
-super(QEMUQtestMachine,
-  self).__init__(binary, args, name=name, test_dir=test_dir,
- socket_scm_helper=socket_scm_helper,
- sock_dir=sock_dir)
+super().__init__(binary, args, name=name, test_dir=test_dir,
+ socket_scm_helper=socket_scm_helper,
+ sock_dir=sock_dir)
 self._qtest = None
 self._qtest_path = os.path.join(sock_dir, name + "-qtest.sock")
 
 def _base_args(self):
-args = super(QEMUQtestMachine, self)._base_args()
+args = super()._base_args()
 args.extend(['-qtest', 'unix:path=' + self._qtest_path,
  '-accel', 'qtest'])
 return args
 
 def _pre_launch(self):
-super(QEMUQtestMachine, self)._pre_launch()
+super()._pre_launch()
 self._qtest = QEMUQtestProtocol(self._qtest_path, server=True)
 
 def _post_launch(self):
-super(QEMUQtestMachine, self)._post_launch()
+super()._post_launch()
 self._qtest.accept()
 
 def _post_shutdown(self):
-super(QEMUQtestMachine, self)._post_shutdown()
+super()._post_shutdown()
 self._remove_if_exists(self._qtest_path)
 
 def qtest(self, cmd):
-- 
2.21.1

[PATCH RFC 05/32] python/qemu/lib: delint; add flake8 config

[John Snow]

Mostly, ignore the "no bare except" rule, because flake8 is not
contextual and cannot determine if we re-raise. Pylint can, though, so
always prefer pylint for that.

Signed-off-by: John Snow 
---
 python/qemu/lib/.flake8|  2 ++
 python/qemu/lib/accel.py   |  9 ++---
 python/qemu/lib/machine.py | 13 +
 python/qemu/lib/qmp.py |  4 ++--
 4 files changed, 19 insertions(+), 9 deletions(-)
 create mode 100644 python/qemu/lib/.flake8

diff --git a/python/qemu/lib/.flake8 b/python/qemu/lib/.flake8
new file mode 100644
index 00..45d8146f3f
--- /dev/null
+++ b/python/qemu/lib/.flake8
@@ -0,0 +1,2 @@
+[flake8]
+extend-ignore = E722  # Pylint handles this, but smarter.
\ No newline at end of file
diff --git a/python/qemu/lib/accel.py b/python/qemu/lib/accel.py
index 36ae85791e..7fabe62920 100644
--- a/python/qemu/lib/accel.py
+++ b/python/qemu/lib/accel.py
@@ -23,11 +23,12 @@
 # Mapping host architecture to any additional architectures it can
 # support which often includes its 32 bit cousin.
 ADDITIONAL_ARCHES = {
-"x86_64" : "i386",
-"aarch64" : "armhf",
-"ppc64le" : "ppc64",
+"x86_64": "i386",
+"aarch64": "armhf",
+"ppc64le": "ppc64",
 }
 
+
 def list_accel(qemu_bin):
 """
 List accelerators enabled in the QEMU binary.
@@ -47,6 +48,7 @@ def list_accel(qemu_bin):
 # Skip the first line which is the header.
 return [acc.strip() for acc in out.splitlines()[1:]]
 
+
 def kvm_available(target_arch=None, qemu_bin=None):
 """
 Check if KVM is available using the following heuristic:
@@ -69,6 +71,7 @@ def kvm_available(target_arch=None, qemu_bin=None):
 return False
 return True
 
+
 def tcg_available(qemu_bin):
 """
 Check if TCG is available.
diff --git a/python/qemu/lib/machine.py b/python/qemu/lib/machine.py
index c79fc8fb89..4b260fa2cb 100644
--- a/python/qemu/lib/machine.py
+++ b/python/qemu/lib/machine.py
@@ -29,6 +29,7 @@
 
 LOG = logging.getLogger(__name__)
 
+
 class QEMUMachineError(Exception):
 """
 Exception called when an error in QEMUMachine happens.
@@ -62,7 +63,8 @@ class QEMUMachine:
 """
 A QEMU VM
 
-Use this object as a context manager to ensure the QEMU process 
terminates::
+Use this object as a context manager to ensure
+the QEMU process terminates::
 
 with VM(binary) as vm:
 ...
@@ -188,8 +190,10 @@ def send_fd_scm(self, fd=None, file_path=None):
 fd_param.append(str(fd))
 
 devnull = open(os.path.devnull, 'rb')
-proc = subprocess.Popen(fd_param, stdin=devnull, 
stdout=subprocess.PIPE,
-stderr=subprocess.STDOUT, close_fds=False)
+proc = subprocess.Popen(
+fd_param, stdin=devnull, stdout=subprocess.PIPE,
+stderr=subprocess.STDOUT, close_fds=False
+)
 output = proc.communicate()[0]
 if output:
 LOG.debug(output)
@@ -491,7 +495,8 @@ def event_wait(self, name, timeout=60.0, match=None):
 
 def events_wait(self, events, timeout=60.0):
 """
-events_wait waits for and returns a named event from QMP with a 
timeout.
+events_wait waits for and returns a named event
+from QMP with a timeout.
 
 events: a sequence of (name, match_criteria) tuples.
 The match criteria are optional and may be None.
diff --git a/python/qemu/lib/qmp.py b/python/qemu/lib/qmp.py
index d6c9b2f4b1..6ae7693965 100644
--- a/python/qemu/lib/qmp.py
+++ b/python/qemu/lib/qmp.py
@@ -168,8 +168,8 @@ def accept(self, timeout=15.0):
 
 @param timeout: timeout in seconds (nonnegative float number, or
 None). The value passed will set the behavior of the
-underneath QMP socket as described in [1]. Default 
value
-is set to 15.0.
+underneath QMP socket as described in [1].
+Default value is set to 15.0.
 @return QMP greeting dict
 @raise OSError on socket connection errors
 @raise QMPConnectError if the greeting is not received
-- 
2.21.1

[PATCH RFC 08/32] python/qemu: Add Pipfile

[John Snow]

pipenv is a tool used for managing virtual environments with precisely
specified dependencies. It is separate from the dependencies listed in
setup.py, which are not pinned to precise versions.

Note that pipenv is not required to install or use this module; this is
just a convenience for in-tree developing.

Here, a "blank" pipfile is added with no dependencies, but specifies
Python 3.6 for the virtual environment.

Signed-off-by: John Snow 
---
 python/Pipfile | 11 +++
 1 file changed, 11 insertions(+)
 create mode 100644 python/Pipfile

diff --git a/python/Pipfile b/python/Pipfile
new file mode 100644
index 00..9534830b5e
--- /dev/null
+++ b/python/Pipfile
@@ -0,0 +1,11 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple;
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+
+[requires]
+python_version = "3.6"
-- 
2.21.1

[PATCH RFC 07/32] python/qemu: add README.rst

[John Snow]

Add a short readme that explains the package hierarchy, which will be
visible while browsing the source on e.g. gitlab/github.

Signed-off-by: John Snow 
---
 python/qemu/README.rst | 8 
 1 file changed, 8 insertions(+)
 create mode 100644 python/qemu/README.rst

diff --git a/python/qemu/README.rst b/python/qemu/README.rst
new file mode 100644
index 00..96447b9061
--- /dev/null
+++ b/python/qemu/README.rst
@@ -0,0 +1,8 @@
+python/qemu/
+
+
+This directory serves as the root of a `Python PEP 420 implicit
+namespace package <`_.
+
+Each directory below is assumed to be an installable Python module that
+will be available under the ``qemu.`` namespace.
-- 
2.21.1

[PATCH RFC 03/32] python//machine.py: remove bare except

[John Snow]

Catch only the timeout error; if there are other problems, allow the
stack trace to be visible.

Signed-off-by: John Snow 
---
 python/qemu/lib/machine.py | 33 +
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/python/qemu/lib/machine.py b/python/qemu/lib/machine.py
index b9a98e2c86..e3ea523571 100644
--- a/python/qemu/lib/machine.py
+++ b/python/qemu/lib/machine.py
@@ -342,7 +342,26 @@ def wait(self):
 self._load_io_log()
 self._post_shutdown()
 
-def shutdown(self, has_quit=False):
+def _issue_shutdown(self, has_quit: bool = False) -> None:
+"""
+Shutdown the VM.
+"""
+if not self.is_running():
+return
+
+if self._qmp is not None:
+if not has_quit:
+self._qmp.cmd('quit')
+self._qmp.close()
+
+try:
+self._popen.wait(timeout=3)
+except subprocess.TimeoutExpired:
+self._popen.kill()
+
+self._popen.wait()
+
+def shutdown(self, has_quit: bool = False) -> None:
 """
 Terminate the VM and clean up
 """
@@ -353,17 +372,7 @@ def shutdown(self, has_quit=False):
 self._console_socket.close()
 self._console_socket = None
 
-if self.is_running():
-if self._qmp:
-try:
-if not has_quit:
-self._qmp.cmd('quit')
-self._qmp.close()
-self._popen.wait(timeout=3)
-except:
-self._popen.kill()
-self._popen.wait()
-
+self._issue_shutdown(has_quit)
 self._load_io_log()
 self._post_shutdown()
 
-- 
2.21.1

[PATCH RFC 00/32] python/qemu: refactor as installable package

[John Snow]

Hey, I got lost on my way to the store and I accidentally got 32 patches
that convert our python library into something that passes pylint,
flake8, and mypy --strict.

...So, a few things:

1. This is just an RFC. The actual design of these libraries really
needs adjusted to be more pythonic. In general this means less
Optional[T] return types and raising more exceptions. This could be
handled later, but we ought to address it before publishing, if we do.

2. We still need to think carefully about how we package this, if we
even want to package it, what the license on the top-level package
should be, etc.

3. We should consider how to version it. For now, I'm using a lockstep
versioning.

4. You can install this package using pip3 or python3 setup.py to a
virtual environment or to your real one. From there, any python code in
the QEMU tree that imports these modules will work with no sys.path
hacking or custom PYTHONPATH exports.

5. You don't have to install it, though. I left all of the usual hacks
in place in the rest of the tree so that everything will just keep
working exactly as-is for right now. It's just that you COULD install it.

6. Here's a cool trick if you don't know about it yet:

> cd qemu/python/qemu
> pip3 install --user -e .

This will install the package in "develop" mode, which installs a
forwarder package. When you update your source tree, the installed
package stays "up to date" with the most recent edits.

Alright, have fun, stay safe!

John Snow (32):
  python/qemu: create qemu.lib module
  scripts/qmp: Fix shebang and imports
  python//machine.py: remove bare except
  python/qemu/lib: delint, add pylintrc
  python/qemu/lib: delint; add flake8 config
  python/qemu: formalize as package
  python/qemu: add README.rst
  python/qemu: Add Pipfile
  python/qemu: add pylint to Pipfile
  python/qemu: Add flake8 to Pipfile
  python/qemu/lib: remove Python2 style super() calls
  python/qemu/lib: fix socket.makefile() typing
  python/qemu/lib: Adjust traceback typing
  python//qmp.py: use True/False for non/blocking modes
  python//qmp.py: Define common types
  python//qmp.py: re-absorb MonitorResponseError
  python//qmp.py: Do not return None from cmd_obj
  python//qmp.py: add casts to JSON deserialization
  python//qmp.py: add QMPProtocolError
  python//qmp.py: assert sockfile is not None
  python//machine.py: remove logging configuration
  python//machine.py: Fix monitor address typing
  python//machine.py: reorder __init__
  python//machine.py: Don't modify state in _base_args()
  python//machine.py: Handle None events in event_wait
  python//machine.py: use qmp.command
  python//machine.py: Add _qmp access shim
  python//machine.py: fix _popen access
  python//qtest.py: Check before accessing _qtest
  python/qemu/lib: make 'args' style arguments immutable
  python/qemu: add mypy to Pipfile
  python/qemu/lib: Add mypy type annotations

 python/README.rst |   6 +
 python/qemu/README.rst|   8 +
 python/Pipfile|  14 +
 python/Pipfile.lock   | 187 +
 python/qemu/__init__.py   |  11 -
 python/qemu/lib/.flake8   |   2 +
 python/qemu/lib/__init__.py   |  57 
 python/qemu/{ => lib}/accel.py|  17 +-
 python/qemu/{ => lib}/machine.py  | 320 +-
 python/qemu/lib/pylintrc  |  58 
 python/qemu/{ => lib}/qmp.py  | 140 +++---
 python/qemu/lib/qtest.py  | 160 +++
 python/qemu/qtest.py  | 119 
 python/setup.py   |  50 
 scripts/device-crash-test |   2 +-
 scripts/qmp/qemu-ga-client|   2 +-
 scripts/qmp/qmp   |   4 +-
 scripts/qmp/qmp-shell |   2 +-
 scripts/qmp/qom-fuse  |   4 +-
 scripts/qmp/qom-get   |   6 +-
 scripts/qmp/qom-list  |   4 +-
 scripts/qmp/qom-set   |   6 +-
 scripts/qmp/qom-tree  |   6 +-
 scripts/render_block_graph.py |   5 +-
 scripts/simplebench/bench_block_job.py|   4 +-
 tests/acceptance/avocado_qemu/__init__.py |   2 +-
 tests/acceptance/boot_linux.py|   3 +-
 tests/acceptance/virtio_check_params.py   |   2 +-
 tests/acceptance/virtio_version.py|   2 +-
 tests/migration/guestperf/engine.py   |   2 +-
 tests/qemu-iotests/235|   2 +-
 tests/qemu-iotests/iotests.py |   2 +-
 tests/vm/basevm.py|   6 +-
 33 files changed, 881 insertions(+), 334 deletions(-)
 create mode 100644 python/README.rst
 create mode 100644 python/qemu/README.rst
 create mode 100644 python/Pipfile
 create mode 100644 python/Pipfile.lock
 delete mode 100644 python/qemu/__init__.py
 create mode 100644 python/qemu/lib/.flake8
 create mode 100644 

[PATCH RFC 02/32] scripts/qmp: Fix shebang and imports

[John Snow]

There's more wrong with these scripts; They are in various stages of
disrepair. That's beyond the scope of this current patchset.

This just mechanically corrects the imports and the shebangs, as part of
ensuring that the python/qemu/lib refactoring didn't break anything
needlessly.

Signed-off-by: John Snow 
---
 scripts/qmp/qmp  | 4 +++-
 scripts/qmp/qom-fuse | 4 +++-
 scripts/qmp/qom-get  | 6 --
 scripts/qmp/qom-list | 4 +++-
 scripts/qmp/qom-set  | 6 --
 scripts/qmp/qom-tree | 6 --
 6 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/scripts/qmp/qmp b/scripts/qmp/qmp
index 0625fc2aba..5981f7c414 100755
--- a/scripts/qmp/qmp
+++ b/scripts/qmp/qmp
@@ -11,7 +11,9 @@
 # See the COPYING file in the top-level directory.
 
 import sys, os
-from qmp import QEMUMonitorProtocol
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
+from qemu.lib import QEMUMonitorProtocol
 
 def print_response(rsp, prefix=[]):
 if type(rsp) == list:
diff --git a/scripts/qmp/qom-fuse b/scripts/qmp/qom-fuse
index 6bada2c33d..6006f1d562 100755
--- a/scripts/qmp/qom-fuse
+++ b/scripts/qmp/qom-fuse
@@ -15,7 +15,9 @@ import fuse, stat
 from fuse import Fuse
 import os, posix
 from errno import *
-from qmp import QEMUMonitorProtocol
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
+from qemu.lib import QEMUMonitorProtocol
 
 fuse.fuse_python_api = (0, 2)
 
diff --git a/scripts/qmp/qom-get b/scripts/qmp/qom-get
index 007b4cd442..1d04d1b119 100755
--- a/scripts/qmp/qom-get
+++ b/scripts/qmp/qom-get
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 ##
 # QEMU Object Model test tools
 #
@@ -13,7 +13,9 @@
 
 import sys
 import os
-from qmp import QEMUMonitorProtocol
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
+from qemu.lib import QEMUMonitorProtocol
 
 cmd, args = sys.argv[0], sys.argv[1:]
 socket_path = None
diff --git a/scripts/qmp/qom-list b/scripts/qmp/qom-list
index 03bda3446b..94e3bc21ba 100755
--- a/scripts/qmp/qom-list
+++ b/scripts/qmp/qom-list
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 ##
 # QEMU Object Model test tools
 #
@@ -13,6 +13,8 @@
 
 import sys
 import os
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 from qmp import QEMUMonitorProtocol
 
 cmd, args = sys.argv[0], sys.argv[1:]
diff --git a/scripts/qmp/qom-set b/scripts/qmp/qom-set
index c37fe78b00..a78080a4fc 100755
--- a/scripts/qmp/qom-set
+++ b/scripts/qmp/qom-set
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 ##
 # QEMU Object Model test tools
 #
@@ -13,7 +13,9 @@
 
 import sys
 import os
-from qmp import QEMUMonitorProtocol
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
+from qemu.lib import QEMUMonitorProtocol
 
 cmd, args = sys.argv[0], sys.argv[1:]
 socket_path = None
diff --git a/scripts/qmp/qom-tree b/scripts/qmp/qom-tree
index 1c8acf61e7..4866c15581 100755
--- a/scripts/qmp/qom-tree
+++ b/scripts/qmp/qom-tree
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 ##
 # QEMU Object Model test tools
 #
@@ -15,7 +15,9 @@
 
 import sys
 import os
-from qmp import QEMUMonitorProtocol
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
+from qemu.lib import QEMUMonitorProtocol
 
 cmd, args = sys.argv[0], sys.argv[1:]
 socket_path = None
-- 
2.21.1

[PATCH RFC 06/32] python/qemu: formalize as package

[John Snow]

NB: I am choosing Python 3.6 here. Although our minimum requirement is
3.5, this code is used only by iotests (so far) under which we have been
using a minimum version of 3.6.

3.6 is being preferred here for variable type hint capability, which
enables us to use mypy for this package.

Signed-off-by: John Snow 
---
 python/README.rst |  6 ++
 python/setup.py   | 50 +++
 2 files changed, 56 insertions(+)
 create mode 100644 python/README.rst
 create mode 100755 python/setup.py

diff --git a/python/README.rst b/python/README.rst
new file mode 100644
index 00..25f6d93fd5
--- /dev/null
+++ b/python/README.rst
@@ -0,0 +1,6 @@
+QEMU Python Tooling
+---
+
+This package provides QEMU tooling used by the QEMU project to build,
+configure, and test QEMU. It is not a fully-fledged SDK and it is subject
+to change at any time.
diff --git a/python/setup.py b/python/setup.py
new file mode 100755
index 00..f897ceac97
--- /dev/null
+++ b/python/setup.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env3 python
+"""
+QEMU tooling installer script
+Copyright (c) 2020 John Snow for Red Hat, Inc.
+"""
+
+import setuptools
+
+def main():
+"""
+QEMU tooling installer
+"""
+
+kwargs = {
+'name': 'qemu',
+'use_scm_version': {
+'root': '..',
+'relative_to': __file__,
+},
+'maintainer': 'QEMU Developer Team',
+'maintainer_email': 'qemu-devel@nongnu.org',
+'url': 'https://www.qemu.org/',
+'download_url': 'https://www.qemu.org/download/',
+'packages': setuptools.find_namespace_packages(),
+'description': 'QEMU Python Build, Debug and SDK tooling.',
+'classifiers': [
+'Development Status :: 5 - Production/Stable',
+'License :: OSI Approved :: GNU General Public License v2 (GPLv2)',
+'Natural Language :: English',
+'Operating System :: OS Independent',
+],
+'platforms': [],
+'keywords': [],
+'setup_requires': [
+'setuptools',
+'setuptools_scm',
+],
+'install_requires': [
+],
+'python_requires': '>=3.6',
+'long_description_content_type': 'text/x-rst',
+}
+
+with open("README.rst", "r") as fh:
+kwargs['long_description'] = fh.read()
+
+setuptools.setup(**kwargs)
+
+if __name__ == '__main__':
+main()
-- 
2.21.1

[PATCH RFC 04/32] python/qemu/lib: delint, add pylintrc

[John Snow]

Bring our these files up to speed with pylint 2.5.0.
Add a pylintrc file to formalize which pylint subset
we are targeting.

The similarity ignore is there to suppress similarity
reports across imports, which for typing constants,
are going to trigger this report erroneously.

Signed-off-by: John Snow 
---
 python/qemu/lib/machine.py |  6 ++--
 python/qemu/lib/pylintrc   | 58 ++
 python/qemu/lib/qtest.py   | 42 +--
 3 files changed, 88 insertions(+), 18 deletions(-)
 create mode 100644 python/qemu/lib/pylintrc

diff --git a/python/qemu/lib/machine.py b/python/qemu/lib/machine.py
index e3ea523571..c79fc8fb89 100644
--- a/python/qemu/lib/machine.py
+++ b/python/qemu/lib/machine.py
@@ -58,7 +58,7 @@ def __init__(self, reply):
 self.reply = reply
 
 
-class QEMUMachine(object):
+class QEMUMachine:
 """
 A QEMU VM
 
@@ -242,7 +242,7 @@ def _base_args(self):
  'chardev=mon,mode=control'])
 if self._machine is not None:
 args.extend(['-machine', self._machine])
-for i in range(self._console_index):
+for _ in range(self._console_index):
 args.extend(['-serial', 'null'])
 if self._console_set:
 self._console_address = os.path.join(self._sock_dir,
@@ -383,7 +383,7 @@ def shutdown(self, has_quit: bool = False) -> None:
 command = ' '.join(self._qemu_full_args)
 else:
 command = ''
-LOG.warning(msg, -exitcode, command)
+LOG.warning(msg, -int(exitcode), command)
 
 self._launched = False
 
diff --git a/python/qemu/lib/pylintrc b/python/qemu/lib/pylintrc
new file mode 100644
index 00..5d6ae7367d
--- /dev/null
+++ b/python/qemu/lib/pylintrc
@@ -0,0 +1,58 @@
+[MASTER]
+
+[MESSAGES CONTROL]
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once). You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use "--disable=all --enable=classes
+# --disable=W".
+disable=too-many-arguments,
+too-many-instance-attributes,
+too-many-public-methods,
+
+[REPORTS]
+
+[REFACTORING]
+
+[MISCELLANEOUS]
+
+[LOGGING]
+
+[BASIC]
+
+# Good variable names which should always be accepted, separated by a comma.
+good-names=i,
+   j,
+   k,
+   ex,
+   Run,
+   _,
+   fd,
+
+[VARIABLES]
+
+[STRING]
+
+[SPELLING]
+
+[FORMAT]
+
+[SIMILARITIES]
+
+# Ignore imports when computing similarities.
+ignore-imports=yes
+
+[TYPECHECK]
+
+[CLASSES]
+
+[IMPORTS]
+
+[DESIGN]
+
+[EXCEPTIONS]
diff --git a/python/qemu/lib/qtest.py b/python/qemu/lib/qtest.py
index d24ad04256..53d814c064 100644
--- a/python/qemu/lib/qtest.py
+++ b/python/qemu/lib/qtest.py
@@ -1,5 +1,11 @@
-# QEMU qtest library
-#
+"""
+QEMU qtest library
+
+qtest offers the QEMUQtestProtocol and QEMUQTestMachine classes, which
+offer a connection to QEMU's qtest protocol socket, and a qtest-enabled
+subclass of QEMUMachine, respectively.
+"""
+
 # Copyright (C) 2015 Red Hat Inc.
 #
 # Authors:
@@ -17,19 +23,21 @@
 from .machine import QEMUMachine
 
 
-class QEMUQtestProtocol(object):
+class QEMUQtestProtocol:
+"""
+QEMUQtestProtocol implements a connection to a qtest socket.
+
+:param address: QEMU address, can be either a unix socket path (string)
+or a tuple in the form ( address, port ) for a TCP
+connection
+:param server: server mode, listens on the socket (bool)
+:raise socket.error: on socket connection errors
+
+.. note::
+   No conection is estabalished by __init__(), this is done
+   by the connect() or accept() methods.
+"""
 def __init__(self, address, server=False):
-"""
-Create a QEMUQtestProtocol object.
-
-@param address: QEMU address, can be either a unix socket path (string)
-or a tuple in the form ( address, port ) for a TCP
-connection
-@param server: server mode, listens on the socket (bool)
-@raise socket.error on socket connection errors
-@note No connection is established, this is done by the connect() or
-  accept() methods
-"""
 self._address = address
 self._sock = self._get_sock()
 self._sockfile = None
@@ -73,15 +81,19 @@ def cmd(self, qtest_cmd):
 return resp
 
 def close(self):
+"""Close this socket."""
 self._sock.close()
 

[PATCH RFC 01/32] python/qemu: create qemu.lib module

[John Snow]

move python/qemu/*.py to python/qemu/lib/*.py.

To create a namespace package, the 'qemu' directory itself shouldn't
have module files in it. Thus, these files will go under a 'lib' package
directory instead.

Bolster the lib/__init__.py file a little bit, Make the top-level
classes and functions available directly inside the `qemu.lib`
namespace, to facilitate a convenient shorthand:

> from qemu.lib import QEMUQtestMachine, QEMUMonitorProtocol

Lastly, update all of the existing import directives.

(Note: these scripts were not necessarily tested to see if they still
work. Some of these scripts are in obvious states of disrepair and it is
beyond the scope of this patch to attempt to fix them.)

Signed-off-by: John Snow 
---
 python/qemu/__init__.py   | 11 -
 python/qemu/lib/__init__.py   | 57 +++
 python/qemu/{ => lib}/accel.py|  0
 python/qemu/{ => lib}/machine.py  |  0
 python/qemu/{ => lib}/qmp.py  |  0
 python/qemu/{ => lib}/qtest.py|  0
 scripts/device-crash-test |  2 +-
 scripts/qmp/qemu-ga-client|  2 +-
 scripts/qmp/qmp-shell |  2 +-
 scripts/render_block_graph.py |  3 +-
 scripts/simplebench/bench_block_job.py|  4 +-
 tests/acceptance/avocado_qemu/__init__.py |  2 +-
 tests/acceptance/boot_linux.py|  3 +-
 tests/acceptance/virtio_check_params.py   |  2 +-
 tests/acceptance/virtio_version.py|  2 +-
 tests/migration/guestperf/engine.py   |  2 +-
 tests/qemu-iotests/235|  2 +-
 tests/qemu-iotests/iotests.py |  2 +-
 tests/vm/basevm.py|  6 +--
 19 files changed, 74 insertions(+), 28 deletions(-)
 delete mode 100644 python/qemu/__init__.py
 create mode 100644 python/qemu/lib/__init__.py
 rename python/qemu/{ => lib}/accel.py (100%)
 rename python/qemu/{ => lib}/machine.py (100%)
 rename python/qemu/{ => lib}/qmp.py (100%)
 rename python/qemu/{ => lib}/qtest.py (100%)

diff --git a/python/qemu/__init__.py b/python/qemu/__init__.py
deleted file mode 100644
index 4ca06c34a4..00
--- a/python/qemu/__init__.py
+++ /dev/null
@@ -1,11 +0,0 @@
-# QEMU library
-#
-# Copyright (C) 2015-2016 Red Hat Inc.
-# Copyright (C) 2012 IBM Corp.
-#
-# Authors:
-#  Fam Zheng 
-#
-# This work is licensed under the terms of the GNU GPL, version 2.  See
-# the COPYING file in the top-level directory.
-#
diff --git a/python/qemu/lib/__init__.py b/python/qemu/lib/__init__.py
new file mode 100644
index 00..afc587bfdc
--- /dev/null
+++ b/python/qemu/lib/__init__.py
@@ -0,0 +1,57 @@
+"""
+QEMU development and testing library.
+
+This library provides a few high-level classes for driving QEMU from a
+test suite, not intended for production use.
+
+- QEMUMachine: Configure and Boot a QEMU VM
+ - QEMUQtestMachine: VM class, with a qtest socket.
+
+- QEMUMonitorProtocol: Connect to, send/receive QMP messages.
+- QEMUQtestProtocol: Connect to, send/receive qtest message.
+
+- list_accel: List available accelerators
+- kvm_available: Probe for KVM support
+- tcg_available: Probe for TCG support
+"""
+
+# Copyright (C) 2020 John Snow for Red Hat Inc.
+# Copyright (C) 2015-2016 Red Hat Inc.
+# Copyright (C) 2012 IBM Corp.
+#
+# Authors:
+#  John Snow 
+#  Fam Zheng 
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+
+from .accel import (
+list_accel,
+kvm_available,
+tcg_available,
+)
+
+from .qmp import (
+QEMUMonitorProtocol,
+)
+
+from .machine import (
+QEMUMachine,
+)
+
+from .qtest import (
+QEMUQtestProtocol,
+QEMUQtestMachine,
+)
+
+__all__ = (
+'list_accel',
+'kvm_available',
+'tcg_available',
+'QEMUMonitorProtocol',
+'QEMUMachine',
+'QEMUQtestProtocol',
+'QEMUQtestMachine',
+)
diff --git a/python/qemu/accel.py b/python/qemu/lib/accel.py
similarity index 100%
rename from python/qemu/accel.py
rename to python/qemu/lib/accel.py
diff --git a/python/qemu/machine.py b/python/qemu/lib/machine.py
similarity index 100%
rename from python/qemu/machine.py
rename to python/qemu/lib/machine.py
diff --git a/python/qemu/qmp.py b/python/qemu/lib/qmp.py
similarity index 100%
rename from python/qemu/qmp.py
rename to python/qemu/lib/qmp.py
diff --git a/python/qemu/qtest.py b/python/qemu/lib/qtest.py
similarity index 100%
rename from python/qemu/qtest.py
rename to python/qemu/lib/qtest.py
diff --git a/scripts/device-crash-test b/scripts/device-crash-test
index 305d0427af..49efd4abd7 100755
--- a/scripts/device-crash-test
+++ b/scripts/device-crash-test
@@ -35,7 +35,7 @@ import argparse
 from itertools import chain
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python'))
-from qemu.machine import QEMUMachine
+from qemu.lib import QEMUMachine
 
 logger = logging.getLogger('device-crash-test')
 dbg = logger.debug
diff --git 

Re: [PATCH v5 08/15] acpi: move aml builder code for floppy device

[Thomas Huth]

On 13/05/2020 22.43, John Snow wrote:
> 
> 
> On 5/7/20 10:05 AM, Philippe Mathieu-Daudé wrote:
>> +Hervé
>>
>> On 5/7/20 3:16 PM, Gerd Hoffmann wrote:
>>> Signed-off-by: Gerd Hoffmann 
>>> Reviewed-by: Igor Mammedov 
>>> ---
>>>   hw/block/fdc.c   | 83 
> 
> oh no...
> 
>>>   hw/i386/acpi-build.c | 83 
>>>   stubs/cmos.c |  7 
>>>   stubs/Makefile.objs  |  1 +
>>>   4 files changed, 91 insertions(+), 83 deletions(-)
>>>   create mode 100644 stubs/cmos.c
>>>
>>> diff --git a/hw/block/fdc.c b/hw/block/fdc.c
>>> index 9628cc171ef8..40faa088b5f7 100644
>>> --- a/hw/block/fdc.c
>>> +++ b/hw/block/fdc.c
>>> @@ -32,6 +32,8 @@
>>>   #include "qapi/error.h"
>>>   #include "qemu/error-report.h"
>>>   #include "qemu/timer.h"
>>> +#include "hw/i386/pc.h"
>>
>> I'd rather not see this target-specific header used in the generic
>> device model... The culprit seems cmos_get_fd_drive_type(). Is the value
>> really PC specific?
>>
>> Hervé, do you know if such info & RTC nvram layout is used in other arch?
>>
> 
> do we need pc-only floppy code somewhere to keep the makefile pure?

Yes, please put this into a separate file that only gets build if the
corresponding CONFIG switches are set. That way you should also get
along without that new stubs/cmos.c file.

 Thomas

Re: [PATCH v4 4/9] blockdev: Promote several bitmap functions to non-static

[Vladimir Sementsov-Ogievskiy]

13.05.2020 04:16, Eric Blake wrote:

The next patch will split blockdev.c, which will require accessing
some previously-static functions from more than one .c file.  But part
of promoting a function to public is picking a naming scheme that does
not reek of exposing too many internals (two of the three functions
were named starting with 'do_').  To make future code motion easier,
perform the function rename and non-static promotion into its own
patch.

Signed-off-by: Eric Blake
Reviewed-by: Max Reitz


Reviewed-by: Vladimir Sementsov-Ogievskiy 

--
Best regards,
Vladimir

Re: [PATCH] qemu-nbd: Close inherited stderr

[Raphael Pour]

On 5/13/20 3:02 PM, Eric Blake wrote:
> Yes, now that we know about it, the bug will be fixed in 5.1; we can
> also cc: qemu-stable to get it backported to the next 5.0.x release
> (downstream developers are also more likely to backport it to their
> ports as well if it lands on qemu-stable).  Would you like to try your
> hand at a v2 patch, or shall I just turn my proposal into a formal patch
> and mention you as the reporter?  Time-wise, it may be faster for me to
> just take over the patch than to spend the time coaching you through to
> the point of your first successful submission, but project-wise, it is
> always better to welcome in new contributors and share the wealth,
> rather than making maintainers become the bottleneck.
> 

I'll give v2 a try and cc qemu-stable.
Thanks for the opportunity.

-- 
Hetzner Online GmbH
Am Datacenter-Park 1
08223 Falkenstein/Vogtland
raphael.p...@hetzner.com
www.hetzner.com

Registergericht Ansbach, HRB 6089
Geschäftsführer: Martin Hetzner, Stephan Konvickova, Günther Müller



signature.asc
Description: OpenPGP digital signature

Re: [PATCH Kernel v19 6/8] vfio iommu: Update UNMAP_DMA ioctl to get dirty bitmap before unmap

[Kirti Wankhede]

On 5/14/2020 10:37 AM, Alex Williamson wrote:

On Thu, 14 May 2020 01:34:37 +0530
Kirti Wankhede  wrote:


DMA mapped pages, including those pinned by mdev vendor drivers, might
get unpinned and unmapped while migration is active and device is still
running. For example, in pre-copy phase while guest driver could access
those pages, host device or vendor driver can dirty these mapped pages.
Such pages should be marked dirty so as to maintain memory consistency
for a user making use of dirty page tracking.

To get bitmap during unmap, user should allocate memory for bitmap, set
size of allocated memory, set page size to be considered for bitmap and
set flag VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP.

Signed-off-by: Kirti Wankhede 
Reviewed-by: Neo Jia 
---
  drivers/vfio/vfio_iommu_type1.c | 102 +++-
  include/uapi/linux/vfio.h   |  10 
  2 files changed, 99 insertions(+), 13 deletions(-)

diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 469b09185b83..4358be26ff80 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -195,11 +195,15 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, 
struct vfio_dma *old)
  static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize)
  {
uint64_t npages = dma->size / pgsize;
+   size_t bitmap_size;
  
  	if (npages > DIRTY_BITMAP_PAGES_MAX)

return -EINVAL;
  
-	dma->bitmap = kvzalloc(DIRTY_BITMAP_BYTES(npages), GFP_KERNEL);

+   /* Allocate extra 64 bits which are used for bitmap manipulation */
+   bitmap_size = DIRTY_BITMAP_BYTES(npages) + sizeof(u64);
+
+   dma->bitmap = kvzalloc(bitmap_size, GFP_KERNEL);
if (!dma->bitmap)
return -ENOMEM;
  
@@ -979,23 +983,25 @@ static int verify_bitmap_size(uint64_t npages, uint64_t bitmap_size)

  }
  
  static int vfio_dma_do_unmap(struct vfio_iommu *iommu,

-struct vfio_iommu_type1_dma_unmap *unmap)
+struct vfio_iommu_type1_dma_unmap *unmap,
+struct vfio_bitmap *bitmap)
  {
-   uint64_t mask;
struct vfio_dma *dma, *dma_last = NULL;
-   size_t unmapped = 0;
-   int ret = 0, retries = 0;
+   size_t unmapped = 0, pgsize;
+   int ret = 0, retries = 0, cnt = 0;
+   unsigned long pgshift, shift = 0, leftover;
  
  	mutex_lock(>lock);
  
-	mask = ((uint64_t)1 << __ffs(iommu->pgsize_bitmap)) - 1;

+   pgshift = __ffs(iommu->pgsize_bitmap);
+   pgsize = (size_t)1 << pgshift;
  
-	if (unmap->iova & mask) {

+   if (unmap->iova & (pgsize - 1)) {
ret = -EINVAL;
goto unlock;
}
  
-	if (!unmap->size || unmap->size & mask) {

+   if (!unmap->size || unmap->size & (pgsize - 1)) {
ret = -EINVAL;
goto unlock;
}
@@ -1006,9 +1012,15 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
goto unlock;
}
  
-	WARN_ON(mask & PAGE_MASK);

-again:
+   /* When dirty tracking is enabled, allow only min supported pgsize */
+   if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&
+   (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) {
+   ret = -EINVAL;
+   goto unlock;
+   }
  
+	WARN_ON((pgsize - 1) & PAGE_MASK);

+again:
/*
 * vfio-iommu-type1 (v1) - User mappings were coalesced together to
 * avoid tracking individual mappings.  This means that the granularity
@@ -1046,6 +1058,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
ret = -EINVAL;
goto unlock;
}
+
dma = vfio_find_dma(iommu, unmap->iova + unmap->size - 1, 0);
if (dma && dma->iova + dma->size != unmap->iova + unmap->size) {
ret = -EINVAL;
@@ -1063,6 +1076,39 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
if (dma->task->mm != current->mm)
break;
  
+		if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&

+   (dma_last != dma)) {
+   unsigned int nbits = dma->size >> pgshift;
+   int curr_lcnt = nbits / BITS_PER_LONG;
+
+   /*
+* mark all pages dirty if all pages are pinned and
+* mapped.
+*/
+   if (dma->iommu_mapped)
+   bitmap_set(dma->bitmap, 0, nbits);
+
+   if (shift) {
+   bitmap_shift_left(dma->bitmap, dma->bitmap,
+ shift, nbits + shift);
+   bitmap_or(dma->bitmap, dma->bitmap, ,
+ shift);
+   nbits += shift;
+  

Re: [PATCH v3 06/10] iotests: add testfinder.py

[Vladimir Sementsov-Ogievskiy]

14.05.2020 08:06, John Snow wrote:



On 5/14/20 12:54 AM, Vladimir Sementsov-Ogievskiy wrote:

14.05.2020 00:58, John Snow wrote:



On 5/7/20 1:43 PM, Vladimir Sementsov-Ogievskiy wrote:

21.04.2020 19:56, Kevin Wolf wrote:

Am 21.04.2020 um 09:35 hat Vladimir Sementsov-Ogievskiy geschrieben:

Add python script with new logic of searching for tests:

Current ./check behavior:
    - tests are named [0-9][0-9][0-9]
    - tests must be registered in group file (even if test doesn't
belong
  to any group, like 142)

Behavior of new test:
    - group file is dropped
    - tests are searched by file-name instead of group file, so it's
not
  needed more to "register the test", just create it with name
  *-test. Old names like [0-9][0-9][0-9] are supported too, but not
  recommended for new tests


I wonder if a tests/ subdirectory instead of the -test suffix would
organise things a bit better.



It will make more difficult to import iotests.py.. Calling common.rc
from
bash tests will need to be modified too.

So, we'll need additional line in all python tests:

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

which doesn't seem to be good practice.. So, instead we'd better call
tests with PYTHONPATH set appropriately..



Just chiming in to say that it's largely bad practice because it
confuses pylint, mypy and friends -- if we want to keep pushing our CI
code analysis for python in that direction, this will be a barrier.

Using PYTHONPATH is better, because it isolates the script itself from
the environment, but requires you to now always set PYTHONPATH to
execute any of the individual iotests.

Not actually a big deal, because iotests already expect a large number
of environment variables to be set. It's not really a huge net loss in
convenience, I think.

looks like that's the direction you're headed in anyway based on
discussion, so that's good.



Hm, does PYTHONPATH-way works good with mypy and friends? Probably, it
should
be set when checking the code? So, actually developers will have to set
PYTHONPATH by hand to always contain some directories within qemu source
tree?



pylint respects PYTHONPATH but mypy doesn't. mypy uses MYPYPATH, but I
wouldn't worry about accommodating it. It's a fussy tool and we're only
ever going to run it from very specific environments.



Hmm, recently I installed dense-analysis/ale plugin into my vim which does mypy 
checking (among other things).. And most probably, I'll have to set these 
variables to keep it working. But it's not a big problem.


You don't need to worry too much about what environment variables these
tools take; it's only worth noting that "sys.path" hacks tend to make
these tools harder to use.


As for setting PYTHONPATH by hand ... There are a few places in the QEMU
tree where we set PYTHONPATH already, and the individual iotests already
don't work if they're not launched by `check`, because they're missing a
ton of environment variables.

It's not going to be too bad to set PYTHONPATH in the launcher script,
is it?

(Or are we replacing the top-level script with a python one?)


Yes we do, bright future is near:) But it's not a problem to set PYTHONPATH in 
it. Anyway, we run all tests as executables, so passing PYTHONPATH is a valid 
thing to do.






Really, the same is true of pylint, too. It's only annoying to deal with
sys.path hacking because it can't be worked around in those CQA tools.




--
Best regards,
Vladimir

Re: [PATCH v4 3/9] block: Make it easier to learn which BDS support bitmaps

[Vladimir Sementsov-Ogievskiy]

13.05.2020 04:16, Eric Blake wrote:

Upcoming patches will enhance bitmap support in qemu-img, but in doing
so, it turns out to be nice to suppress output when persistent bitmaps
make no sense (such as on a qcow2 v2 image).  Add a hook to make this
easier to query.

This patch adds a new callback .bdrv_supports_persistent_dirty_bitmap,
rather than trying to shoehorn the answer in via existing callbacks.
In particular, while it might have been possible to overload
.bdrv_co_can_store_new_dirty_bitmap to special-case a NULL input to
answer whether any persistent bitmaps are supported, that is at odds
with whether a particular bitmap can be stored (for example, even on
an image that supports persistent bitmaps but has currently filled up
the maximum number of bitmaps, attempts to store another one should
fail); and the new functionality doesn't require coroutine safety.
Similarly, we could have added one more piece of information to
.bdrv_get_info, but then again, most callers to that function tend to
already discard extraneous information, and making it a catch-all
rather than a series of dedicated scalar queries hasn't really
simplified life.

In the future, when we improve the ability to look up bitmaps through
a filter, we will probably also want to teach the block layer to
automatically let filters pass this request on through.


Hm. I think that bitmap at filter bs is a valid thing (moreover I have a plan 
to use it for one issue), so I'm not sure that it's good idea to do any generic 
logic around bitmaps work through filters, better to always address the exact 
node you mean..



Signed-off-by: Eric Blake 


Reviewed-by: Vladimir Sementsov-Ogievskiy 

--
Best regards,
Vladimir

Re: [PATCH] bitmaps: Add myself as maintainer

[John Snow]

On 5/14/20 12:49 AM, Vladimir Sementsov-Ogievskiy wrote:
> 13.05.2020 23:24, John Snow wrote:
>>
>>
>> On 5/13/20 10:14 AM, Eric Blake wrote:
>>> Dirty bitmaps are important to incremental backups, including exposure
>>> over NBD where I'm already maintainer.  Also, I'm aware that lately I
>>> have been doing as much code/review on bitmaps as John Snow, who is
>>> hoping to scale back on this front.
>>>
>>> Signed-off-by: Eric Blake 
>>>
>>> ---
>>> This still leaves John as maintainer for: IDE, Floppy, and Block Jobs,
>>> which I did not feel comfortable taking on at this time.
>>>
>>> If this patch makes sense, I'm happy to take on my recent 'qemu-img
>>> bitmap' series as my first bitmap-related pull request, once it has
>>> sufficient review.
>>> ---
>>>   MAINTAINERS | 1 +
>>>   1 file changed, 1 insertion(+)
>>>
>>> diff --git a/MAINTAINERS b/MAINTAINERS
>>> index 6a8dc1e69d42..f7caf7c0cc7f 100644
>>> --- a/MAINTAINERS
>>> +++ b/MAINTAINERS
>>> @@ -2002,6 +2002,7 @@ T: git https://repo.or.cz/qemu/armbru.git
>>> block-next
>>>
>>>   Dirty Bitmaps
>>>   M: John Snow 
>>> +M: Eric Blake 
>>>   R: Vladimir Sementsov-Ogievskiy 
>>>   L: qemu-bl...@nongnu.org
>>>   S: Supported
>>>
>>
>> I'd also like to point out that I wouldn't mind if Vladimir became an
>> official maintainer, but I can't remember if he wanted the title when we
>> last spoke at KVM Forum.
> 
> Actually, it would be nice, I'd glad to get it, thanks :)
> I can send a separate patch, or we may s/R/M/ in this one?
> 

That would be very good!

I'd be quite happy to be demoted to reviewer; it's about all the time
I've been truthfully able to give lately.

(I won't speak for Eric!)

--js

Re: [PATCH v4 2/9] qemu-img: Fix stale comments on doc location

[Vladimir Sementsov-Ogievskiy]

13.05.2020 04:16, Eric Blake wrote:

Missed in commit e13c59fa.

Signed-off-by: Eric Blake
Reviewed-by: Max Reitz


Reviewed-by: Vladimir Sementsov-Ogievskiy 

--
Best regards,
Vladimir

Re: [PATCH Kernel v19 6/8] vfio iommu: Update UNMAP_DMA ioctl to get dirty bitmap before unmap

[Alex Williamson]

On Thu, 14 May 2020 01:34:37 +0530
Kirti Wankhede  wrote:

> DMA mapped pages, including those pinned by mdev vendor drivers, might
> get unpinned and unmapped while migration is active and device is still
> running. For example, in pre-copy phase while guest driver could access
> those pages, host device or vendor driver can dirty these mapped pages.
> Such pages should be marked dirty so as to maintain memory consistency
> for a user making use of dirty page tracking.
> 
> To get bitmap during unmap, user should allocate memory for bitmap, set
> size of allocated memory, set page size to be considered for bitmap and
> set flag VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP.
> 
> Signed-off-by: Kirti Wankhede 
> Reviewed-by: Neo Jia 
> ---
>  drivers/vfio/vfio_iommu_type1.c | 102 
> +++-
>  include/uapi/linux/vfio.h   |  10 
>  2 files changed, 99 insertions(+), 13 deletions(-)
> 
> diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
> index 469b09185b83..4358be26ff80 100644
> --- a/drivers/vfio/vfio_iommu_type1.c
> +++ b/drivers/vfio/vfio_iommu_type1.c
> @@ -195,11 +195,15 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, 
> struct vfio_dma *old)
>  static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize)
>  {
>   uint64_t npages = dma->size / pgsize;
> + size_t bitmap_size;
>  
>   if (npages > DIRTY_BITMAP_PAGES_MAX)
>   return -EINVAL;
>  
> - dma->bitmap = kvzalloc(DIRTY_BITMAP_BYTES(npages), GFP_KERNEL);
> + /* Allocate extra 64 bits which are used for bitmap manipulation */
> + bitmap_size = DIRTY_BITMAP_BYTES(npages) + sizeof(u64);
> +
> + dma->bitmap = kvzalloc(bitmap_size, GFP_KERNEL);
>   if (!dma->bitmap)
>   return -ENOMEM;
>  
> @@ -979,23 +983,25 @@ static int verify_bitmap_size(uint64_t npages, uint64_t 
> bitmap_size)
>  }
>  
>  static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
> -  struct vfio_iommu_type1_dma_unmap *unmap)
> +  struct vfio_iommu_type1_dma_unmap *unmap,
> +  struct vfio_bitmap *bitmap)
>  {
> - uint64_t mask;
>   struct vfio_dma *dma, *dma_last = NULL;
> - size_t unmapped = 0;
> - int ret = 0, retries = 0;
> + size_t unmapped = 0, pgsize;
> + int ret = 0, retries = 0, cnt = 0;
> + unsigned long pgshift, shift = 0, leftover;
>  
>   mutex_lock(>lock);
>  
> - mask = ((uint64_t)1 << __ffs(iommu->pgsize_bitmap)) - 1;
> + pgshift = __ffs(iommu->pgsize_bitmap);
> + pgsize = (size_t)1 << pgshift;
>  
> - if (unmap->iova & mask) {
> + if (unmap->iova & (pgsize - 1)) {
>   ret = -EINVAL;
>   goto unlock;
>   }
>  
> - if (!unmap->size || unmap->size & mask) {
> + if (!unmap->size || unmap->size & (pgsize - 1)) {
>   ret = -EINVAL;
>   goto unlock;
>   }
> @@ -1006,9 +1012,15 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
>   goto unlock;
>   }
>  
> - WARN_ON(mask & PAGE_MASK);
> -again:
> + /* When dirty tracking is enabled, allow only min supported pgsize */
> + if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&
> + (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) {
> + ret = -EINVAL;
> + goto unlock;
> + }
>  
> + WARN_ON((pgsize - 1) & PAGE_MASK);
> +again:
>   /*
>* vfio-iommu-type1 (v1) - User mappings were coalesced together to
>* avoid tracking individual mappings.  This means that the granularity
> @@ -1046,6 +1058,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
>   ret = -EINVAL;
>   goto unlock;
>   }
> +
>   dma = vfio_find_dma(iommu, unmap->iova + unmap->size - 1, 0);
>   if (dma && dma->iova + dma->size != unmap->iova + unmap->size) {
>   ret = -EINVAL;
> @@ -1063,6 +1076,39 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
>   if (dma->task->mm != current->mm)
>   break;
>  
> + if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&
> + (dma_last != dma)) {
> + unsigned int nbits = dma->size >> pgshift;
> + int curr_lcnt = nbits / BITS_PER_LONG;
> +
> + /*
> +  * mark all pages dirty if all pages are pinned and
> +  * mapped.
> +  */
> + if (dma->iommu_mapped)
> + bitmap_set(dma->bitmap, 0, nbits);
> +
> + if (shift) {
> + bitmap_shift_left(dma->bitmap, dma->bitmap,
> +   shift, nbits + shift);
> + bitmap_or(dma->bitmap, dma->bitmap, ,
> +  

Re: [PATCH v3 06/10] iotests: add testfinder.py

[John Snow]

On 5/14/20 12:54 AM, Vladimir Sementsov-Ogievskiy wrote:
> 14.05.2020 00:58, John Snow wrote:
>>
>>
>> On 5/7/20 1:43 PM, Vladimir Sementsov-Ogievskiy wrote:
>>> 21.04.2020 19:56, Kevin Wolf wrote:
 Am 21.04.2020 um 09:35 hat Vladimir Sementsov-Ogievskiy geschrieben:
> Add python script with new logic of searching for tests:
>
> Current ./check behavior:
>    - tests are named [0-9][0-9][0-9]
>    - tests must be registered in group file (even if test doesn't
> belong
>  to any group, like 142)
>
> Behavior of new test:
>    - group file is dropped
>    - tests are searched by file-name instead of group file, so it's
> not
>  needed more to "register the test", just create it with name
>  *-test. Old names like [0-9][0-9][0-9] are supported too, but not
>  recommended for new tests

 I wonder if a tests/ subdirectory instead of the -test suffix would
 organise things a bit better.

>>>
>>> It will make more difficult to import iotests.py.. Calling common.rc
>>> from
>>> bash tests will need to be modified too.
>>>
>>> So, we'll need additional line in all python tests:
>>>
>>> sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
>>>
>>> which doesn't seem to be good practice.. So, instead we'd better call
>>> tests with PYTHONPATH set appropriately..
>>>
>>
>> Just chiming in to say that it's largely bad practice because it
>> confuses pylint, mypy and friends -- if we want to keep pushing our CI
>> code analysis for python in that direction, this will be a barrier.
>>
>> Using PYTHONPATH is better, because it isolates the script itself from
>> the environment, but requires you to now always set PYTHONPATH to
>> execute any of the individual iotests.
>>
>> Not actually a big deal, because iotests already expect a large number
>> of environment variables to be set. It's not really a huge net loss in
>> convenience, I think.
>>
>> looks like that's the direction you're headed in anyway based on
>> discussion, so that's good.
>>
> 
> Hm, does PYTHONPATH-way works good with mypy and friends? Probably, it
> should
> be set when checking the code? So, actually developers will have to set
> PYTHONPATH by hand to always contain some directories within qemu source
> tree?
> 

pylint respects PYTHONPATH but mypy doesn't. mypy uses MYPYPATH, but I
wouldn't worry about accommodating it. It's a fussy tool and we're only
ever going to run it from very specific environments.

You don't need to worry too much about what environment variables these
tools take; it's only worth noting that "sys.path" hacks tend to make
these tools harder to use.


As for setting PYTHONPATH by hand ... There are a few places in the QEMU
tree where we set PYTHONPATH already, and the individual iotests already
don't work if they're not launched by `check`, because they're missing a
ton of environment variables.

It's not going to be too bad to set PYTHONPATH in the launcher script,
is it?

(Or are we replacing the top-level script with a python one?)




Really, the same is true of pylint, too. It's only annoying to deal with
sys.path hacking because it can't be worked around in those CQA tools.

Re: [PATCH v4 1/9] docs: Sort sections on qemu-img subcommand parameters

[Vladimir Sementsov-Ogievskiy]

13.05.2020 04:16, Eric Blake wrote:

We already list the subcommand summaries alphabetically, we should do
the same for the documentation related to subcommand-specific
parameters.

Signed-off-by: Eric Blake
Reviewed-by: Max Reitz


Reviewed-by: Vladimir Sementsov-Ogievskiy 

--
Best regards,
Vladimir

Re: [PATCH Kernel v19 5/8] vfio iommu: Implementation of ioctl for dirty pages tracking

[Alex Williamson]

On Thu, 14 May 2020 01:34:36 +0530
Kirti Wankhede  wrote:

> VFIO_IOMMU_DIRTY_PAGES ioctl performs three operations:
> - Start dirty pages tracking while migration is active
> - Stop dirty pages tracking.
> - Get dirty pages bitmap. Its user space application's responsibility to
>   copy content of dirty pages from source to destination during migration.
> 
> To prevent DoS attack, memory for bitmap is allocated per vfio_dma
> structure. Bitmap size is calculated considering smallest supported page
> size. Bitmap is allocated for all vfio_dmas when dirty logging is enabled
> 
> Bitmap is populated for already pinned pages when bitmap is allocated for
> a vfio_dma with the smallest supported page size. Update bitmap from
> pinning functions when tracking is enabled. When user application queries
> bitmap, check if requested page size is same as page size used to
> populated bitmap. If it is equal, copy bitmap, but if not equal, return
> error.
> 
> Signed-off-by: Kirti Wankhede 
> Reviewed-by: Neo Jia 
> 
> Fixed error reported by build bot by changing pgsize type from uint64_t
> to size_t.
> Reported-by: kbuild test robot 
> ---
>  drivers/vfio/vfio_iommu_type1.c | 274 
> +++-
>  1 file changed, 268 insertions(+), 6 deletions(-)
> 
> diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
> index 6f09fbabed12..469b09185b83 100644
> --- a/drivers/vfio/vfio_iommu_type1.c
> +++ b/drivers/vfio/vfio_iommu_type1.c
> @@ -72,6 +72,7 @@ struct vfio_iommu {
>   uint64_tpgsize_bitmap;
>   boolv2;
>   boolnesting;
> + booldirty_page_tracking;
>  };
>  
>  struct vfio_domain {
> @@ -92,6 +93,7 @@ struct vfio_dma {
>   boollock_cap;   /* capable(CAP_IPC_LOCK) */
>   struct task_struct  *task;
>   struct rb_root  pfn_list;   /* Ex-user pinned pfn list */
> + unsigned long   *bitmap;
>  };
>  
>  struct vfio_group {
> @@ -126,6 +128,19 @@ struct vfio_regions {
>  #define IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)  \
>   (!list_empty(>domain_list))
>  
> +#define DIRTY_BITMAP_BYTES(n)(ALIGN(n, BITS_PER_TYPE(u64)) / 
> BITS_PER_BYTE)
> +
> +/*
> + * Input argument of number of bits to bitmap_set() is unsigned integer, 
> which
> + * further casts to signed integer for unaligned multi-bit operation,
> + * __bitmap_set().
> + * Then maximum bitmap size supported is 2^31 bits divided by 2^3 bits/byte,
> + * that is 2^28 (256 MB) which maps to 2^31 * 2^12 = 2^43 (8TB) on 4K page
> + * system.
> + */
> +#define DIRTY_BITMAP_PAGES_MAX((u64)INT_MAX)
> +#define DIRTY_BITMAP_SIZE_MAX 
> DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX)
> +
>  static int put_pfn(unsigned long pfn, int prot);
>  
>  /*
> @@ -176,6 +191,77 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, 
> struct vfio_dma *old)
>   rb_erase(>node, >dma_list);
>  }
>  
> +
> +static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize)
> +{
> + uint64_t npages = dma->size / pgsize;
> +
> + if (npages > DIRTY_BITMAP_PAGES_MAX)
> + return -EINVAL;
> +
> + dma->bitmap = kvzalloc(DIRTY_BITMAP_BYTES(npages), GFP_KERNEL);
> + if (!dma->bitmap)
> + return -ENOMEM;
> +
> + return 0;
> +}
> +
> +static void vfio_dma_bitmap_free(struct vfio_dma *dma)
> +{
> + kfree(dma->bitmap);
> + dma->bitmap = NULL;
> +}
> +
> +static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize)
> +{
> + struct rb_node *p;
> +
> + if (RB_EMPTY_ROOT(>pfn_list))
> + return;

I don't think this is optimizing anything:

#define RB_EMPTY_ROOT(root)  (READ_ONCE((root)->rb_node) == NULL)

struct rb_node *rb_first(const struct rb_root *root)
{
struct rb_node  *n;

n = root->rb_node;
if (!n)
return NULL;

So the loop below won't be entered if the tree is empty.

> +
> + for (p = rb_first(>pfn_list); p; p = rb_next(p)) {
> + struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node);
> +
> + bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) / pgsize, 1);
> + }
> +}
> +
> +static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize)
> +{
> + struct rb_node *n = rb_first(>dma_list);
> +
> + for (; n; n = rb_next(n)) {
> + struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
> + int ret;
> +
> + ret = vfio_dma_bitmap_alloc(dma, pgsize);
> + if (ret) {
> + struct rb_node *p = rb_prev(n);
> +
> + for (; p; p = rb_prev(p)) {
> + struct vfio_dma *dma = rb_entry(n,
> + struct vfio_dma, node);
> +
> + vfio_dma_bitmap_free(dma);
> + }
> +  

Re: [PATCH Kernel v19 7/8] vfio iommu: Add migration capability to report supported features

[Alex Williamson]

On Thu, 14 May 2020 01:34:38 +0530
Kirti Wankhede  wrote:

> Added migration capability in IOMMU info chain.
> User application should check IOMMU info chain for migration capability
> to use dirty page tracking feature provided by kernel module.
> User application must check page sizes supported and maximum dirty
> bitmap size returned by this capability structure for ioctls used to get
> dirty bitmap.
> 
> Signed-off-by: Kirti Wankhede 
> ---
>  drivers/vfio/vfio_iommu_type1.c | 24 +++-
>  include/uapi/linux/vfio.h   | 21 +
>  2 files changed, 44 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
> index 4358be26ff80..77351497a9c2 100644
> --- a/drivers/vfio/vfio_iommu_type1.c
> +++ b/drivers/vfio/vfio_iommu_type1.c
> @@ -2389,6 +2389,22 @@ static int vfio_iommu_iova_build_caps(struct 
> vfio_iommu *iommu,
>   return ret;
>  }
>  
> +static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu,
> +struct vfio_info_cap *caps)
> +{
> + struct vfio_iommu_type1_info_cap_migration cap_mig;
> +
> + cap_mig.header.id = VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION;
> + cap_mig.header.version = 1;
> + cap_mig.flags = VFIO_IOMMU_INFO_CAPS_MIGRATION_DIRTY_PAGE_TRACK;
> +
> + /* support minimum pgsize */
> + cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap);
> + cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX;
> +
> + return vfio_info_add_capability(caps, _mig.header, sizeof(cap_mig));
> +}
> +
>  static long vfio_iommu_type1_ioctl(void *iommu_data,
>  unsigned int cmd, unsigned long arg)
>  {
> @@ -2433,10 +2449,16 @@ static long vfio_iommu_type1_ioctl(void *iommu_data,
>   mutex_lock(>lock);
>   info.flags = VFIO_IOMMU_INFO_PGSIZES;
>  
> + vfio_pgsize_bitmap(iommu);


Why is it necessary to rebuild the bitmap here?  The user can't get to
this ioctl until they've added a group to the container and set the
IOMMU model.


>   info.iova_pgsizes = iommu->pgsize_bitmap;
>  
> - ret = vfio_iommu_iova_build_caps(iommu, );
> + ret = vfio_iommu_migration_build_caps(iommu, );
> +
> + if (!ret)
> + ret = vfio_iommu_iova_build_caps(iommu, );
> +
>   mutex_unlock(>lock);
> +
>   if (ret)
>   return ret;
>  
> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> index e3cbf8b78623..c90604322798 100644
> --- a/include/uapi/linux/vfio.h
> +++ b/include/uapi/linux/vfio.h
> @@ -1013,6 +1013,27 @@ struct vfio_iommu_type1_info_cap_iova_range {
>   struct  vfio_iova_range iova_ranges[];
>  };
>  
> +/*
> + * The migration capability allows to report supported features for 
> migration.
> + *
> + * The structures below define version 1 of this capability.
> + *
> + * pgsize_bitmap: Kernel driver returns supported page sizes bitmap for dirty
> + * page tracking.
> + * max_dirty_bitmap_size: Kernel driver returns maximum supported dirty 
> bitmap
> + * size in bytes to be used by user application for ioctls to get dirty 
> bitmap.
> + */
> +#define VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION  1
> +
> +struct vfio_iommu_type1_info_cap_migration {
> + struct  vfio_info_cap_header header;
> + __u32   flags;
> + /* supports dirty page tracking */
> +#define VFIO_IOMMU_INFO_CAPS_MIGRATION_DIRTY_PAGE_TRACK  (1 << 0)

This flag is a bit redundant to the purpose of this capability, isn't
it?  I think exposing the capability itself is indicating support for
dirty page tracking.  We should probably be explicit in the comment
about exactly what interface this capability implies.  Thanks,

Alex

> + __u64   pgsize_bitmap;
> + __u64   max_dirty_bitmap_size;  /* in bytes */
> +};
> +
>  #define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
>  
>  /**

Re: [PATCH] bitmaps: Add myself as maintainer

[Vladimir Sementsov-Ogievskiy]

13.05.2020 23:24, John Snow wrote:



On 5/13/20 10:14 AM, Eric Blake wrote:

Dirty bitmaps are important to incremental backups, including exposure
over NBD where I'm already maintainer.  Also, I'm aware that lately I
have been doing as much code/review on bitmaps as John Snow, who is
hoping to scale back on this front.

Signed-off-by: Eric Blake 

---
This still leaves John as maintainer for: IDE, Floppy, and Block Jobs,
which I did not feel comfortable taking on at this time.

If this patch makes sense, I'm happy to take on my recent 'qemu-img
bitmap' series as my first bitmap-related pull request, once it has
sufficient review.
---
  MAINTAINERS | 1 +
  1 file changed, 1 insertion(+)

diff --git a/MAINTAINERS b/MAINTAINERS
index 6a8dc1e69d42..f7caf7c0cc7f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2002,6 +2002,7 @@ T: git https://repo.or.cz/qemu/armbru.git block-next

  Dirty Bitmaps
  M: John Snow 
+M: Eric Blake 
  R: Vladimir Sementsov-Ogievskiy 
  L: qemu-bl...@nongnu.org
  S: Supported



I'd also like to point out that I wouldn't mind if Vladimir became an
official maintainer, but I can't remember if he wanted the title when we
last spoke at KVM Forum.


Actually, it would be nice, I'd glad to get it, thanks :)
I can send a separate patch, or we may s/R/M/ in this one?



Regardless; I am clearly not keeping up with reviews right now as my
attention has been pulled elsewhere, so:

Reviewed-by: John Snow 




--
Best regards,
Vladimir

Re: [PATCH v3 06/10] iotests: add testfinder.py

[Vladimir Sementsov-Ogievskiy]

14.05.2020 00:58, John Snow wrote:



On 5/7/20 1:43 PM, Vladimir Sementsov-Ogievskiy wrote:

21.04.2020 19:56, Kevin Wolf wrote:

Am 21.04.2020 um 09:35 hat Vladimir Sementsov-Ogievskiy geschrieben:

Add python script with new logic of searching for tests:

Current ./check behavior:
   - tests are named [0-9][0-9][0-9]
   - tests must be registered in group file (even if test doesn't belong
     to any group, like 142)

Behavior of new test:
   - group file is dropped
   - tests are searched by file-name instead of group file, so it's not
     needed more to "register the test", just create it with name
     *-test. Old names like [0-9][0-9][0-9] are supported too, but not
     recommended for new tests


I wonder if a tests/ subdirectory instead of the -test suffix would
organise things a bit better.



It will make more difficult to import iotests.py.. Calling common.rc from
bash tests will need to be modified too.

So, we'll need additional line in all python tests:

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

which doesn't seem to be good practice.. So, instead we'd better call
tests with PYTHONPATH set appropriately..



Just chiming in to say that it's largely bad practice because it
confuses pylint, mypy and friends -- if we want to keep pushing our CI
code analysis for python in that direction, this will be a barrier.

Using PYTHONPATH is better, because it isolates the script itself from
the environment, but requires you to now always set PYTHONPATH to
execute any of the individual iotests.

Not actually a big deal, because iotests already expect a large number
of environment variables to be set. It's not really a huge net loss in
convenience, I think.

looks like that's the direction you're headed in anyway based on
discussion, so that's good.



Hm, does PYTHONPATH-way works good with mypy and friends? Probably, it should
be set when checking the code? So, actually developers will have to set
PYTHONPATH by hand to always contain some directories within qemu source tree?

--
Best regards,
Vladimir

[PATCH v6 13/20] hw/block/nvme: factor out block backend setup

[Klaus Jensen]

From: Klaus Jensen 

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 13 ++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 617b2f0005df..c83f0bb1ab20 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -1412,6 +1412,13 @@ static void nvme_init_state(NvmeCtrl *n)
 n->cq = g_new0(NvmeCQueue *, n->params.max_ioqpairs + 1);
 }
 
+static void nvme_init_blk(NvmeCtrl *n, Error **errp)
+{
+blkconf_blocksizes(>conf);
+blkconf_apply_backend_options(>conf, blk_is_read_only(n->conf.blk),
+  false, errp);
+}
+
 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 {
 NvmeCtrl *n = NVME(pci_dev);
@@ -1436,9 +1443,9 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 return;
 }
 
-blkconf_blocksizes(>conf);
-if (!blkconf_apply_backend_options(>conf, blk_is_read_only(n->conf.blk),
-   false, errp)) {
+nvme_init_blk(n, _err);
+if (local_err) {
+error_propagate(errp, local_err);
 return;
 }
 
-- 
2.26.2

[PATCH v6 15/20] hw/block/nvme: factor out namespace setup

[Klaus Jensen]

From: Klaus Jensen 

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky  
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 46 ++
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index e24d376b0d71..8e9312534fcd 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -1419,6 +1419,27 @@ static void nvme_init_blk(NvmeCtrl *n, Error **errp)
   false, errp);
 }
 
+static void nvme_init_namespace(NvmeCtrl *n, NvmeNamespace *ns, Error **errp)
+{
+int64_t bs_size;
+NvmeIdNs *id_ns = >id_ns;
+
+bs_size = blk_getlength(n->conf.blk);
+if (bs_size < 0) {
+error_setg_errno(errp, -bs_size, "could not get backing file size");
+return;
+}
+
+n->ns_size = bs_size;
+
+id_ns->lbaf[0].ds = BDRV_SECTOR_BITS;
+id_ns->nsze = cpu_to_le64(nvme_ns_nlbas(n, ns));
+
+/* no thin provisioning */
+id_ns->ncap = id_ns->nsze;
+id_ns->nuse = id_ns->ncap;
+}
+
 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 {
 NvmeCtrl *n = NVME(pci_dev);
@@ -1426,7 +1447,6 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 Error *local_err = NULL;
 
 int i;
-int64_t bs_size;
 uint8_t *pci_conf;
 
 nvme_check_constraints(n, _err);
@@ -1437,12 +1457,6 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 
 nvme_init_state(n);
 
-bs_size = blk_getlength(n->conf.blk);
-if (bs_size < 0) {
-error_setg(errp, "could not get backing file size");
-return;
-}
-
 nvme_init_blk(n, _err);
 if (local_err) {
 error_propagate(errp, local_err);
@@ -1455,8 +1469,6 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 pci_config_set_class(pci_dev->config, PCI_CLASS_STORAGE_EXPRESS);
 pcie_endpoint_cap_init(pci_dev, 0x80);
 
-n->ns_size = bs_size / (uint64_t)n->num_namespaces;
-
 memory_region_init_io(>iomem, OBJECT(n), _mmio_ops, n,
   "nvme", n->reg_size);
 pci_register_bar(pci_dev, 0,
@@ -1565,17 +1577,11 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 }
 
 for (i = 0; i < n->num_namespaces; i++) {
-NvmeNamespace *ns = >namespaces[i];
-NvmeIdNs *id_ns = >id_ns;
-id_ns->nsfeat = 0;
-id_ns->nlbaf = 0;
-id_ns->flbas = 0;
-id_ns->mc = 0;
-id_ns->dpc = 0;
-id_ns->dps = 0;
-id_ns->lbaf[0].ds = BDRV_SECTOR_BITS;
-id_ns->ncap  = id_ns->nuse = id_ns->nsze =
-cpu_to_le64(nvme_ns_nlbas(n, ns));
+nvme_init_namespace(n, >namespaces[i], _err);
+if (local_err) {
+error_propagate(errp, local_err);
+return;
+}
 }
 }
 
-- 
2.26.2

[PATCH v6 11/20] hw/block/nvme: factor out property/constraint checks

[Klaus Jensen]

From: Klaus Jensen 

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 48 ++--
 1 file changed, 30 insertions(+), 18 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index e93ed729aef5..1d4fbba9c5da 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -1356,24 +1356,19 @@ static const MemoryRegionOps nvme_cmb_ops = {
 },
 };
 
-static void nvme_realize(PCIDevice *pci_dev, Error **errp)
+static void nvme_check_constraints(NvmeCtrl *n, Error **errp)
 {
-NvmeCtrl *n = NVME(pci_dev);
-NvmeIdCtrl *id = >id_ctrl;
+NvmeParams *params = >params;
 
-int i;
-int64_t bs_size;
-uint8_t *pci_conf;
-
-if (n->params.num_queues) {
+if (params->num_queues) {
 warn_report("num_queues is deprecated; please use max_ioqpairs "
 "instead");
 
-n->params.max_ioqpairs = n->params.num_queues - 1;
+params->max_ioqpairs = params->num_queues - 1;
 }
 
-if (n->params.max_ioqpairs < 1 ||
-n->params.max_ioqpairs > PCI_MSIX_FLAGS_QSIZE) {
+if (params->max_ioqpairs < 1 ||
+params->max_ioqpairs > PCI_MSIX_FLAGS_QSIZE) {
 error_setg(errp, "max_ioqpairs must be between 1 and %d",
PCI_MSIX_FLAGS_QSIZE);
 return;
@@ -1384,13 +1379,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 return;
 }
 
-bs_size = blk_getlength(n->conf.blk);
-if (bs_size < 0) {
-error_setg(errp, "could not get backing file size");
-return;
-}
-
-if (!n->params.serial) {
+if (!params->serial) {
 error_setg(errp, "serial property not set");
 return;
 }
@@ -1410,6 +1399,29 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 
 host_memory_backend_set_mapped(n->pmrdev, true);
 }
+}
+
+static void nvme_realize(PCIDevice *pci_dev, Error **errp)
+{
+NvmeCtrl *n = NVME(pci_dev);
+NvmeIdCtrl *id = >id_ctrl;
+Error *local_err = NULL;
+
+int i;
+int64_t bs_size;
+uint8_t *pci_conf;
+
+nvme_check_constraints(n, _err);
+if (local_err) {
+error_propagate(errp, local_err);
+return;
+}
+
+bs_size = blk_getlength(n->conf.blk);
+if (bs_size < 0) {
+error_setg(errp, "could not get backing file size");
+return;
+}
 
 blkconf_blocksizes(>conf);
 if (!blkconf_apply_backend_options(>conf, blk_is_read_only(n->conf.blk),
-- 
2.26.2

[PATCH v6 18/20] hw/block/nvme: factor out pmr setup

[Klaus Jensen]

From: Klaus Jensen 

Signed-off-by: Klaus Jensen 
Reviewed-by: Maxim Levitsky 
---
 hw/block/nvme.c | 95 ++---
 1 file changed, 51 insertions(+), 44 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index d71a5f142d51..7254b66ae199 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -58,6 +58,7 @@
 #define NVME_REG_SIZE 0x1000
 #define NVME_DB_SIZE  4
 #define NVME_CMB_BIR 2
+#define NVME_PMR_BIR 2
 
 #define NVME_GUEST_ERR(trace, fmt, ...) \
 do { \
@@ -1463,6 +1464,55 @@ static void nvme_init_cmb(NvmeCtrl *n, PCIDevice 
*pci_dev)
  PCI_BASE_ADDRESS_MEM_PREFETCH, >ctrl_mem);
 }
 
+static void nvme_init_pmr(NvmeCtrl *n, PCIDevice *pci_dev)
+{
+/* Controller Capabilities register */
+NVME_CAP_SET_PMRS(n->bar.cap, 1);
+
+/* PMR Capabities register */
+n->bar.pmrcap = 0;
+NVME_PMRCAP_SET_RDS(n->bar.pmrcap, 0);
+NVME_PMRCAP_SET_WDS(n->bar.pmrcap, 0);
+NVME_PMRCAP_SET_BIR(n->bar.pmrcap, NVME_PMR_BIR);
+NVME_PMRCAP_SET_PMRTU(n->bar.pmrcap, 0);
+/* Turn on bit 1 support */
+NVME_PMRCAP_SET_PMRWBM(n->bar.pmrcap, 0x02);
+NVME_PMRCAP_SET_PMRTO(n->bar.pmrcap, 0);
+NVME_PMRCAP_SET_CMSS(n->bar.pmrcap, 0);
+
+/* PMR Control register */
+n->bar.pmrctl = 0;
+NVME_PMRCTL_SET_EN(n->bar.pmrctl, 0);
+
+/* PMR Status register */
+n->bar.pmrsts = 0;
+NVME_PMRSTS_SET_ERR(n->bar.pmrsts, 0);
+NVME_PMRSTS_SET_NRDY(n->bar.pmrsts, 0);
+NVME_PMRSTS_SET_HSTS(n->bar.pmrsts, 0);
+NVME_PMRSTS_SET_CBAI(n->bar.pmrsts, 0);
+
+/* PMR Elasticity Buffer Size register */
+n->bar.pmrebs = 0;
+NVME_PMREBS_SET_PMRSZU(n->bar.pmrebs, 0);
+NVME_PMREBS_SET_RBB(n->bar.pmrebs, 0);
+NVME_PMREBS_SET_PMRWBZ(n->bar.pmrebs, 0);
+
+/* PMR Sustained Write Throughput register */
+n->bar.pmrswtp = 0;
+NVME_PMRSWTP_SET_PMRSWTU(n->bar.pmrswtp, 0);
+NVME_PMRSWTP_SET_PMRSWTV(n->bar.pmrswtp, 0);
+
+/* PMR Memory Space Control register */
+n->bar.pmrmsc = 0;
+NVME_PMRMSC_SET_CMSE(n->bar.pmrmsc, 0);
+NVME_PMRMSC_SET_CBA(n->bar.pmrmsc, 0);
+
+pci_register_bar(pci_dev, NVME_PMRCAP_BIR(n->bar.pmrcap),
+ PCI_BASE_ADDRESS_SPACE_MEMORY |
+ PCI_BASE_ADDRESS_MEM_TYPE_64 |
+ PCI_BASE_ADDRESS_MEM_PREFETCH, >pmrdev->mr);
+}
+
 static void nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev)
 {
 uint8_t *pci_conf = pci_dev->config;
@@ -1541,50 +1591,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 if (n->params.cmb_size_mb) {
 nvme_init_cmb(n, pci_dev);
 } else if (n->pmrdev) {
-/* Controller Capabilities register */
-NVME_CAP_SET_PMRS(n->bar.cap, 1);
-
-/* PMR Capabities register */
-n->bar.pmrcap = 0;
-NVME_PMRCAP_SET_RDS(n->bar.pmrcap, 0);
-NVME_PMRCAP_SET_WDS(n->bar.pmrcap, 0);
-NVME_PMRCAP_SET_BIR(n->bar.pmrcap, 2);
-NVME_PMRCAP_SET_PMRTU(n->bar.pmrcap, 0);
-/* Turn on bit 1 support */
-NVME_PMRCAP_SET_PMRWBM(n->bar.pmrcap, 0x02);
-NVME_PMRCAP_SET_PMRTO(n->bar.pmrcap, 0);
-NVME_PMRCAP_SET_CMSS(n->bar.pmrcap, 0);
-
-/* PMR Control register */
-n->bar.pmrctl = 0;
-NVME_PMRCTL_SET_EN(n->bar.pmrctl, 0);
-
-/* PMR Status register */
-n->bar.pmrsts = 0;
-NVME_PMRSTS_SET_ERR(n->bar.pmrsts, 0);
-NVME_PMRSTS_SET_NRDY(n->bar.pmrsts, 0);
-NVME_PMRSTS_SET_HSTS(n->bar.pmrsts, 0);
-NVME_PMRSTS_SET_CBAI(n->bar.pmrsts, 0);
-
-/* PMR Elasticity Buffer Size register */
-n->bar.pmrebs = 0;
-NVME_PMREBS_SET_PMRSZU(n->bar.pmrebs, 0);
-NVME_PMREBS_SET_RBB(n->bar.pmrebs, 0);
-NVME_PMREBS_SET_PMRWBZ(n->bar.pmrebs, 0);
-
-/* PMR Sustained Write Throughput register */
-n->bar.pmrswtp = 0;
-NVME_PMRSWTP_SET_PMRSWTU(n->bar.pmrswtp, 0);
-NVME_PMRSWTP_SET_PMRSWTV(n->bar.pmrswtp, 0);
-
-/* PMR Memory Space Control register */
-n->bar.pmrmsc = 0;
-NVME_PMRMSC_SET_CMSE(n->bar.pmrmsc, 0);
-NVME_PMRMSC_SET_CBA(n->bar.pmrmsc, 0);
-
-pci_register_bar(pci_dev, NVME_PMRCAP_BIR(n->bar.pmrcap),
-PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64 |
-PCI_BASE_ADDRESS_MEM_PREFETCH, >pmrdev->mr);
+nvme_init_pmr(n, pci_dev);
 }
 
 for (i = 0; i < n->num_namespaces; i++) {
-- 
2.26.2

[PATCH v6 14/20] hw/block/nvme: add namespace helpers

[Klaus Jensen]

From: Klaus Jensen 

Introduce some small helpers to make the next patches easier on the eye.

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c |  3 +--
 hw/block/nvme.h | 17 +
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index c83f0bb1ab20..e24d376b0d71 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -1575,8 +1575,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 id_ns->dps = 0;
 id_ns->lbaf[0].ds = BDRV_SECTOR_BITS;
 id_ns->ncap  = id_ns->nuse = id_ns->nsze =
-cpu_to_le64(n->ns_size >>
-id_ns->lbaf[NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas)].ds);
+cpu_to_le64(nvme_ns_nlbas(n, ns));
 }
 }
 
diff --git a/hw/block/nvme.h b/hw/block/nvme.h
index cedc8022dbb3..61dd9b23b81d 100644
--- a/hw/block/nvme.h
+++ b/hw/block/nvme.h
@@ -61,6 +61,17 @@ typedef struct NvmeNamespace {
 NvmeIdNsid_ns;
 } NvmeNamespace;
 
+static inline NvmeLBAF *nvme_ns_lbaf(NvmeNamespace *ns)
+{
+NvmeIdNs *id_ns = >id_ns;
+return _ns->lbaf[NVME_ID_NS_FLBAS_INDEX(id_ns->flbas)];
+}
+
+static inline uint8_t nvme_ns_lbads(NvmeNamespace *ns)
+{
+return nvme_ns_lbaf(ns)->ds;
+}
+
 #define TYPE_NVME "nvme"
 #define NVME(obj) \
 OBJECT_CHECK(NvmeCtrl, (obj), TYPE_NVME)
@@ -97,4 +108,10 @@ typedef struct NvmeCtrl {
 NvmeIdCtrl  id_ctrl;
 } NvmeCtrl;
 
+/* calculate the number of LBAs that the namespace can accomodate */
+static inline uint64_t nvme_ns_nlbas(NvmeCtrl *n, NvmeNamespace *ns)
+{
+return n->ns_size >> nvme_ns_lbads(ns);
+}
+
 #endif /* HW_NVME_H */
-- 
2.26.2

[PATCH v6 20/20] hw/block/nvme: factor out controller identify setup

[Klaus Jensen]

From: Klaus Jensen 

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 49 ++---
 1 file changed, 26 insertions(+), 23 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 2addcc86034a..149abef497c4 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -1535,32 +1535,11 @@ static void nvme_init_pci(NvmeCtrl *n, PCIDevice 
*pci_dev)
 }
 }
 
-static void nvme_realize(PCIDevice *pci_dev, Error **errp)
+static void nvme_init_ctrl(NvmeCtrl *n, PCIDevice *pci_dev)
 {
-NvmeCtrl *n = NVME(pci_dev);
 NvmeIdCtrl *id = >id_ctrl;
-Error *local_err = NULL;
+uint8_t *pci_conf = pci_dev->config;
 
-int i;
-uint8_t *pci_conf;
-
-nvme_check_constraints(n, _err);
-if (local_err) {
-error_propagate(errp, local_err);
-return;
-}
-
-nvme_init_state(n);
-
-nvme_init_blk(n, _err);
-if (local_err) {
-error_propagate(errp, local_err);
-return;
-}
-
-nvme_init_pci(n, pci_dev);
-
-pci_conf = pci_dev->config;
 id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
 id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
 strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
@@ -1593,6 +1572,30 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 
 n->bar.vs = 0x00010200;
 n->bar.intmc = n->bar.intms = 0;
+}
+
+static void nvme_realize(PCIDevice *pci_dev, Error **errp)
+{
+NvmeCtrl *n = NVME(pci_dev);
+Error *local_err = NULL;
+
+int i;
+
+nvme_check_constraints(n, _err);
+if (local_err) {
+error_propagate(errp, local_err);
+return;
+}
+
+nvme_init_state(n);
+nvme_init_blk(n, _err);
+if (local_err) {
+error_propagate(errp, local_err);
+return;
+}
+
+nvme_init_pci(n, pci_dev);
+nvme_init_ctrl(n, pci_dev);
 
 for (i = 0; i < n->num_namespaces; i++) {
 nvme_init_namespace(n, >namespaces[i], _err);
-- 
2.26.2

[PATCH v6 06/20] hw/block/nvme: refactor nvme_addr_read

[Klaus Jensen]

From: Klaus Jensen 

Pull the controller memory buffer check to its own function. The check
will be used on its own in later patches.

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 16 
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 4058f2c79796..623a88be93dc 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -66,14 +66,22 @@
 
 static void nvme_process_sq(void *opaque);
 
+static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr)
+{
+hwaddr low = n->ctrl_mem.addr;
+hwaddr hi  = n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size);
+
+return addr >= low && addr < hi;
+}
+
 static void nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
 {
-if (n->cmbsz && addr >= n->ctrl_mem.addr &&
-addr < (n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size))) {
+if (n->cmbsz && nvme_addr_is_cmb(n, addr)) {
 memcpy(buf, (void *)>cmbuf[addr - n->ctrl_mem.addr], size);
-} else {
-pci_dma_read(>parent_obj, addr, buf, size);
+return;
 }
+
+pci_dma_read(>parent_obj, addr, buf, size);
 }
 
 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
-- 
2.26.2

[PATCH v6 12/20] hw/block/nvme: factor out device state setup

[Klaus Jensen]

From: Klaus Jensen 

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 22 +-
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 1d4fbba9c5da..617b2f0005df 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -1401,6 +1401,17 @@ static void nvme_check_constraints(NvmeCtrl *n, Error 
**errp)
 }
 }
 
+static void nvme_init_state(NvmeCtrl *n)
+{
+n->num_namespaces = 1;
+/* add one to max_ioqpairs to account for the admin queue pair */
+n->reg_size = pow2ceil(NVME_REG_SIZE +
+   2 * (n->params.max_ioqpairs + 1) * NVME_DB_SIZE);
+n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
+n->sq = g_new0(NvmeSQueue *, n->params.max_ioqpairs + 1);
+n->cq = g_new0(NvmeCQueue *, n->params.max_ioqpairs + 1);
+}
+
 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 {
 NvmeCtrl *n = NVME(pci_dev);
@@ -1417,6 +1428,8 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 return;
 }
 
+nvme_init_state(n);
+
 bs_size = blk_getlength(n->conf.blk);
 if (bs_size < 0) {
 error_setg(errp, "could not get backing file size");
@@ -1435,17 +1448,8 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 pci_config_set_class(pci_dev->config, PCI_CLASS_STORAGE_EXPRESS);
 pcie_endpoint_cap_init(pci_dev, 0x80);
 
-n->num_namespaces = 1;
-
-/* add one to max_ioqpairs to account for the admin queue pair */
-n->reg_size = pow2ceil(NVME_REG_SIZE +
-   2 * (n->params.max_ioqpairs + 1) * NVME_DB_SIZE);
 n->ns_size = bs_size / (uint64_t)n->num_namespaces;
 
-n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
-n->sq = g_new0(NvmeSQueue *, n->params.max_ioqpairs + 1);
-n->cq = g_new0(NvmeCQueue *, n->params.max_ioqpairs + 1);
-
 memory_region_init_io(>iomem, OBJECT(n), _mmio_ops, n,
   "nvme", n->reg_size);
 pci_register_bar(pci_dev, 0,
-- 
2.26.2

[PATCH v6 17/20] hw/block/nvme: factor out cmb setup

[Klaus Jensen]

From: Klaus Jensen 

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 43 ---
 1 file changed, 24 insertions(+), 19 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 2fcab706c759..d71a5f142d51 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -57,6 +57,7 @@
 
 #define NVME_REG_SIZE 0x1000
 #define NVME_DB_SIZE  4
+#define NVME_CMB_BIR 2
 
 #define NVME_GUEST_ERR(trace, fmt, ...) \
 do { \
@@ -1440,6 +1441,28 @@ static void nvme_init_namespace(NvmeCtrl *n, 
NvmeNamespace *ns, Error **errp)
 id_ns->nuse = id_ns->ncap;
 }
 
+static void nvme_init_cmb(NvmeCtrl *n, PCIDevice *pci_dev)
+{
+NVME_CMBLOC_SET_BIR(n->bar.cmbloc, NVME_CMB_BIR);
+NVME_CMBLOC_SET_OFST(n->bar.cmbloc, 0);
+
+NVME_CMBSZ_SET_SQS(n->bar.cmbsz, 1);
+NVME_CMBSZ_SET_CQS(n->bar.cmbsz, 0);
+NVME_CMBSZ_SET_LISTS(n->bar.cmbsz, 0);
+NVME_CMBSZ_SET_RDS(n->bar.cmbsz, 1);
+NVME_CMBSZ_SET_WDS(n->bar.cmbsz, 1);
+NVME_CMBSZ_SET_SZU(n->bar.cmbsz, 2); /* MBs */
+NVME_CMBSZ_SET_SZ(n->bar.cmbsz, n->params.cmb_size_mb);
+
+n->cmbuf = g_malloc0(NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
+memory_region_init_io(>ctrl_mem, OBJECT(n), _cmb_ops, n,
+  "nvme-cmb", NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
+pci_register_bar(pci_dev, NVME_CMBLOC_BIR(n->bar.cmbloc),
+ PCI_BASE_ADDRESS_SPACE_MEMORY |
+ PCI_BASE_ADDRESS_MEM_TYPE_64 |
+ PCI_BASE_ADDRESS_MEM_PREFETCH, >ctrl_mem);
+}
+
 static void nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev)
 {
 uint8_t *pci_conf = pci_dev->config;
@@ -1516,25 +1539,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 n->bar.intmc = n->bar.intms = 0;
 
 if (n->params.cmb_size_mb) {
-
-NVME_CMBLOC_SET_BIR(n->bar.cmbloc, 2);
-NVME_CMBLOC_SET_OFST(n->bar.cmbloc, 0);
-
-NVME_CMBSZ_SET_SQS(n->bar.cmbsz, 1);
-NVME_CMBSZ_SET_CQS(n->bar.cmbsz, 0);
-NVME_CMBSZ_SET_LISTS(n->bar.cmbsz, 0);
-NVME_CMBSZ_SET_RDS(n->bar.cmbsz, 1);
-NVME_CMBSZ_SET_WDS(n->bar.cmbsz, 1);
-NVME_CMBSZ_SET_SZU(n->bar.cmbsz, 2); /* MBs */
-NVME_CMBSZ_SET_SZ(n->bar.cmbsz, n->params.cmb_size_mb);
-
-n->cmbuf = g_malloc0(NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
-memory_region_init_io(>ctrl_mem, OBJECT(n), _cmb_ops, n,
-  "nvme-cmb", NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
-pci_register_bar(pci_dev, NVME_CMBLOC_BIR(n->bar.cmbloc),
-PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64 |
-PCI_BASE_ADDRESS_MEM_PREFETCH, >ctrl_mem);
-
+nvme_init_cmb(n, pci_dev);
 } else if (n->pmrdev) {
 /* Controller Capabilities register */
 NVME_CAP_SET_PMRS(n->bar.cap, 1);
-- 
2.26.2

[PATCH v6 19/20] hw/block/nvme: do cmb/pmr init as part of pci init

[Klaus Jensen]

From: Klaus Jensen 

Signed-off-by: Klaus Jensen 
Reviewed-by: Maxim Levitsky 
---
 hw/block/nvme.c | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 7254b66ae199..2addcc86034a 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -1527,6 +1527,12 @@ static void nvme_init_pci(NvmeCtrl *n, PCIDevice 
*pci_dev)
 pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
  PCI_BASE_ADDRESS_MEM_TYPE_64, >iomem);
 msix_init_exclusive_bar(pci_dev, n->params.max_ioqpairs + 1, 4, NULL);
+
+if (n->params.cmb_size_mb) {
+nvme_init_cmb(n, pci_dev);
+} else if (n->pmrdev) {
+nvme_init_pmr(n, pci_dev);
+}
 }
 
 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
@@ -1588,12 +1594,6 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 n->bar.vs = 0x00010200;
 n->bar.intmc = n->bar.intms = 0;
 
-if (n->params.cmb_size_mb) {
-nvme_init_cmb(n, pci_dev);
-} else if (n->pmrdev) {
-nvme_init_pmr(n, pci_dev);
-}
-
 for (i = 0; i < n->num_namespaces; i++) {
 nvme_init_namespace(n, >namespaces[i], _err);
 if (local_err) {
-- 
2.26.2

[PATCH v6 09/20] hw/block/nvme: add max_ioqpairs device parameter

[Klaus Jensen]

From: Klaus Jensen 

The num_queues device paramater has a slightly confusing meaning because
it accounts for the admin queue pair which is not really optional.
Secondly, it is really a maximum value of queues allowed.

Add a new max_ioqpairs parameter that only accounts for I/O queue pairs,
but keep num_queues for compatibility.

Signed-off-by: Klaus Jensen 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 49 ++---
 hw/block/nvme.h |  3 ++-
 2 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index a2e6734c7a25..88ab5d55b8c9 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -20,7 +20,7 @@
  *  -device nvme,drive=,serial=,id=, \
  *  cmb_size_mb=, \
  *  [pmrdev=,] \
- *  num_queues=
+ *  max_ioqpairs=
  *
  * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
  * offset 0 in BAR2 and supports only WDS, RDS and SQS for now.
@@ -36,6 +36,7 @@
 
 #include "qemu/osdep.h"
 #include "qemu/units.h"
+#include "qemu/error-report.h"
 #include "hw/block/block.h"
 #include "hw/pci/msix.h"
 #include "hw/pci/pci.h"
@@ -86,12 +87,12 @@ static void nvme_addr_read(NvmeCtrl *n, hwaddr addr, void 
*buf, int size)
 
 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
 {
-return sqid < n->params.num_queues && n->sq[sqid] != NULL ? 0 : -1;
+return sqid < n->params.max_ioqpairs + 1 && n->sq[sqid] != NULL ? 0 : -1;
 }
 
 static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
 {
-return cqid < n->params.num_queues && n->cq[cqid] != NULL ? 0 : -1;
+return cqid < n->params.max_ioqpairs + 1 && n->cq[cqid] != NULL ? 0 : -1;
 }
 
 static void nvme_inc_cq_tail(NvmeCQueue *cq)
@@ -809,8 +810,8 @@ static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, 
NvmeRequest *req)
 trace_pci_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
 break;
 case NVME_NUMBER_OF_QUEUES:
-result = cpu_to_le32((n->params.num_queues - 2) |
- ((n->params.num_queues - 2) << 16));
+result = cpu_to_le32((n->params.max_ioqpairs - 1) |
+ ((n->params.max_ioqpairs - 1) << 16));
 trace_pci_nvme_getfeat_numq(result);
 break;
 case NVME_TIMESTAMP:
@@ -854,10 +855,10 @@ static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd 
*cmd, NvmeRequest *req)
 case NVME_NUMBER_OF_QUEUES:
 trace_pci_nvme_setfeat_numq((dw11 & 0x) + 1,
 ((dw11 >> 16) & 0x) + 1,
-n->params.num_queues - 1,
-n->params.num_queues - 1);
-req->cqe.result = cpu_to_le32((n->params.num_queues - 2) |
-  ((n->params.num_queues - 2) << 16));
+n->params.max_ioqpairs,
+n->params.max_ioqpairs);
+req->cqe.result = cpu_to_le32((n->params.max_ioqpairs - 1) |
+  ((n->params.max_ioqpairs - 1) << 16));
 break;
 case NVME_TIMESTAMP:
 return nvme_set_feature_timestamp(n, cmd);
@@ -928,12 +929,12 @@ static void nvme_clear_ctrl(NvmeCtrl *n)
 
 blk_drain(n->conf.blk);
 
-for (i = 0; i < n->params.num_queues; i++) {
+for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
 if (n->sq[i] != NULL) {
 nvme_free_sq(n->sq[i], n);
 }
 }
-for (i = 0; i < n->params.num_queues; i++) {
+for (i = 0; i < n->params.max_ioqpairs + 1; i++) {
 if (n->cq[i] != NULL) {
 nvme_free_cq(n->cq[i], n);
 }
@@ -1364,8 +1365,17 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 int64_t bs_size;
 uint8_t *pci_conf;
 
-if (!n->params.num_queues) {
-error_setg(errp, "num_queues can't be zero");
+if (n->params.num_queues) {
+warn_report("num_queues is deprecated; please use max_ioqpairs "
+"instead");
+
+n->params.max_ioqpairs = n->params.num_queues - 1;
+}
+
+if (n->params.max_ioqpairs < 1 ||
+n->params.max_ioqpairs > PCI_MSIX_FLAGS_QSIZE) {
+error_setg(errp, "max_ioqpairs must be between 1 and %d",
+   PCI_MSIX_FLAGS_QSIZE);
 return;
 }
 
@@ -1415,21 +1425,21 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 
 n->num_namespaces = 1;
 
-/* num_queues is really number of pairs, so each has two doorbells */
+/* add one to max_ioqpairs to account for the admin queue pair */
 n->reg_size = pow2ceil(NVME_REG_SIZE +
-   2 * n->params.num_queues * NVME_DB_SIZE);
+   2 * (n->params.max_ioqpairs + 1) * NVME_DB_SIZE);
 n->ns_size = bs_size / (uint64_t)n->num_namespaces;
 
 n->namespaces = g_new0(NvmeNamespace, 

[PATCH v6 16/20] hw/block/nvme: factor out pci setup

[Klaus Jensen]

From: Klaus Jensen 

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 30 ++
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 8e9312534fcd..2fcab706c759 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -1440,6 +1440,22 @@ static void nvme_init_namespace(NvmeCtrl *n, 
NvmeNamespace *ns, Error **errp)
 id_ns->nuse = id_ns->ncap;
 }
 
+static void nvme_init_pci(NvmeCtrl *n, PCIDevice *pci_dev)
+{
+uint8_t *pci_conf = pci_dev->config;
+
+pci_conf[PCI_INTERRUPT_PIN] = 1;
+pci_config_set_prog_interface(pci_conf, 0x2);
+pci_config_set_class(pci_conf, PCI_CLASS_STORAGE_EXPRESS);
+pcie_endpoint_cap_init(pci_dev, 0x80);
+
+memory_region_init_io(>iomem, OBJECT(n), _mmio_ops, n, "nvme",
+  n->reg_size);
+pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
+ PCI_BASE_ADDRESS_MEM_TYPE_64, >iomem);
+msix_init_exclusive_bar(pci_dev, n->params.max_ioqpairs + 1, 4, NULL);
+}
+
 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 {
 NvmeCtrl *n = NVME(pci_dev);
@@ -1463,19 +1479,9 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 return;
 }
 
+nvme_init_pci(n, pci_dev);
+
 pci_conf = pci_dev->config;
-pci_conf[PCI_INTERRUPT_PIN] = 1;
-pci_config_set_prog_interface(pci_dev->config, 0x2);
-pci_config_set_class(pci_dev->config, PCI_CLASS_STORAGE_EXPRESS);
-pcie_endpoint_cap_init(pci_dev, 0x80);
-
-memory_region_init_io(>iomem, OBJECT(n), _mmio_ops, n,
-  "nvme", n->reg_size);
-pci_register_bar(pci_dev, 0,
-PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64,
->iomem);
-msix_init_exclusive_bar(pci_dev, n->params.max_ioqpairs + 1, 4, NULL);
-
 id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
 id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
 strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
-- 
2.26.2

[PATCH v6 07/20] hw/block/nvme: fix pin-based interrupt behavior

[Klaus Jensen]

From: Klaus Jensen 

First, since the device only supports MSI-X or pin-based interrupt, if
MSI-X is not enabled, it should not accept interrupt vectors different
from 0 when creating completion queues.

Secondly, the irq_status NvmeCtrl member is meant to be compared to the
INTMS register, so it should only be 32 bits wide. And it is really only
useful when used with multi-message MSI.

Third, since we do not force a 1-to-1 correspondence between cqid and
interrupt vector, the irq_status register should not have bits set
according to cqid, but according to the associated interrupt vector.

Fix these issues, but keep irq_status available so we can easily support
multi-message MSI down the line.

Fixes: 5e9aa92eb1a5 ("hw/block: Fix pin-based interrupt behaviour of NVMe")
Cc: "Michael S. Tsirkin" 
Cc: Marcel Apfelbaum 
Signed-off-by: Klaus Jensen 
---
 hw/block/nvme.c | 12 
 hw/block/nvme.h |  2 +-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 623a88be93dc..c9d10df1f763 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -138,8 +138,8 @@ static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
 msix_notify(&(n->parent_obj), cq->vector);
 } else {
 trace_pci_nvme_irq_pin();
-assert(cq->cqid < 64);
-n->irq_status |= 1 << cq->cqid;
+assert(cq->vector < 32);
+n->irq_status |= 1 << cq->vector;
 nvme_irq_check(n);
 }
 } else {
@@ -153,8 +153,8 @@ static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
 if (msix_enabled(&(n->parent_obj))) {
 return;
 } else {
-assert(cq->cqid < 64);
-n->irq_status &= ~(1 << cq->cqid);
+assert(cq->vector < 32);
+n->irq_status &= ~(1 << cq->vector);
 nvme_irq_check(n);
 }
 }
@@ -653,6 +653,10 @@ static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
 trace_pci_nvme_err_invalid_create_cq_addr(prp1);
 return NVME_INVALID_FIELD | NVME_DNR;
 }
+if (unlikely(!msix_enabled(>parent_obj) && vector)) {
+trace_pci_nvme_err_invalid_create_cq_vector(vector);
+return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
+}
 if (unlikely(vector > n->params.num_queues)) {
 trace_pci_nvme_err_invalid_create_cq_vector(vector);
 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
diff --git a/hw/block/nvme.h b/hw/block/nvme.h
index 9df244c93c02..91f16c812582 100644
--- a/hw/block/nvme.h
+++ b/hw/block/nvme.h
@@ -84,7 +84,7 @@ typedef struct NvmeCtrl {
 uint32_tcmbsz;
 uint32_tcmbloc;
 uint8_t *cmbuf;
-uint64_tirq_status;
+uint32_tirq_status;
 uint64_thost_timestamp; /* Timestamp sent by the host 
*/
 uint64_ttimestamp_set_qemu_clock_ms;/* QEMU clock time */
 
-- 
2.26.2

[PATCH v6 05/20] hw/block/nvme: use constants in identify

[Klaus Jensen]

From: Klaus Jensen 

Signed-off-by: Klaus Jensen 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c  | 8 
 include/block/nvme.h | 8 
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index e26db7591574..4058f2c79796 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -693,7 +693,7 @@ static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeIdentify 
*c)
 
 static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeIdentify *c)
 {
-static const int data_len = 4 * KiB;
+static const int data_len = NVME_IDENTIFY_DATA_SIZE;
 uint32_t min_nsid = le32_to_cpu(c->nsid);
 uint64_t prp1 = le64_to_cpu(c->prp1);
 uint64_t prp2 = le64_to_cpu(c->prp2);
@@ -723,11 +723,11 @@ static uint16_t nvme_identify(NvmeCtrl *n, NvmeCmd *cmd)
 NvmeIdentify *c = (NvmeIdentify *)cmd;
 
 switch (le32_to_cpu(c->cns)) {
-case 0x00:
+case NVME_ID_CNS_NS:
 return nvme_identify_ns(n, c);
-case 0x01:
+case NVME_ID_CNS_CTRL:
 return nvme_identify_ctrl(n, c);
-case 0x02:
+case NVME_ID_CNS_NS_ACTIVE_LIST:
 return nvme_identify_nslist(n, c);
 default:
 trace_pci_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
diff --git a/include/block/nvme.h b/include/block/nvme.h
index 5525c8e34308..1720ee1d5158 100644
--- a/include/block/nvme.h
+++ b/include/block/nvme.h
@@ -705,6 +705,14 @@ typedef struct NvmePSD {
 uint8_t resv[16];
 } NvmePSD;
 
+#define NVME_IDENTIFY_DATA_SIZE 4096
+
+enum {
+NVME_ID_CNS_NS = 0x0,
+NVME_ID_CNS_CTRL   = 0x1,
+NVME_ID_CNS_NS_ACTIVE_LIST = 0x2,
+};
+
 typedef struct NvmeIdCtrl {
 uint16_tvid;
 uint16_tssvid;
-- 
2.26.2

[PATCH v6 10/20] hw/block/nvme: remove redundant cmbloc/cmbsz members

[Klaus Jensen]

From: Klaus Jensen 

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 7 ++-
 hw/block/nvme.h | 2 --
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 88ab5d55b8c9..e93ed729aef5 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -77,7 +77,7 @@ static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr)
 
 static void nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
 {
-if (n->cmbsz && nvme_addr_is_cmb(n, addr)) {
+if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr)) {
 memcpy(buf, (void *)>cmbuf[addr - n->ctrl_mem.addr], size);
 return;
 }
@@ -171,7 +171,7 @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector 
*iov, uint64_t prp1,
 if (unlikely(!prp1)) {
 trace_pci_nvme_err_invalid_prp();
 return NVME_INVALID_FIELD | NVME_DNR;
-} else if (n->cmbsz && prp1 >= n->ctrl_mem.addr &&
+} else if (n->bar.cmbsz && prp1 >= n->ctrl_mem.addr &&
prp1 < n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size)) {
 qsg->nsg = 0;
 qemu_iovec_init(iov, num_prps);
@@ -1487,9 +1487,6 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 NVME_CMBSZ_SET_SZU(n->bar.cmbsz, 2); /* MBs */
 NVME_CMBSZ_SET_SZ(n->bar.cmbsz, n->params.cmb_size_mb);
 
-n->cmbloc = n->bar.cmbloc;
-n->cmbsz = n->bar.cmbsz;
-
 n->cmbuf = g_malloc0(NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
 memory_region_init_io(>ctrl_mem, OBJECT(n), _cmb_ops, n,
   "nvme-cmb", NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
diff --git a/hw/block/nvme.h b/hw/block/nvme.h
index 26c38bd913be..cedc8022dbb3 100644
--- a/hw/block/nvme.h
+++ b/hw/block/nvme.h
@@ -82,8 +82,6 @@ typedef struct NvmeCtrl {
 uint32_tnum_namespaces;
 uint32_tmax_q_ents;
 uint64_tns_size;
-uint32_tcmbsz;
-uint32_tcmbloc;
 uint8_t *cmbuf;
 uint32_tirq_status;
 uint64_thost_timestamp; /* Timestamp sent by the host 
*/
-- 
2.26.2

[PATCH v6 08/20] hw/block/nvme: allow use of any valid msix vector

[Klaus Jensen]

From: Klaus Jensen 

If the device uses MSI-X, any of the 2048 MSI-X interrupt vectors are
valid. If the device is not using MSI-X, vector will and can only be
zero at this point.

Cc: "Michael S. Tsirkin" 
Cc: Marcel Apfelbaum 
Signed-off-by: Klaus Jensen 
---
 hw/block/nvme.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index c9d10df1f763..a2e6734c7a25 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -657,7 +657,7 @@ static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
 trace_pci_nvme_err_invalid_create_cq_vector(vector);
 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
 }
-if (unlikely(vector > n->params.num_queues)) {
+if (unlikely(vector > PCI_MSIX_FLAGS_QSIZE)) {
 trace_pci_nvme_err_invalid_create_cq_vector(vector);
 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
 }
-- 
2.26.2

[PATCH v6 02/20] hw/block/nvme: rename trace events to pci_nvme

[Klaus Jensen]

From: Klaus Jensen 

Change the prefix of all nvme device related trace events to 'pci_nvme'
to not clash with trace events from the nvme block driver.

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c   | 198 +-
 hw/block/trace-events | 180 +++---
 2 files changed, 188 insertions(+), 190 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 1d7d7fb3c67a..6702812ecdd1 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -126,16 +126,16 @@ static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
 {
 if (cq->irq_enabled) {
 if (msix_enabled(&(n->parent_obj))) {
-trace_nvme_irq_msix(cq->vector);
+trace_pci_nvme_irq_msix(cq->vector);
 msix_notify(&(n->parent_obj), cq->vector);
 } else {
-trace_nvme_irq_pin();
+trace_pci_nvme_irq_pin();
 assert(cq->cqid < 64);
 n->irq_status |= 1 << cq->cqid;
 nvme_irq_check(n);
 }
 } else {
-trace_nvme_irq_masked();
+trace_pci_nvme_irq_masked();
 }
 }
 
@@ -160,7 +160,7 @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector 
*iov, uint64_t prp1,
 int num_prps = (len >> n->page_bits) + 1;
 
 if (unlikely(!prp1)) {
-trace_nvme_err_invalid_prp();
+trace_pci_nvme_err_invalid_prp();
 return NVME_INVALID_FIELD | NVME_DNR;
 } else if (n->cmbsz && prp1 >= n->ctrl_mem.addr &&
prp1 < n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size)) {
@@ -174,7 +174,7 @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector 
*iov, uint64_t prp1,
 len -= trans_len;
 if (len) {
 if (unlikely(!prp2)) {
-trace_nvme_err_invalid_prp2_missing();
+trace_pci_nvme_err_invalid_prp2_missing();
 goto unmap;
 }
 if (len > n->page_size) {
@@ -190,7 +190,7 @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector 
*iov, uint64_t prp1,
 
 if (i == n->max_prp_ents - 1 && len > n->page_size) {
 if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
-trace_nvme_err_invalid_prplist_ent(prp_ent);
+trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
 goto unmap;
 }
 
@@ -203,7 +203,7 @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector 
*iov, uint64_t prp1,
 }
 
 if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
-trace_nvme_err_invalid_prplist_ent(prp_ent);
+trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
 goto unmap;
 }
 
@@ -218,7 +218,7 @@ static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector 
*iov, uint64_t prp1,
 }
 } else {
 if (unlikely(prp2 & (n->page_size - 1))) {
-trace_nvme_err_invalid_prp2_align(prp2);
+trace_pci_nvme_err_invalid_prp2_align(prp2);
 goto unmap;
 }
 if (qsg->nsg) {
@@ -266,20 +266,20 @@ static uint16_t nvme_dma_read_prp(NvmeCtrl *n, uint8_t 
*ptr, uint32_t len,
 QEMUIOVector iov;
 uint16_t status = NVME_SUCCESS;
 
-trace_nvme_dma_read(prp1, prp2);
+trace_pci_nvme_dma_read(prp1, prp2);
 
 if (nvme_map_prp(, , prp1, prp2, len, n)) {
 return NVME_INVALID_FIELD | NVME_DNR;
 }
 if (qsg.nsg > 0) {
 if (unlikely(dma_buf_read(ptr, len, ))) {
-trace_nvme_err_invalid_dma();
+trace_pci_nvme_err_invalid_dma();
 status = NVME_INVALID_FIELD | NVME_DNR;
 }
 qemu_sglist_destroy();
 } else {
 if (unlikely(qemu_iovec_from_buf(, 0, ptr, len) != len)) {
-trace_nvme_err_invalid_dma();
+trace_pci_nvme_err_invalid_dma();
 status = NVME_INVALID_FIELD | NVME_DNR;
 }
 qemu_iovec_destroy();
@@ -368,7 +368,7 @@ static uint16_t nvme_write_zeros(NvmeCtrl *n, NvmeNamespace 
*ns, NvmeCmd *cmd,
 uint32_t count = nlb << data_shift;
 
 if (unlikely(slba + nlb > ns->id_ns.nsze)) {
-trace_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
+trace_pci_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
 return NVME_LBA_RANGE | NVME_DNR;
 }
 
@@ -396,11 +396,11 @@ static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, 
NvmeCmd *cmd,
 int is_write = rw->opcode == NVME_CMD_WRITE ? 1 : 0;
 enum BlockAcctType acct = is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ;
 
-trace_nvme_rw(is_write ? "write" : "read", nlb, data_size, slba);
+trace_pci_nvme_rw(is_write ? "write" : "read", nlb, data_size, slba);
 
 if (unlikely((slba + nlb) > ns->id_ns.nsze)) {
 block_acct_invalid(blk_get_stats(n->conf.blk), 

[PATCH v6 00/20] nvme: small fixes, refactoring and cleanups

[Klaus Jensen]

From: Klaus Jensen 

Changes since v5

* Prefixed all patches with "hw/block/nvme" to avoid confusion with the
  nvme block driver.

* Added patch two patches:

hw/block/nvme: fix pin-based interrupt behavior
hw/block/nvme: allow use of any valid msix vector

  These were previously posted separately, but I'm including them in this
  series since I didnt get any response on the separate series anyway.

* Fixed Maxim's email in the R-b on "hw/block/nvme: refactor
  nvme_addr_read"


Klaus Jensen (20):
  hw/block/nvme: fix pci doorbell size calculation
  hw/block/nvme: rename trace events to pci_nvme
  hw/block/nvme: remove superfluous breaks
  hw/block/nvme: move device parameters to separate struct
  hw/block/nvme: use constants in identify
  hw/block/nvme: refactor nvme_addr_read
  hw/block/nvme: fix pin-based interrupt behavior
  hw/block/nvme: allow use of any valid msix vector
  hw/block/nvme: add max_ioqpairs device parameter
  hw/block/nvme: remove redundant cmbloc/cmbsz members
  hw/block/nvme: factor out property/constraint checks
  hw/block/nvme: factor out device state setup
  hw/block/nvme: factor out block backend setup
  hw/block/nvme: add namespace helpers
  hw/block/nvme: factor out namespace setup
  hw/block/nvme: factor out pci setup
  hw/block/nvme: factor out cmb setup
  hw/block/nvme: factor out pmr setup
  hw/block/nvme: do cmb/pmr init as part of pci init
  hw/block/nvme: factor out controller identify setup

 hw/block/nvme.c   | 555 --
 hw/block/nvme.h   |  33 ++-
 hw/block/trace-events | 180 +++---
 include/block/nvme.h  |   8 +
 4 files changed, 438 insertions(+), 338 deletions(-)

-- 
2.26.2

[PATCH v6 01/20] hw/block/nvme: fix pci doorbell size calculation

[Klaus Jensen]

From: Klaus Jensen 

The size of the BAR is 0x1000 (main registers) + 8 bytes for each
queue. Currently, the size of the BAR is calculated like so:

n->reg_size = pow2ceil(0x1004 + 2 * (n->num_queues + 1) * 4);

Since the 'num_queues' parameter already accounts for the admin queue,
this should in any case not need to be incremented by one. Also, the
size should be initialized to (0x1000).

n->reg_size = pow2ceil(0x1000 + 2 * n->num_queues * 4);

This, with the default value of num_queues (64), we will set aside room
for 1 admin queue and 63 I/O queues (4 bytes per doorbell, 2 doorbells
per queue).

Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 9b453423cf2c..1d7d7fb3c67a 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -54,6 +54,9 @@
 #include "trace.h"
 #include "nvme.h"
 
+#define NVME_REG_SIZE 0x1000
+#define NVME_DB_SIZE  4
+
 #define NVME_GUEST_ERR(trace, fmt, ...) \
 do { \
 (trace_##trace)(__VA_ARGS__); \
@@ -1403,7 +1406,9 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 pcie_endpoint_cap_init(pci_dev, 0x80);
 
 n->num_namespaces = 1;
-n->reg_size = pow2ceil(0x1004 + 2 * (n->num_queues + 1) * 4);
+
+/* num_queues is really number of pairs, so each has two doorbells */
+n->reg_size = pow2ceil(NVME_REG_SIZE + 2 * n->num_queues * NVME_DB_SIZE);
 n->ns_size = bs_size / (uint64_t)n->num_namespaces;
 
 n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
-- 
2.26.2

[PATCH v6 04/20] hw/block/nvme: move device parameters to separate struct

[Klaus Jensen]

From: Klaus Jensen 

Move device configuration parameters to separate struct to make it
explicit what is configurable and what is set internally.

Signed-off-by: Klaus Jensen 
Signed-off-by: Klaus Jensen 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Maxim Levitsky 
---
 hw/block/nvme.c | 49 ++---
 hw/block/nvme.h | 11 ---
 2 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index f67499d85f3a..e26db7591574 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -78,12 +78,12 @@ static void nvme_addr_read(NvmeCtrl *n, hwaddr addr, void 
*buf, int size)
 
 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
 {
-return sqid < n->num_queues && n->sq[sqid] != NULL ? 0 : -1;
+return sqid < n->params.num_queues && n->sq[sqid] != NULL ? 0 : -1;
 }
 
 static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
 {
-return cqid < n->num_queues && n->cq[cqid] != NULL ? 0 : -1;
+return cqid < n->params.num_queues && n->cq[cqid] != NULL ? 0 : -1;
 }
 
 static void nvme_inc_cq_tail(NvmeCQueue *cq)
@@ -645,7 +645,7 @@ static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
 trace_pci_nvme_err_invalid_create_cq_addr(prp1);
 return NVME_INVALID_FIELD | NVME_DNR;
 }
-if (unlikely(vector > n->num_queues)) {
+if (unlikely(vector > n->params.num_queues)) {
 trace_pci_nvme_err_invalid_create_cq_vector(vector);
 return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
 }
@@ -797,7 +797,8 @@ static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, 
NvmeRequest *req)
 trace_pci_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
 break;
 case NVME_NUMBER_OF_QUEUES:
-result = cpu_to_le32((n->num_queues - 2) | ((n->num_queues - 2) << 
16));
+result = cpu_to_le32((n->params.num_queues - 2) |
+ ((n->params.num_queues - 2) << 16));
 trace_pci_nvme_getfeat_numq(result);
 break;
 case NVME_TIMESTAMP:
@@ -841,9 +842,10 @@ static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd 
*cmd, NvmeRequest *req)
 case NVME_NUMBER_OF_QUEUES:
 trace_pci_nvme_setfeat_numq((dw11 & 0x) + 1,
 ((dw11 >> 16) & 0x) + 1,
-n->num_queues - 1, n->num_queues - 1);
-req->cqe.result =
-cpu_to_le32((n->num_queues - 2) | ((n->num_queues - 2) << 16));
+n->params.num_queues - 1,
+n->params.num_queues - 1);
+req->cqe.result = cpu_to_le32((n->params.num_queues - 2) |
+  ((n->params.num_queues - 2) << 16));
 break;
 case NVME_TIMESTAMP:
 return nvme_set_feature_timestamp(n, cmd);
@@ -914,12 +916,12 @@ static void nvme_clear_ctrl(NvmeCtrl *n)
 
 blk_drain(n->conf.blk);
 
-for (i = 0; i < n->num_queues; i++) {
+for (i = 0; i < n->params.num_queues; i++) {
 if (n->sq[i] != NULL) {
 nvme_free_sq(n->sq[i], n);
 }
 }
-for (i = 0; i < n->num_queues; i++) {
+for (i = 0; i < n->params.num_queues; i++) {
 if (n->cq[i] != NULL) {
 nvme_free_cq(n->cq[i], n);
 }
@@ -1350,7 +1352,7 @@ static void nvme_realize(PCIDevice *pci_dev, Error **errp)
 int64_t bs_size;
 uint8_t *pci_conf;
 
-if (!n->num_queues) {
+if (!n->params.num_queues) {
 error_setg(errp, "num_queues can't be zero");
 return;
 }
@@ -1366,12 +1368,12 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 return;
 }
 
-if (!n->serial) {
+if (!n->params.serial) {
 error_setg(errp, "serial property not set");
 return;
 }
 
-if (!n->cmb_size_mb && n->pmrdev) {
+if (!n->params.cmb_size_mb && n->pmrdev) {
 if (host_memory_backend_is_mapped(n->pmrdev)) {
 char *path = 
object_get_canonical_path_component(OBJECT(n->pmrdev));
 error_setg(errp, "can't use already busy memdev: %s", path);
@@ -1402,25 +1404,26 @@ static void nvme_realize(PCIDevice *pci_dev, Error 
**errp)
 n->num_namespaces = 1;
 
 /* num_queues is really number of pairs, so each has two doorbells */
-n->reg_size = pow2ceil(NVME_REG_SIZE + 2 * n->num_queues * NVME_DB_SIZE);
+n->reg_size = pow2ceil(NVME_REG_SIZE +
+   2 * n->params.num_queues * NVME_DB_SIZE);
 n->ns_size = bs_size / (uint64_t)n->num_namespaces;
 
 n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
-n->sq = g_new0(NvmeSQueue *, n->num_queues);
-n->cq = g_new0(NvmeCQueue *, n->num_queues);
+n->sq = g_new0(NvmeSQueue *, n->params.num_queues);
+n->cq = g_new0(NvmeCQueue *, n->params.num_queues);
 
 memory_region_init_io(>iomem, OBJECT(n), _mmio_ops, n,
   "nvme", n->reg_size);
 pci_register_bar(pci_dev, 0,
 

[PATCH v6 03/20] hw/block/nvme: remove superfluous breaks

[Klaus Jensen]

From: Klaus Jensen 

These break statements was left over when commit 3036a626e9ef ("nvme:
add Get/Set Feature Timestamp support") was merged.

Signed-off-by: Klaus Jensen 
Reviewed-by: Maxim Levitsky 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Keith Busch 
---
 hw/block/nvme.c | 4 
 1 file changed, 4 deletions(-)

diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 6702812ecdd1..f67499d85f3a 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -802,7 +802,6 @@ static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, 
NvmeRequest *req)
 break;
 case NVME_TIMESTAMP:
 return nvme_get_feature_timestamp(n, cmd);
-break;
 default:
 trace_pci_nvme_err_invalid_getfeat(dw10);
 return NVME_INVALID_FIELD | NVME_DNR;
@@ -846,11 +845,8 @@ static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd 
*cmd, NvmeRequest *req)
 req->cqe.result =
 cpu_to_le32((n->num_queues - 2) | ((n->num_queues - 2) << 16));
 break;
-
 case NVME_TIMESTAMP:
 return nvme_set_feature_timestamp(n, cmd);
-break;
-
 default:
 trace_pci_nvme_err_invalid_setfeat(dw10);
 return NVME_INVALID_FIELD | NVME_DNR;
-- 
2.26.2

[PATCH] python: remove more instances of sys.version_info

[John Snow]

We guarantee 3.5+ everywhere; remove more dead checks. In general, try
to avoid using version checks and instead prefer to attempt behavior
when possible.

Signed-off-by: John Snow 
---
 scripts/analyze-migration.py |  5 -
 scripts/decodetree.py| 25 +---
 scripts/qmp/qmp-shell|  3 ---
 tests/docker/docker.py   |  5 +++--
 tests/qemu-iotests/nbd-fault-injector.py |  5 +
 5 files changed, 13 insertions(+), 30 deletions(-)

diff --git a/scripts/analyze-migration.py b/scripts/analyze-migration.py
index 96a31d3974..95838cbff3 100755
--- a/scripts/analyze-migration.py
+++ b/scripts/analyze-migration.py
@@ -25,11 +25,6 @@
 import sys
 
 
-MIN_PYTHON = (3, 2)
-if sys.version_info < MIN_PYTHON:
-sys.exit("Python %s.%s or later is required.\n" % MIN_PYTHON)
-
-
 def mkdir_p(path):
 try:
 os.makedirs(path)
diff --git a/scripts/decodetree.py b/scripts/decodetree.py
index 46ab917807..f9d204aa36 100755
--- a/scripts/decodetree.py
+++ b/scripts/decodetree.py
@@ -75,13 +75,6 @@ def output(*args):
 output_fd.write(a)
 
 
-if sys.version_info >= (3, 4):
-re_fullmatch = re.fullmatch
-else:
-def re_fullmatch(pat, str):
-return re.match('^' + pat + '$', str)
-
-
 def output_autogen():
 output('/* This file is autogenerated by scripts/decodetree.py.  */\n\n')
 
@@ -428,18 +421,18 @@ def parse_field(lineno, name, toks):
 width = 0
 func = None
 for t in toks:
-if re_fullmatch('!function=' + re_ident, t):
+if re.fullmatch('!function=' + re_ident, t):
 if func:
 error(lineno, 'duplicate function')
 func = t.split('=')
 func = func[1]
 continue
 
-if re_fullmatch('[0-9]+:s[0-9]+', t):
+if re.fullmatch('[0-9]+:s[0-9]+', t):
 # Signed field extract
 subtoks = t.split(':s')
 sign = True
-elif re_fullmatch('[0-9]+:[0-9]+', t):
+elif re.fullmatch('[0-9]+:[0-9]+', t):
 # Unsigned field extract
 subtoks = t.split(':')
 sign = False
@@ -488,11 +481,11 @@ def parse_arguments(lineno, name, toks):
 flds = []
 extern = False
 for t in toks:
-if re_fullmatch('!extern', t):
+if re.fullmatch('!extern', t):
 extern = True
 anyextern = True
 continue
-if not re_fullmatch(re_ident, t):
+if not re.fullmatch(re_ident, t):
 error(lineno, 'invalid argument set token "{0}"'.format(t))
 if t in flds:
 error(lineno, 'duplicate argument "{0}"'.format(t))
@@ -621,13 +614,13 @@ def parse_generic(lineno, is_format, name, toks):
 continue
 
 # 'Foo=%Bar' imports a field with a different name.
-if re_fullmatch(re_ident + '=%' + re_ident, t):
+if re.fullmatch(re_ident + '=%' + re_ident, t):
 (fname, iname) = t.split('=%')
 flds = add_field_byname(lineno, flds, fname, iname)
 continue
 
 # 'Foo=number' sets an argument field to a constant value
-if re_fullmatch(re_ident + '=[+-]?[0-9]+', t):
+if re.fullmatch(re_ident + '=[+-]?[0-9]+', t):
 (fname, value) = t.split('=')
 value = int(value)
 flds = add_field(lineno, flds, fname, ConstField(value))
@@ -635,7 +628,7 @@ def parse_generic(lineno, is_format, name, toks):
 
 # Pattern of 0s, 1s, dots and dashes indicate required zeros,
 # required ones, or dont-cares.
-if re_fullmatch('[01.-]+', t):
+if re.fullmatch('[01.-]+', t):
 shift = len(t)
 fms = t.replace('0', '1')
 fms = fms.replace('.', '0')
@@ -652,7 +645,7 @@ def parse_generic(lineno, is_format, name, toks):
 fixedmask = (fixedmask << shift) | fms
 undefmask = (undefmask << shift) | ubm
 # Otherwise, fieldname:fieldwidth
-elif re_fullmatch(re_ident + ':s?[0-9]+', t):
+elif re.fullmatch(re_ident + ':s?[0-9]+', t):
 (fname, flen) = t.split(':')
 sign = False
 if flen[0] == 's':
diff --git a/scripts/qmp/qmp-shell b/scripts/qmp/qmp-shell
index a01d31de1e..c5eef06f3f 100755
--- a/scripts/qmp/qmp-shell
+++ b/scripts/qmp/qmp-shell
@@ -77,9 +77,6 @@ import re
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
 from qemu import qmp
 
-if sys.version_info[0] == 2:
-input = raw_input
-
 class QMPCompleter(list):
 def complete(self, text, state):
 for cmd in self:
diff --git a/tests/docker/docker.py b/tests/docker/docker.py
index d8268c..5a9735db78 100755
--- a/tests/docker/docker.py
+++ b/tests/docker/docker.py
@@ -258,12 +258,13 @@ def _kill_instances(self, *args, **kwargs):
 return self._do_kill_instances(True)
 
 def _output(self, cmd, **kwargs):
-if sys.version_info[1] >= 6:
+ 

[PATCH RFC v2 4/5] iotests: move bitmap helpers into their own file

[John Snow]

Signed-off-by: John Snow 
---
 tests/qemu-iotests/257| 110 +---
 tests/qemu-iotests/bitmaps.py | 131 ++
 2 files changed, 132 insertions(+), 109 deletions(-)
 create mode 100644 tests/qemu-iotests/bitmaps.py

diff --git a/tests/qemu-iotests/257 b/tests/qemu-iotests/257
index 2933e2670b..adf4cdfe6e 100755
--- a/tests/qemu-iotests/257
+++ b/tests/qemu-iotests/257
@@ -24,120 +24,12 @@ import os
 
 import iotests
 from iotests import log, qemu_img
+from bitmaps import EmulatedBitmap, GROUPS
 
 SIZE = 64 * 1024 * 1024
 GRANULARITY = 64 * 1024
 
 
-class Pattern:
-def __init__(self, byte, offset, size=GRANULARITY):
-self.byte = byte
-self.offset = offset
-self.size = size
-
-def bits(self, granularity):
-lower = self.offset // granularity
-upper = (self.offset + self.size - 1) // granularity
-return set(range(lower, upper + 1))
-
-
-class PatternGroup:
-"""Grouping of Pattern objects. Initialize with an iterable of Patterns."""
-def __init__(self, patterns):
-self.patterns = patterns
-
-def bits(self, granularity):
-"""Calculate the unique bits dirtied by this pattern grouping"""
-res = set()
-for pattern in self.patterns:
-res |= pattern.bits(granularity)
-return res
-
-
-GROUPS = [
-PatternGroup([
-# Batch 0: 4 clusters
-Pattern('0x49', 0x000),
-Pattern('0x6c', 0x010),   # 1M
-Pattern('0x6f', 0x200),   # 32M
-Pattern('0x76', 0x3ff)]), # 64M - 64K
-PatternGroup([
-# Batch 1: 6 clusters (3 new)
-Pattern('0x65', 0x000),   # Full overwrite
-Pattern('0x77', 0x00f8000),   # Partial-left (1M-32K)
-Pattern('0x72', 0x2008000),   # Partial-right (32M+32K)
-Pattern('0x69', 0x3fe)]), # Adjacent-left (64M - 128K)
-PatternGroup([
-# Batch 2: 7 clusters (3 new)
-Pattern('0x74', 0x001),   # Adjacent-right
-Pattern('0x69', 0x00e8000),   # Partial-left  (1M-96K)
-Pattern('0x6e', 0x2018000),   # Partial-right (32M+96K)
-Pattern('0x67', 0x3fe,
-2*GRANULARITY)]), # Overwrite [(64M-128K)-64M)
-PatternGroup([
-# Batch 3: 8 clusters (5 new)
-# Carefully chosen such that nothing re-dirties the one cluster
-# that copies out successfully before failure in Group #1.
-Pattern('0xaa', 0x001,
-3*GRANULARITY),   # Overwrite and 2x Adjacent-right
-Pattern('0xbb', 0x00d8000),   # Partial-left (1M-160K)
-Pattern('0xcc', 0x2028000),   # Partial-right (32M+160K)
-Pattern('0xdd', 0x3fc)]), # New; leaving a gap to the right
-]
-
-
-class EmulatedBitmap:
-def __init__(self, granularity=GRANULARITY):
-self._bits = set()
-self.granularity = granularity
-
-def dirty_bits(self, bits):
-self._bits |= set(bits)
-
-def dirty_group(self, n):
-self.dirty_bits(GROUPS[n].bits(self.granularity))
-
-def clear(self):
-self._bits = set()
-
-def clear_bits(self, bits):
-self._bits -= set(bits)
-
-def clear_bit(self, bit):
-self.clear_bits({bit})
-
-def clear_group(self, n):
-self.clear_bits(GROUPS[n].bits(self.granularity))
-
-@property
-def first_bit(self):
-return sorted(self.bits)[0]
-
-@property
-def bits(self):
-return self._bits
-
-@property
-def count(self):
-return len(self.bits)
-
-def compare(self, qmp_bitmap):
-"""
-Print a nice human-readable message checking that a bitmap as reported
-by the QMP interface has as many bits set as we expect it to.
-"""
-
-name = qmp_bitmap.get('name', '(anonymous)')
-log("= Checking Bitmap {:s} =".format(name))
-
-want = self.count
-have = qmp_bitmap['count'] // qmp_bitmap['granularity']
-
-log("expecting {:d} dirty sectors; have {:d}. {:s}".format(
-want, have, "OK!" if want == have else "ERROR!"))
-log('')
-
-
 class Drive:
 """Represents, vaguely, a drive attached to a VM.
 Includes format, graph, and device information."""
diff --git a/tests/qemu-iotests/bitmaps.py b/tests/qemu-iotests/bitmaps.py
new file mode 100644
index 00..522fc25171
--- /dev/null
+++ b/tests/qemu-iotests/bitmaps.py
@@ -0,0 +1,131 @@
+# Bitmap-related helper utilities
+#
+# Copyright (c) 2020 John Snow for Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR 

[PATCH RFC v2 3/5] qmp: expose block-dirty-bitmap-populate

[John Snow]

This is a new job-creating command.

Signed-off-by: John Snow 
---
 qapi/block-core.json  | 18 +++
 qapi/transaction.json |  2 ++
 blockdev.c| 74 +++
 3 files changed, 94 insertions(+)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 0fb527a9a1..f7cae77fc9 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -2250,6 +2250,24 @@
 '*auto-finalize': 'bool',
 '*auto-dismiss': 'bool' } }
 
+##
+# @block-dirty-bitmap-populate:
+#
+# Creates a new job that writes a pattern into a dirty bitmap.
+#
+# Since: 5.0
+#
+# Example:
+#
+# -> { "execute": "block-dirty-bitmap-populate",
+#  "arguments": { "node": "drive0", "target": "bitmap0",
+# "job-id": "job0", "pattern": "allocate-top" } }
+# <- { "return": {} }
+#
+##
+{ 'command': 'block-dirty-bitmap-populate', 'boxed': true,
+  'data': 'BlockDirtyBitmapPopulate' }
+
 ##
 # @BlockDirtyBitmapSha256:
 #
diff --git a/qapi/transaction.json b/qapi/transaction.json
index b6c11158f0..44be517de5 100644
--- a/qapi/transaction.json
+++ b/qapi/transaction.json
@@ -50,6 +50,7 @@
 # - @block-dirty-bitmap-enable: since 4.0
 # - @block-dirty-bitmap-disable: since 4.0
 # - @block-dirty-bitmap-merge: since 4.0
+# - @block-dirty-bitmap-populate: since 5.0
 # - @blockdev-backup: since 2.3
 # - @blockdev-snapshot: since 2.5
 # - @blockdev-snapshot-internal-sync: since 1.7
@@ -67,6 +68,7 @@
'block-dirty-bitmap-enable': 'BlockDirtyBitmap',
'block-dirty-bitmap-disable': 'BlockDirtyBitmap',
'block-dirty-bitmap-merge': 'BlockDirtyBitmapMerge',
+   'block-dirty-bitmap-populate': 'BlockDirtyBitmapPopulate',
'blockdev-backup': 'BlockdevBackup',
'blockdev-snapshot': 'BlockdevSnapshot',
'blockdev-snapshot-internal-sync': 'BlockdevSnapshotInternal',
diff --git a/blockdev.c b/blockdev.c
index d3e8a6ca22..08844e9e33 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -2233,6 +2233,63 @@ static void 
block_dirty_bitmap_remove_commit(BlkActionState *common)
 bdrv_release_dirty_bitmap(state->bitmap);
 }
 
+static void block_dirty_bitmap_populate_prepare(BlkActionState *common,
+Error **errp)
+{
+BlockJobActionState *state = DO_UPCAST(BlockJobActionState, common, 
common);
+BlockDirtyBitmapPopulate *bitpop;
+BlockDriverState *bs;
+AioContext *aio_context;
+BdrvDirtyBitmap *bmap = NULL;
+int job_flags = JOB_DEFAULT;
+
+assert(common->action->type == \
+   TRANSACTION_ACTION_KIND_BLOCK_DIRTY_BITMAP_POPULATE);
+bitpop = common->action->u.block_dirty_bitmap_populate.data;
+
+bmap = block_dirty_bitmap_lookup(bitpop->node, bitpop->name, , errp);
+if (!bmap) {
+return;
+}
+
+aio_context = bdrv_get_aio_context(bs);
+aio_context_acquire(aio_context);
+state->bs = bs;
+
+/* Paired with .clean() */
+bdrv_drained_begin(state->bs);
+
+if (!bitpop->has_on_error) {
+bitpop->on_error = BLOCKDEV_ON_ERROR_REPORT;
+}
+if (!bitpop->has_auto_finalize) {
+bitpop->auto_finalize = true;
+}
+if (!bitpop->has_auto_dismiss) {
+bitpop->auto_dismiss = true;
+}
+
+if (!bitpop->auto_finalize) {
+job_flags |= JOB_MANUAL_FINALIZE;
+}
+if (!bitpop->auto_dismiss) {
+job_flags |= JOB_MANUAL_DISMISS;
+}
+
+state->job = bitpop_job_create(
+bitpop->job_id,
+bs,
+bmap,
+bitpop->pattern,
+bitpop->on_error,
+job_flags,
+NULL, NULL,
+common->block_job_txn,
+errp);
+
+aio_context_release(aio_context);
+}
+
 static void abort_prepare(BlkActionState *common, Error **errp)
 {
 error_setg(errp, "Transaction aborted using Abort action");
@@ -2316,6 +2373,13 @@ static const BlkActionOps actions[] = {
 .commit = block_dirty_bitmap_remove_commit,
 .abort = block_dirty_bitmap_remove_abort,
 },
+[TRANSACTION_ACTION_KIND_BLOCK_DIRTY_BITMAP_POPULATE] = {
+.instance_size = sizeof(BlockJobActionState),
+.prepare = block_dirty_bitmap_populate_prepare,
+.commit = blockdev_backup_commit,
+.abort = blockdev_backup_abort,
+.clean = blockdev_backup_clean,
+},
 /* Where are transactions for MIRROR, COMMIT and STREAM?
  * Although these blockjobs use transaction callbacks like the backup job,
  * these jobs do not necessarily adhere to transaction semantics.
@@ -2672,6 +2736,16 @@ void qmp_block_dirty_bitmap_merge(const char *node, 
const char *target,
 do_block_dirty_bitmap_merge(node, target, bitmaps, NULL, errp);
 }
 
+void qmp_block_dirty_bitmap_populate(BlockDirtyBitmapPopulate *bitpop,
+ Error **errp)
+{
+TransactionAction action = {
+.type = TRANSACTION_ACTION_KIND_BLOCK_DIRTY_BITMAP_POPULATE,
+.u.block_dirty_bitmap_populate.data = bitpop,
+};
+

[PATCH RFC v2 1/5] block: add bitmap-populate job

[John Snow]

This job copies the allocation map into a bitmap. It's a job because
there's no guarantee that allocation interrogation will be quick (or
won't hang), so it cannot be retrofit into block-dirty-bitmap-merge.

It was designed with different possible population patterns in mind,
but only top layer allocation was implemented for now.

Signed-off-by: John Snow 
---
 qapi/block-core.json  |  48 +
 qapi/job.json |   2 +-
 include/block/block_int.h |  21 
 block/bitmap-alloc.c  | 207 ++
 blockjob.c|   3 +-
 block/Makefile.objs   |   1 +
 6 files changed, 280 insertions(+), 2 deletions(-)
 create mode 100644 block/bitmap-alloc.c

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 943df1926a..0fb527a9a1 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -2202,6 +2202,54 @@
   { 'command': 'block-dirty-bitmap-merge',
 'data': 'BlockDirtyBitmapMerge' }
 
+##
+# @BitmapPattern:
+#
+# An enumeration of possible patterns that can be written into a bitmap.
+#
+# @allocation-top: The allocation status of the top layer
+#  of the attached storage node.
+#
+# Since: 5.0
+##
+{ 'enum': 'BitmapPattern',
+  'data': ['allocation-top'] }
+
+##
+# @BlockDirtyBitmapPopulate:
+#
+# @job-id: identifier for the newly-created block job.
+#
+# @pattern: What pattern should be written into the bitmap?
+#
+# @on-error: the action to take if an error is encountered on a bitmap's
+#attached node, default 'report'.
+#'stop' and 'enospc' can only be used if the block device supports
+#io-status (see BlockInfo).
+#
+# @auto-finalize: When false, this job will wait in a PENDING state after it 
has
+# finished its work, waiting for @block-job-finalize before
+# making any block graph changes.
+# When true, this job will automatically
+# perform its abort or commit actions.
+# Defaults to true.
+#
+# @auto-dismiss: When false, this job will wait in a CONCLUDED state after it
+#has completely ceased all work, and awaits @block-job-dismiss.
+#When true, this job will automatically disappear from the 
query
+#list without user intervention.
+#Defaults to true.
+#
+# Since: 5.0
+##
+{ 'struct': 'BlockDirtyBitmapPopulate',
+  'base': 'BlockDirtyBitmap',
+  'data': { 'job-id': 'str',
+'pattern': 'BitmapPattern',
+'*on-error': 'BlockdevOnError',
+'*auto-finalize': 'bool',
+'*auto-dismiss': 'bool' } }
+
 ##
 # @BlockDirtyBitmapSha256:
 #
diff --git a/qapi/job.json b/qapi/job.json
index 5e658281f5..5f496d4630 100644
--- a/qapi/job.json
+++ b/qapi/job.json
@@ -22,7 +22,7 @@
 # Since: 1.7
 ##
 { 'enum': 'JobType',
-  'data': ['commit', 'stream', 'mirror', 'backup', 'create'] }
+  'data': ['commit', 'stream', 'mirror', 'backup', 'create', 
'bitmap-populate'] }
 
 ##
 # @JobStatus:
diff --git a/include/block/block_int.h b/include/block/block_int.h
index df6d0273d6..eb68a5639a 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -1222,6 +1222,27 @@ BlockJob *backup_job_create(const char *job_id, 
BlockDriverState *bs,
 BlockCompletionFunc *cb, void *opaque,
 JobTxn *txn, Error **errp);
 
+/*
+ * bitpop_job_create: Create a new bitmap population job.
+ *
+ * @job_id: The id of the newly-created job.
+ * @bs: Block device associated with the @target_bitmap.
+ * @target_bitmap: The bitmap to populate.
+ * @on_error: What to do if an error on @bs is encountered.
+ * @creation_flags: Flags that control the behavior of the Job lifetime.
+ *  See @BlockJobCreateFlags
+ * @cb: Completion function for the job.
+ * @opaque: Opaque pointer value passed to @cb.
+ * @txn: Transaction that this job is part of (may be NULL).
+ */
+BlockJob *bitpop_job_create(const char *job_id, BlockDriverState *bs,
+BdrvDirtyBitmap *target_bitmap,
+BitmapPattern pattern,
+BlockdevOnError on_error,
+int creation_flags,
+BlockCompletionFunc *cb, void *opaque,
+JobTxn *txn, Error **errp);
+
 BdrvChild *bdrv_root_attach_child(BlockDriverState *child_bs,
   const char *child_name,
   const BdrvChildRole *child_role,
diff --git a/block/bitmap-alloc.c b/block/bitmap-alloc.c
new file mode 100644
index 00..47d542dc12
--- /dev/null
+++ b/block/bitmap-alloc.c
@@ -0,0 +1,207 @@
+/*
+ * Async Dirty Bitmap Populator
+ *
+ * Copyright (C) 2020 Red Hat, Inc.
+ *
+ * Authors:
+ *  John Snow 
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level 

[PATCH RFC v2 0/5] block: add block-dirty-bitmap-populate job

[John Snow]

Hi,

This is a new (very small) block job that writes a pattern into a
bitmap. The only pattern implemented is the top allocation information.

This can be used to "recover" an incremental bitmap chain if an external
snapshot was taken without creating a new bitmap first: any writes made
to the image will be reflected by the allocation status and can be
written back into a bitmap.

This is useful for e.g. libvirt managing backup chains if a user creates
an external snapshot outside of libvirt.

v2:
 - Addressed some, but not all feedback
 - Rebased on latest 'job-runner' series; but it's not clear if it
   should be kept.
 - This version doesn't address all of the feedback from v1,
   but I am posting it to the list as an RFC.

John Snow (5):
  block: add bitmap-populate job
  blockdev: combine DriveBackupState and BlockdevBackupState
  qmp: expose block-dirty-bitmap-populate
  iotests: move bitmap helpers into their own file
  iotests: add 287 for block-dirty-bitmap-populate

 qapi/block-core.json  |   66 +
 qapi/job.json |2 +-
 qapi/transaction.json |2 +
 include/block/block_int.h |   21 +
 block/bitmap-alloc.c  |  207 ++
 blockdev.c|  104 +-
 blockjob.c|3 +-
 block/Makefile.objs   |1 +
 tests/qemu-iotests/257|  110 +-
 tests/qemu-iotests/287|  242 ++
 tests/qemu-iotests/287.out| 4544 +
 tests/qemu-iotests/bitmaps.py |  131 +
 tests/qemu-iotests/group  |1 +
 13 files changed, 5305 insertions(+), 129 deletions(-)
 create mode 100644 block/bitmap-alloc.c
 create mode 100755 tests/qemu-iotests/287
 create mode 100644 tests/qemu-iotests/287.out
 create mode 100644 tests/qemu-iotests/bitmaps.py

-- 
2.21.1

[PATCH RFC v2 2/5] blockdev: combine DriveBackupState and BlockdevBackupState

[John Snow]

They have the same fields -- rename it BlockJobState.

Signed-off-by: John Snow 
---
 blockdev.c | 30 --
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/blockdev.c b/blockdev.c
index b3c840ec03..d3e8a6ca22 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -1702,11 +1702,11 @@ static void external_snapshot_clean(BlkActionState 
*common)
 aio_context_release(aio_context);
 }
 
-typedef struct DriveBackupState {
+typedef struct BlockJobActionState {
 BlkActionState common;
 BlockDriverState *bs;
 BlockJob *job;
-} DriveBackupState;
+} BlockJobActionState;
 
 static BlockJob *do_backup_common(BackupCommon *backup,
   BlockDriverState *bs,
@@ -1716,7 +1716,7 @@ static BlockJob *do_backup_common(BackupCommon *backup,
 
 static void drive_backup_prepare(BlkActionState *common, Error **errp)
 {
-DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
+BlockJobActionState *state = DO_UPCAST(BlockJobActionState, common, 
common);
 DriveBackup *backup;
 BlockDriverState *bs;
 BlockDriverState *target_bs;
@@ -1853,7 +1853,7 @@ out:
 
 static void drive_backup_commit(BlkActionState *common)
 {
-DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
+BlockJobActionState *state = DO_UPCAST(BlockJobActionState, common, 
common);
 AioContext *aio_context;
 
 aio_context = bdrv_get_aio_context(state->bs);
@@ -1867,7 +1867,7 @@ static void drive_backup_commit(BlkActionState *common)
 
 static void drive_backup_abort(BlkActionState *common)
 {
-DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
+BlockJobActionState *state = DO_UPCAST(BlockJobActionState, common, 
common);
 
 if (state->job) {
 AioContext *aio_context;
@@ -1883,7 +1883,7 @@ static void drive_backup_abort(BlkActionState *common)
 
 static void drive_backup_clean(BlkActionState *common)
 {
-DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
+BlockJobActionState *state = DO_UPCAST(BlockJobActionState, common, 
common);
 AioContext *aio_context;
 
 if (!state->bs) {
@@ -1898,15 +1898,9 @@ static void drive_backup_clean(BlkActionState *common)
 aio_context_release(aio_context);
 }
 
-typedef struct BlockdevBackupState {
-BlkActionState common;
-BlockDriverState *bs;
-BlockJob *job;
-} BlockdevBackupState;
-
 static void blockdev_backup_prepare(BlkActionState *common, Error **errp)
 {
-BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, 
common);
+BlockJobActionState *state = DO_UPCAST(BlockJobActionState, common, 
common);
 BlockdevBackup *backup;
 BlockDriverState *bs;
 BlockDriverState *target_bs;
@@ -1954,7 +1948,7 @@ static void blockdev_backup_prepare(BlkActionState 
*common, Error **errp)
 
 static void blockdev_backup_commit(BlkActionState *common)
 {
-BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, 
common);
+BlockJobActionState *state = DO_UPCAST(BlockJobActionState, common, 
common);
 AioContext *aio_context;
 
 aio_context = bdrv_get_aio_context(state->bs);
@@ -1968,7 +1962,7 @@ static void blockdev_backup_commit(BlkActionState *common)
 
 static void blockdev_backup_abort(BlkActionState *common)
 {
-BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, 
common);
+BlockJobActionState *state = DO_UPCAST(BlockJobActionState, common, 
common);
 
 if (state->job) {
 AioContext *aio_context;
@@ -1984,7 +1978,7 @@ static void blockdev_backup_abort(BlkActionState *common)
 
 static void blockdev_backup_clean(BlkActionState *common)
 {
-BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, 
common);
+BlockJobActionState *state = DO_UPCAST(BlockJobActionState, common, 
common);
 AioContext *aio_context;
 
 if (!state->bs) {
@@ -2265,14 +2259,14 @@ static const BlkActionOps actions[] = {
 .clean = external_snapshot_clean,
 },
 [TRANSACTION_ACTION_KIND_DRIVE_BACKUP] = {
-.instance_size = sizeof(DriveBackupState),
+.instance_size = sizeof(BlockJobActionState),
 .prepare = drive_backup_prepare,
 .commit = drive_backup_commit,
 .abort = drive_backup_abort,
 .clean = drive_backup_clean,
 },
 [TRANSACTION_ACTION_KIND_BLOCKDEV_BACKUP] = {
-.instance_size = sizeof(BlockdevBackupState),
+.instance_size = sizeof(BlockJobActionState),
 .prepare = blockdev_backup_prepare,
 .commit = blockdev_backup_commit,
 .abort = blockdev_backup_abort,
-- 
2.21.1

Re: [PATCH RFC WIP 0/6] iotests: delinting work-in-progress

[no-reply]

Patchew URL: https://patchew.org/QEMU/20200513214130.15375-1-js...@redhat.com/



Hi,

This series failed the docker-quick@centos7 build test. Please find the testing 
commands and
their output below. If you have Docker installed, you can probably reproduce it
locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
make docker-image-centos7 V=1 NETWORK=1
time make docker-test-quick@centos7 SHOW_ENV=1 J=14 NETWORK=1
=== TEST SCRIPT END ===

Not run: 259
Failures: 031 036 054 061 114
Failed 5 of 118 iotests
make: *** [check-tests/check-block.sh] Error 1
make: *** Waiting for unfinished jobs
  TESTcheck-qtest-aarch64: tests/qtest/qos-test
Traceback (most recent call last):
---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', 
'--label', 'com.qemu.instance.uuid=a364a3916b304854887e4b7475175383', '-u', 
'1003', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=', 
'-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 
'SHOW_ENV=1', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', 
'/home/patchew2/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', 
'/var/tmp/patchew-tester-tmp-n73ihxu6/src/docker-src.2020-05-13-23.00.06.31673:/var/tmp/qemu:z,ro',
 'qemu:centos7', '/var/tmp/qemu/run', 'test-quick']' returned non-zero exit 
status 2.
filter=--filter=label=com.qemu.instance.uuid=a364a3916b304854887e4b7475175383
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-n73ihxu6/src'
make: *** [docker-run-test-quick@centos7] Error 2

real15m47.213s
user0m5.361s


The full log is available at
http://patchew.org/logs/20200513214130.15375-1-js...@redhat.com/testing.docker-quick@centos7/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-de...@redhat.com

[PATCH v4 2/3] iotests: add JobRunner class

[John Snow]

The idea is that instead of increasing the arguments to job_run all the
time, create a more general-purpose job runner that can be subclassed to
do interesting things with.

pylint note: the 'callbacks' option guards against unused warning
arguments in functions designated as callbacks. It does not currently
guard against "no-self-use" though; hence a once-off ignore.

mypy note: QapiEvent is only a weak alias; it's fully interchangable
with the type it's declared as. In the future, we may wish to tighten
these types. For now, this communicates the rough shape of the type and
(more importantly) the intent.

Signed-off-by: John Snow 
---
 tests/qemu-iotests/055|  17 +--
 tests/qemu-iotests/155|  11 +-
 tests/qemu-iotests/255|   9 +-
 tests/qemu-iotests/257|  54 +
 tests/qemu-iotests/iotests.py | 206 +-
 tests/qemu-iotests/pylintrc   |  11 ++
 6 files changed, 225 insertions(+), 83 deletions(-)

diff --git a/tests/qemu-iotests/055 b/tests/qemu-iotests/055
index 4d3744b0d3..7e13585a94 100755
--- a/tests/qemu-iotests/055
+++ b/tests/qemu-iotests/055
@@ -115,19 +115,22 @@ class TestSingleDrive(iotests.QMPTestCase):
 self.do_test_pause('blockdev-backup', 'drive1', blockdev_target_img)
 
 def do_test_resize_blockdev_backup(self, device, node):
-def pre_finalize():
-result = self.vm.qmp('block_resize', device=device, size=65536)
-self.assert_qmp(result, 'error/class', 'GenericError')
-
-result = self.vm.qmp('block_resize', node_name=node, size=65536)
-self.assert_qmp(result, 'error/class', 'GenericError')
+class JobRunner(iotests.TestJobRunner):
+def on_pending(self, event):
+qmp = self._vm.qmp
+result = qmp('block_resize', device=device, size=65536)
+self.test.assert_qmp(result, 'error/class', 'GenericError')
+result = qmp('block_resize', node_name=node, size=65536)
+self.test.assert_qmp(result, 'error/class', 'GenericError')
+super().on_pending(event)
 
 result = self.vm.qmp('blockdev-backup', job_id='job0', device='drive0',
  target='drive1', sync='full', auto_finalize=False,
  auto_dismiss=False)
 self.assert_qmp(result, 'return', {})
 
-self.vm.run_job('job0', auto_finalize=False, pre_finalize=pre_finalize)
+runner = JobRunner(self.vm, 'job0', test=self, auto_finalize=False)
+runner.run()
 
 def test_source_resize_blockdev_backup(self):
 self.do_test_resize_blockdev_backup('drive0', 'source')
diff --git a/tests/qemu-iotests/155 b/tests/qemu-iotests/155
index cb371d4649..1aa6cfefb8 100755
--- a/tests/qemu-iotests/155
+++ b/tests/qemu-iotests/155
@@ -163,6 +163,12 @@ class BaseClass(iotests.QMPTestCase):
 self.assert_qmp_absent(node, 'image/backing-image')
 
 
+class MirrorJob(iotests.TestJobRunner):
+def on_pending(self, event):
+self.test.openBacking()
+super().on_pending(event)
+
+
 # Class variables for controlling its behavior:
 #
 # cmd: Mirroring command to execute, either drive-mirror or blockdev-mirror
@@ -188,8 +194,9 @@ class MirrorBaseClass(BaseClass):
 
 self.assert_qmp(result, 'return', {})
 
-self.vm.run_job('mirror-job', auto_finalize=False,
-pre_finalize=self.openBacking, auto_dismiss=True)
+job = MirrorJob(self.vm, 'mirror-job', test=self,
+auto_finalize=False, auto_dismiss=True)
+job.run()
 
 def testFull(self):
 self.runMirror('full')
diff --git a/tests/qemu-iotests/255 b/tests/qemu-iotests/255
index 8f08f741da..74487548fa 100755
--- a/tests/qemu-iotests/255
+++ b/tests/qemu-iotests/255
@@ -71,8 +71,13 @@ with iotests.FilePath('t.qcow2') as disk_path, \
 result = vm.qmp_log('block-commit', job_id='job0', auto_finalize=False,
 device='overlay', top_node='mid')
 
-vm.run_job('job0', auto_finalize=False, pre_finalize=start_requests,
-auto_dismiss=True)
+class JobRunner(iotests.JobRunner):
+def on_pending(self, event):
+start_requests()
+super().on_pending(event)
+
+runner = JobRunner(vm, 'job0', auto_finalize=False, auto_dismiss=True)
+runner.run()
 
 vm.shutdown()
 
diff --git a/tests/qemu-iotests/257 b/tests/qemu-iotests/257
index 004a433b8b..2933e2670b 100755
--- a/tests/qemu-iotests/257
+++ b/tests/qemu-iotests/257
@@ -352,30 +352,40 @@ def test_bitmap_sync(bsync_mode, msync_mode='bitmap', 
failure=None):
 job = backup(drive0, 1, bsync1, msync_mode,
  bitmap="bitmap0", bitmap_mode=bsync_mode)
 
-def _callback():
-"""Issue writes while the job is open to test bitmap divergence."""
-# Note: when `failure` is 'intermediate', this isn't called.
-

[PATCH v4 1/3] qmp.py: change event_wait to use a dict

[John Snow]

It's easier to work with than a list of tuples, because we can check the
keys for membership.

Signed-off-by: John Snow 
---
 python/qemu/machine.py| 10 +-
 tests/qemu-iotests/040| 12 ++--
 tests/qemu-iotests/260|  5 +++--
 tests/qemu-iotests/iotests.py | 16 
 4 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/python/qemu/machine.py b/python/qemu/machine.py
index b9a98e2c86..eaedc25172 100644
--- a/python/qemu/machine.py
+++ b/python/qemu/machine.py
@@ -478,21 +478,21 @@ def event_wait(self, name, timeout=60.0, match=None):
 timeout: QEMUMonitorProtocol.pull_event timeout parameter.
 match: Optional match criteria. See event_match for details.
 """
-return self.events_wait([(name, match)], timeout)
+return self.events_wait({name: match}, timeout)
 
 def events_wait(self, events, timeout=60.0):
 """
 events_wait waits for and returns a named event from QMP with a 
timeout.
 
-events: a sequence of (name, match_criteria) tuples.
+events: a mapping containing {name: match_criteria}.
 The match criteria are optional and may be None.
 See event_match for details.
 timeout: QEMUMonitorProtocol.pull_event timeout parameter.
 """
 def _match(event):
-for name, match in events:
-if event['event'] == name and self.event_match(event, match):
-return True
+name = event['event']
+if name in events:
+return self.event_match(event, events[name])
 return False
 
 # Search cached events
diff --git a/tests/qemu-iotests/040 b/tests/qemu-iotests/040
index 32c82b4ec6..90b59081ff 100755
--- a/tests/qemu-iotests/040
+++ b/tests/qemu-iotests/040
@@ -485,12 +485,12 @@ class TestErrorHandling(iotests.QMPTestCase):
 
 def run_job(self, expected_events, error_pauses_job=False):
 match_device = {'data': {'device': 'job0'}}
-events = [
-('BLOCK_JOB_COMPLETED', match_device),
-('BLOCK_JOB_CANCELLED', match_device),
-('BLOCK_JOB_ERROR', match_device),
-('BLOCK_JOB_READY', match_device),
-]
+events = {
+'BLOCK_JOB_COMPLETED': match_device,
+'BLOCK_JOB_CANCELLED': match_device,
+'BLOCK_JOB_ERROR': match_device,
+'BLOCK_JOB_READY': match_device,
+}
 
 completed = False
 log = []
diff --git a/tests/qemu-iotests/260 b/tests/qemu-iotests/260
index 804a7addb9..729f031122 100755
--- a/tests/qemu-iotests/260
+++ b/tests/qemu-iotests/260
@@ -67,8 +67,9 @@ def test(persistent, restart):
 
 vm.qmp_log('block-commit', device='drive0', top=top,
filters=[iotests.filter_qmp_testfiles])
-ev = vm.events_wait((('BLOCK_JOB_READY', None),
- ('BLOCK_JOB_COMPLETED', None)))
+ev = vm.events_wait({
+'BLOCK_JOB_READY': None,
+'BLOCK_JOB_COMPLETED': None })
 log(filter_qmp_event(ev))
 if (ev['event'] == 'BLOCK_JOB_COMPLETED'):
 vm.shutdown()
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index 6c0e781af7..aada94f4f9 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -635,14 +635,14 @@ def run_job(self, job, auto_finalize=True, 
auto_dismiss=False,
 """
 match_device = {'data': {'device': job}}
 match_id = {'data': {'id': job}}
-events = [
-('BLOCK_JOB_COMPLETED', match_device),
-('BLOCK_JOB_CANCELLED', match_device),
-('BLOCK_JOB_ERROR', match_device),
-('BLOCK_JOB_READY', match_device),
-('BLOCK_JOB_PENDING', match_id),
-('JOB_STATUS_CHANGE', match_id)
-]
+events = {
+'BLOCK_JOB_COMPLETED': match_device,
+'BLOCK_JOB_CANCELLED': match_device,
+'BLOCK_JOB_ERROR': match_device,
+'BLOCK_JOB_READY': match_device,
+'BLOCK_JOB_PENDING': match_id,
+'JOB_STATUS_CHANGE': match_id,
+}
 error = None
 while True:
 ev = filter_qmp_event(self.events_wait(events, timeout=wait))
-- 
2.21.1

[PATCH v4 3/3] iotests: modify test 040 to use JobRunner

[John Snow]

Instead of having somewhat reproduced it for itself.

Signed-off-by: John Snow 
---
 tests/qemu-iotests/040 | 51 +-
 1 file changed, 25 insertions(+), 26 deletions(-)

diff --git a/tests/qemu-iotests/040 b/tests/qemu-iotests/040
index 90b59081ff..e2ef3bb812 100755
--- a/tests/qemu-iotests/040
+++ b/tests/qemu-iotests/040
@@ -483,34 +483,33 @@ class TestErrorHandling(iotests.QMPTestCase):
   file=('top-dbg' if top_debug else 'top-file'),
   backing='mid-fmt')
 
+
+class TestJobRunner(iotests.JobRunner):
+expected_events = ('BLOCK_JOB_COMPLETED',
+   'BLOCK_JOB_ERROR',
+   'BLOCK_JOB_READY')
+
+def __init__(self, *args, test, **kwargs):
+super().__init__(*args, **kwargs)
+self.log = []
+self.test = test
+
+def on_pause(self, event):
+super().on_pause(event)
+result = self._vm.qmp('block-job-resume', device=self._id)
+self.test.assert_qmp(result, 'return', {})
+
+def on_block_job_event(self, event):
+if event['event'] not in self.expected_events:
+self.test.fail("Unexpected event: %s" % event)
+super().on_block_job_event(event)
+self.log.append(event)
+
 def run_job(self, expected_events, error_pauses_job=False):
-match_device = {'data': {'device': 'job0'}}
-events = {
-'BLOCK_JOB_COMPLETED': match_device,
-'BLOCK_JOB_CANCELLED': match_device,
-'BLOCK_JOB_ERROR': match_device,
-'BLOCK_JOB_READY': match_device,
-}
-
-completed = False
-log = []
-while not completed:
-ev = self.vm.events_wait(events, timeout=5.0)
-if ev['event'] == 'BLOCK_JOB_COMPLETED':
-completed = True
-elif ev['event'] == 'BLOCK_JOB_ERROR':
-if error_pauses_job:
-result = self.vm.qmp('block-job-resume', device='job0')
-self.assert_qmp(result, 'return', {})
-elif ev['event'] == 'BLOCK_JOB_READY':
-result = self.vm.qmp('block-job-complete', device='job0')
-self.assert_qmp(result, 'return', {})
-else:
-self.fail("Unexpected event: %s" % ev)
-log.append(iotests.filter_qmp_event(ev))
-
+job = self.TestJobRunner(self.vm, 'job0', test=self)
+job.run()
 self.maxDiff = None
-self.assertEqual(expected_events, log)
+self.assertEqual(expected_events, job.log)
 
 def event_error(self, op, action):
 return {
-- 
2.21.1

[PATCH v4 0/3] iotests: add JobRunner framework

[John Snow]

The basic idea is to make a generic job runtime manager and allow
callers to subclass the manager. Then, instead of adding callback
arguments to the function all the time, we have à la carte customization
of the loop.

To showcase this a little bit, I removed the pre_finalization argument
and made existing callers use a custom JobRunner; and then converted
test 040 to use this style of job runner.

Is it a simplification? No. Is it cool? Maybe. Did it remove the
duplicated job-running code in 040? yes.

As of V4, I'm not sure I really want to pursue this at the moment; it
does feel a bit harder to use than the old interface; though it's more
"complete". I might look into refining the idea and trying to include it
directly in python/qemu; and providing some more 'bespoke' wrappers in
iotests that make using it in the test framework a little less
cumbersome.

Regardless, I am posting it to the list for archiving for now.

v4:
 - Rebased on current master.
 - Converted new usages is test 055.

V3:
 - Rebased on logging series v8
 - Converted 155's new usage of job_run

V2:
 - Rebased on logging series; logging conditionals are pretty now.
 - Inlined callback login in 257
 - No longer based on bitmap-populate job (no test 287)
 - Moved super() call to the beginning of test 040's callback
 - Added docstrings and type annotations

John Snow (3):
  qmp.py: change event_wait to use a dict
  iotests: add JobRunner class
  iotests: modify test 040 to use JobRunner

 python/qemu/machine.py|  10 +-
 tests/qemu-iotests/040|  51 +
 tests/qemu-iotests/055|  17 +--
 tests/qemu-iotests/155|  11 +-
 tests/qemu-iotests/255|   9 +-
 tests/qemu-iotests/257|  54 +
 tests/qemu-iotests/260|   5 +-
 tests/qemu-iotests/iotests.py | 206 +-
 tests/qemu-iotests/pylintrc   |  11 ++
 9 files changed, 258 insertions(+), 116 deletions(-)

-- 
2.21.1

Re: [PATCH v1 0/8] plugins/next (cleanup, cpu_index and lockstep)

[no-reply]

Patchew URL: 
https://patchew.org/QEMU/20200513173200.11830-1-alex.ben...@linaro.org/



Hi,

This series seems to have some coding style problems. See output below for
more information:

Message-id: 20200513173200.11830-1-alex.ben...@linaro.org
Subject: [PATCH  v1 0/8] plugins/next (cleanup, cpu_index and lockstep)
Type: series

=== TEST SCRIPT BEGIN ===
#!/bin/bash
git rev-parse base > /dev/null || exit 0
git config --local diff.renamelimit 0
git config --local diff.renames True
git config --local diff.algorithm histogram
./scripts/checkpatch.pl --mailback base..
=== TEST SCRIPT END ===

Switched to a new branch 'test'
6edd795 plugins: new lockstep plugin for debugging TCG changes
4202a73 tests/tcg: add new threadcount test
1f6f667 linux-user: properly "unrealize" vCPU object
6200dd7 cpus-common: ensure auto-assigned cpu_indexes don't clash
888408f MAINTAINERS: update the orphaned cpus-common.c file
6c78116 qemu/qemu-plugin: Make qemu_plugin_hwaddr_is_io() hwaddr argument const
ce430e0 qemu/plugin: Move !CONFIG_PLUGIN stubs altogether
e69a9cc qemu/plugin: Trivial code movement

=== OUTPUT BEGIN ===
1/8 Checking commit e69a9cc66e44 (qemu/plugin: Trivial code movement)
2/8 Checking commit ce430e01e1b3 (qemu/plugin: Move !CONFIG_PLUGIN stubs 
altogether)
3/8 Checking commit 6c781160c4cd (qemu/qemu-plugin: Make 
qemu_plugin_hwaddr_is_io() hwaddr argument const)
4/8 Checking commit 888408f0e702 (MAINTAINERS: update the orphaned 
cpus-common.c file)
5/8 Checking commit 6200dd769e32 (cpus-common: ensure auto-assigned cpu_indexes 
don't clash)
6/8 Checking commit 1f6f66791a8f (linux-user: properly "unrealize" vCPU object)
7/8 Checking commit 4202a73ecb0e (tests/tcg: add new threadcount test)
WARNING: added, moved or deleted file(s), does MAINTAINERS need updating?
#30: 
new file mode 100644

ERROR: open brace '{' following function declarations go on the next line
#62: FILE: tests/tcg/multiarch/threadcount.c:28:
+static void *thread_fn(void* varg)  {

ERROR: "foo* bar" should be "foo *bar"
#63: FILE: tests/tcg/multiarch/threadcount.c:29:
+ThreadArg* arg = varg;

ERROR: open brace '{' following function declarations go on the next line
#69: FILE: tests/tcg/multiarch/threadcount.c:35:
+int main(int argc, char **argv) {

ERROR: "foo* bar" should be "foo *bar"
#79: FILE: tests/tcg/multiarch/threadcount.c:45:
+ThreadArg* arg = calloc(sizeof(ThreadArg), 1);

total: 4 errors, 1 warnings, 70 lines checked

Patch 7/8 has style problems, please review.  If any of these errors
are false positives report them to the maintainer, see
CHECKPATCH in MAINTAINERS.

8/8 Checking commit 6edd795dd0a4 (plugins: new lockstep plugin for debugging 
TCG changes)
WARNING: added, moved or deleted file(s), does MAINTAINERS need updating?
#56: 
new file mode 100644

ERROR: "foo * bar" should be "foo *bar"
#163: FILE: tests/plugin/lockstep.c:103:
+static inline const char * ord_ind(int n) {

ERROR: open brace '{' following function declarations go on the next line
#163: FILE: tests/plugin/lockstep.c:103:
+static inline const char * ord_ind(int n) {

WARNING: line over 80 characters
#220: FILE: tests/plugin/lockstep.c:160:
+g_string_append_printf(out, "  previously @ %#016lx/%ld (%ld 
insns)\n",

ERROR: do not use C99 // comments
#292: FILE: tests/plugin/lockstep.c:232:
+// save a reference so we can free later

total: 3 errors, 2 warnings, 360 lines checked

Patch 8/8 has style problems, please review.  If any of these errors
are false positives report them to the maintainer, see
CHECKPATCH in MAINTAINERS.

=== OUTPUT END ===

Test command exited with code: 1


The full log is available at
http://patchew.org/logs/20200513173200.11830-1-alex.ben...@linaro.org/testing.checkpatch/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-de...@redhat.com

Re: [PATCH v1 0/8] plugins/next (cleanup, cpu_index and lockstep)

[no-reply]

Patchew URL: 
https://patchew.org/QEMU/20200513173200.11830-1-alex.ben...@linaro.org/



Hi,

This series failed the asan build test. Please find the testing commands and
their output below. If you have Docker installed, you can probably reproduce it
locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
export ARCH=x86_64
make docker-image-fedora V=1 NETWORK=1
time make docker-test-debug@fedora TARGET_LIST=x86_64-softmmu J=14 NETWORK=1
=== TEST SCRIPT END ===

PASS 25 test-qobject-output-visitor /visitor/output/list_union/string
PASS 26 test-qobject-output-visitor /visitor/output/list_union/number
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  
tests/test-clone-visitor -m=quick -k --tap < /dev/null | 
./scripts/tap-driver.pl --test-name="test-clone-visitor" 
==6834==WARNING: ASan doesn't fully support makecontext/swapcontext functions 
and may produce false positives in some cases!
PASS 4 fdc-test /x86_64/fdc/media_change
PASS 5 fdc-test /x86_64/fdc/sense_interrupt
PASS 6 fdc-test /x86_64/fdc/relative_seek
---
PASS 32 test-opts-visitor /visitor/opts/range/beyond
PASS 33 test-opts-visitor /visitor/opts/dict/unvisited
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  
tests/test-coroutine -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl 
--test-name="test-coroutine" 
==6876==WARNING: ASan doesn't fully support makecontext/swapcontext functions 
and may produce false positives in some cases!
PASS 1 test-coroutine /basic/no-dangling-access
PASS 2 test-coroutine /basic/lifecycle
PASS 3 test-coroutine /basic/yield
==6876==WARNING: ASan is ignoring requested __asan_handle_no_return: stack top: 
0x7ffd64376000; bottom 0x7f9487ba2000; size: 0x0068dc7d4000 (450375794688)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
PASS 4 test-coroutine /basic/nesting
---
PASS 11 test-aio /aio/event/wait
PASS 12 test-aio /aio/event/flush
PASS 13 test-aio /aio/event/wait/no-flush-cb
==6891==WARNING: ASan doesn't fully support makecontext/swapcontext functions 
and may produce false positives in some cases!
PASS 14 test-aio /aio/timer/schedule
PASS 15 test-aio /aio/coroutine/queue-chaining
PASS 16 test-aio /aio-gsource/flush
---
PASS 12 fdc-test /x86_64/fdc/read_no_dma_19
PASS 13 fdc-test /x86_64/fdc/fuzz-registers
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  
QTEST_QEMU_BINARY=x86_64-softmmu/qemu-system-x86_64 QTEST_QEMU_IMG=qemu-img 
tests/qtest/ide-test -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl 
--test-name="ide-test" 
==6899==WARNING: ASan doesn't fully support makecontext/swapcontext functions 
and may produce false positives in some cases!
PASS 1 ide-test /x86_64/ide/identify
PASS 28 test-aio /aio-gsource/timer/schedule
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  
tests/test-aio-multithread -m=quick -k --tap < /dev/null | 
./scripts/tap-driver.pl --test-name="test-aio-multithread" 
==6908==WARNING: ASan doesn't fully support makecontext/swapcontext functions 
and may produce false positives in some cases!
PASS 1 test-aio-multithread /aio/multi/lifecycle
==6905==WARNING: ASan doesn't fully support makecontext/swapcontext functions 
and may produce false positives in some cases!
PASS 2 ide-test /x86_64/ide/flush
==6925==WARNING: ASan doesn't fully support makecontext/swapcontext functions 
and may produce false positives in some cases!
PASS 2 test-aio-multithread /aio/multi/schedule
PASS 3 ide-test /x86_64/ide/bmdma/simple_rw
==6936==WARNING: ASan doesn't fully support makecontext/swapcontext functions 
and may produce false positives in some cases!
PASS 3 test-aio-multithread /aio/multi/mutex/contended
PASS 4 ide-test /x86_64/ide/bmdma/trim
==6947==WARNING: ASan doesn't fully support makecontext/swapcontext functions 
and may produce false positives in some cases!
PASS 4 test-aio-multithread /aio/multi/mutex/handoff
PASS 5 test-aio-multithread /aio/multi/mutex/mcs
PASS 6 test-aio-multithread /aio/multi/mutex/pthread
---
PASS 6 test-throttle /throttle/detach_attach
PASS 7 test-throttle /throttle/config_functions
PASS 8 test-throttle /throttle/accounting
==6964==WARNING: ASan doesn't fully support makecontext/swapcontext functions 
and may produce false positives in some cases!
PASS 9 test-throttle /throttle/groups
PASS 10 test-throttle /throttle/config/enabled
PASS 11 test-throttle /throttle/config/conflicting
---
PASS 14 test-throttle /throttle/config/max
PASS 15 test-throttle /throttle/config/iops_size
MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(( ${RANDOM:-0} % 255 + 1))}  
tests/test-thread-pool -m=quick -k --tap < /dev/null | ./scripts/tap-driver.pl 
--test-name="test-thread-pool" 
==6968==WARNING: ASan doesn't fully support makecontext/swapcontext functions 
and may produce false positives in some cases!
PASS 1 test-thread-pool /thread-pool/submit
PASS 2 test-thread-pool /thread-pool/submit-aio
PASS 3 test-thread-pool /thread-pool/submit-co
PASS 4 test-thread-pool 

Re:Re: Is qemu could be a "FSM" state machine if running on a "quiet and clean" host pc without random event input?

[tugouxp]

Thank you, much appreciate your thought and meaningful for me!

















At 2020-05-14 07:22:12, "John Snow"  wrote:
>
>
>On 5/11/20 10:07 AM, tugouxp wrote:
>> Hi folks:
>>   i want to know about whether therr are limitations during qemu
>> emulation systems, for exampe, did the regular bugs corener case cant be
>> duplicated on qeme but exist on real boads?
>> 
>
>It's possible, yes. QEMU emulates instead of simulates. We do not try to
>reproduce accurate cycle timings. There may be bugs that exist in real
>hardware but don't reproduce with QEMU, or, of course, the other way around.
>
>> why thing this is that , i have ever use hdl simulator (modsim and
>> iverilog) and openrisc processor to emulate the linux and ucos running,
>> and see the waveform of the simulateion process of the operations systems.
>> i found an interesting things, if i take just the tick interrupt as the
>> only testbech event source,the  kernel simulation waveform is identical
>> duplicated again and again, which means i can predicate future behavior.
>> 
>
>Yeah, we are not operating on the verilog level for any of the hardware
>we emulate.
>
>> i think this something like qemu work principle and so want to know,
>> whether the qemu has this limitation? is the simulation proces a "FSM" 
>> that with definition output if the input event are all regular and
>> without random?
>> 
>
>Well, if you only want determinism and not accuracy, it might be
>possible, but I have to admit to you that I've never tried to instrument
>or measure this. I assume there are many places where cycle
>indeterminancy comes in from the host system.
>
>And I trust x86 about as far as I can throw it.
>
>I assume there are many barriers to this, but maybe folks who worked on
>the replay debugger could give some more authoritative idea.
>
>
>> thank you
>> 

Re: [PATCH v1 0/8] plugins/next (cleanup, cpu_index and lockstep)

[no-reply]

Patchew URL: 
https://patchew.org/QEMU/20200513173200.11830-1-alex.ben...@linaro.org/



Hi,

This series failed the docker-quick@centos7 build test. Please find the testing 
commands and
their output below. If you have Docker installed, you can probably reproduce it
locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
make docker-image-centos7 V=1 NETWORK=1
time make docker-test-quick@centos7 SHOW_ENV=1 J=14 NETWORK=1
=== TEST SCRIPT END ===

  TESTcheck-qtest-aarch64: tests/qtest/qom-test
Broken pipe
/tmp/qemu-test/src/tests/qtest/libqtest.c:175: kill_qemu() detected QEMU death 
from signal 11 (Segmentation fault) (core dumped)
ERROR - too few tests run (expected 65, got 0)
make: *** [check-qtest-aarch64] Error 1
make: *** Waiting for unfinished jobs
  TESTiotest-qcow2: 191
  TESTiotest-qcow2: 192
---
qemu-system-x86_64: /tmp/qemu-test/src/hw/i386/x86.c:175: 
x86_cpu_index_to_props: Assertion `cpu_index < possible_cpus->len' failed.
Broken pipe
/tmp/qemu-test/src/tests/qtest/libqtest.c:175: kill_qemu() detected QEMU death 
from signal 6 (Aborted) (core dumped)
ERROR - too few tests run (expected 8, got 6)
make: *** [check-qtest-x86_64] Error 1
  TESTiotest-qcow2: 220
  TESTiotest-qcow2: 226
  TESTiotest-qcow2: 229
---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', 
'--label', 'com.qemu.instance.uuid=08679829d9d5406eac35a1969274235b', '-u', 
'1003', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=', 
'-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 
'SHOW_ENV=1', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', 
'/home/patchew2/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', 
'/var/tmp/patchew-tester-tmp-82odvgxj/src/docker-src.2020-05-13-20.42.51.22402:/var/tmp/qemu:z,ro',
 'qemu:centos7', '/var/tmp/qemu/run', 'test-quick']' returned non-zero exit 
status 2.
filter=--filter=label=com.qemu.instance.uuid=08679829d9d5406eac35a1969274235b
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-82odvgxj/src'
make: *** [docker-run-test-quick@centos7] Error 2

real13m11.300s
user0m8.296s


The full log is available at
http://patchew.org/logs/20200513173200.11830-1-alex.ben...@linaro.org/testing.docker-quick@centos7/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-de...@redhat.com

Re: [PATCH v3 2/3] tests/util-sockets: add abstract unix socket cases

[xiaoqiang zhao]

在 2020/5/13 下午11:49, Daniel P. Berrangé 写道:

On Sun, May 10, 2020 at 02:14:21PM +0800, xiaoqiang zhao wrote:

add cases to test tight and non-tight for abstract address type

Signed-off-by: xiaoqiang zhao
---
  tests/test-util-sockets.c | 83 +++
  1 file changed, 83 insertions(+)

diff --git a/tests/test-util-sockets.c b/tests/test-util-sockets.c
index 5fd947c7bf..8042fb9276 100644
--- a/tests/test-util-sockets.c
+++ b/tests/test-util-sockets.c
@@ -227,6 +227,84 @@ static void test_socket_fd_pass_num_nocli(void)
  g_free(addr.u.fd.str);
  }
  
+static gpointer unix_server_thread_func(gpointer user_data)

+{
+SocketAddress addr;
+Error *err = NULL;
+int fd = -1;
+int connfd = -1;
+struct sockaddr_un un;
+socklen_t len = sizeof(un);
+char name[] = "/tmp/unix-test.sock";

Fixed filenames are bad, as even though this is the abstract
namespace and thus safe from on-disk clashes, the abstract
namespace is still OS global. We should append both the PID
and a sequence of random bytes to get something which doesnt
clash if two copies of the unit test run concurrently.


agree.

Re: [PATCH] spapr: Add a new level of NUMA for GPUs

[Reza Arbab]

Hi David,

Thanks for your quick response!

On Mon, May 11, 2020 at 04:17:45PM +1000, David Gibson wrote:

1)

This would all be much simpler, if PAPR's representation of NUMA
distances weren't so awful.  Somehow it manages to be both so complex
that it's very hard to understand, and yet very limited in that it
has no way to represent distances in any absolute units, or even
specific ratios between distances.

Both qemu and the guest kernel can have an arbitrary set of nodes,
with an arbitrary matrix of distances between each pair, which we then
have to lossily encode into this PAPR nonsense.


Completely agree. I've revisited that section many times now and still 
find the descriptions of these properties almost incomprehensible.


The only way I've been able to make sense of it comes from reading the 
implementation, or experimentally tweaking the code/device tree to see 
if distances behave the way I expect.



2)

Alas, I don't think we can simply change this information.  We'll have
to do it conditionally on a new machine type.  This is guest visible
information, which shouldn't change under a running guest across
migration between different qemu versions.  At least for Linux guests
we'd probably get away with it, since I think it only reads this info
at boot, and across a migration we'd at worst get non-optimal
behaviour, not actual breakage.


Sure, that makes sense. I'll try making the change conditional on a flag 
that can be set on new machine types.



3)

I'm not sure that this version is totally correct w.r.t. PAPR.  But
then, I'm also not really sure of that for the existing version.
Specifically it's not at all clear from PAPR if the IDs used at each
level of the ibm,associativity need to be:
  a) globally unique
  b) unique only within the associativity level they appear at
or c) unique only within the "node" at the next higher level they
 belong to


Again, I'm no authority but it seems to be (b):

  "To determine the associativity between any two resources, the OS
scans down the two resources associativity lists in all pair wise 
combinations counting how many domains are the same until the first 
domain where the two list do not agree."


FWIW, using the same number for id at multiple levels has been working 
in practice.



4)

I'm not totally clear on the rationale for using the individual gpu's
numa ID at all levels, rather than just one.  I'm guessing this is so
that the gpu memory blocks are distant from each other as well as
distant from main memory.  Is that right?


It's not necessary to use it at all levels--that was me trying to 
completely replicate that old firmware change. Strictly speaking, since 
only reference-points 4 (and now 2) are significant, that part could 
just be:


@@ -362,7 +362,7 @@ void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, 
void *fdt)
 uint32_t associativity[] = {
 cpu_to_be32(0x4),
 SPAPR_GPU_NUMA_ID,
-SPAPR_GPU_NUMA_ID,
+cpu_to_be32(nvslot->numa_id),
 SPAPR_GPU_NUMA_ID,
 cpu_to_be32(nvslot->numa_id)
 };

I think the rationale is that if those other levels got added to 
reference-points in the future, you'd likely want the GPU to be distinct 
there too.


--
Reza Arbab

[PATCH 4/4] target/i386: fix fbstp handling of out-of-range values

[Joseph Myers]

The fbstp implementation fails to check for out-of-range and invalid
values, instead just taking the result of conversion to int64_t and
storing its sign and low 18 decimal digits.  Fix this by checking for
an out-of-range result (invalid conversions always result in INT64_MAX
or INT64_MIN from the softfloat code, which are large enough to be
considered as out-of-range by this code) and storing the packed BCD
indefinite encoding in that case.

Signed-off-by: Joseph Myers 
---
 target/i386/fpu_helper.c |  10 +++
 tests/tcg/i386/test-i386-fbstp.c | 115 +++
 2 files changed, 125 insertions(+)

diff --git a/target/i386/fpu_helper.c b/target/i386/fpu_helper.c
index e1872b3fc0..96c512fedf 100644
--- a/target/i386/fpu_helper.c
+++ b/target/i386/fpu_helper.c
@@ -732,6 +732,16 @@ void helper_fbst_ST0(CPUX86State *env, target_ulong ptr)
 
 val = floatx80_to_int64(ST0, >fp_status);
 mem_ref = ptr;
+if (val >= 100LL || val <= -100LL) {
+float_raise(float_flag_invalid, >fp_status);
+while (mem_ref < ptr + 7) {
+cpu_stb_data_ra(env, mem_ref++, 0, GETPC());
+}
+cpu_stb_data_ra(env, mem_ref++, 0xc0, GETPC());
+cpu_stb_data_ra(env, mem_ref++, 0xff, GETPC());
+cpu_stb_data_ra(env, mem_ref++, 0xff, GETPC());
+return;
+}
 mem_end = mem_ref + 9;
 if (SIGND(temp)) {
 cpu_stb_data_ra(env, mem_end, 0x80, GETPC());
diff --git a/tests/tcg/i386/test-i386-fbstp.c b/tests/tcg/i386/test-i386-fbstp.c
index d368949188..73bf56b9dc 100644
--- a/tests/tcg/i386/test-i386-fbstp.c
+++ b/tests/tcg/i386/test-i386-fbstp.c
@@ -1,8 +1,19 @@
 /* Test fbstp instruction.  */
 
+#include 
 #include 
 #include 
 
+union u {
+struct { uint64_t sig; uint16_t sign_exp; } s;
+long double ld;
+};
+
+volatile union u ld_invalid_1 = { .s = { 1, 1234 } };
+volatile union u ld_invalid_2 = { .s = { 0, 1234 } };
+volatile union u ld_invalid_3 = { .s = { 0, 0x7fff } };
+volatile union u ld_invalid_4 = { .s = { (UINT64_C(1) << 63) - 1, 0x7fff } };
+
 int main(void)
 {
 int ret = 0;
@@ -21,5 +32,109 @@ int main(void)
 printf("FAIL: fbstp -0.1\n");
 ret = 1;
 }
+memset(out, 0x1f, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-987654321987654321.0L) :
+  "st");
+out[9] &= 0x80;
+if (memcmp(out, "\x21\x43\x65\x87\x19\x32\x54\x76\x98\x80",
+   sizeof out) != 0) {
+printf("FAIL: fbstp -987654321987654321\n");
+ret = 1;
+}
+memset(out, 0x12, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (99.5L) :
+  "st");
+if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+printf("FAIL: fbstp 99.5\n");
+ret = 1;
+}
+memset(out, 0x12, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (100.0L) :
+  "st");
+if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+printf("FAIL: fbstp 100\n");
+ret = 1;
+}
+memset(out, 0x12, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (1e30L) : "st");
+if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+printf("FAIL: fbstp 1e30\n");
+ret = 1;
+}
+memset(out, 0x12, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-99.5L) :
+  "st");
+if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+printf("FAIL: fbstp -99.5\n");
+ret = 1;
+}
+memset(out, 0x12, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-100.0L) :
+  "st");
+if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+printf("FAIL: fbstp -100\n");
+ret = 1;
+}
+memset(out, 0x12, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-1e30L) : "st");
+if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+printf("FAIL: fbstp -1e30\n");
+ret = 1;
+}
+memset(out, 0x12, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (__builtin_infl()) : "st");
+if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+printf("FAIL: fbstp inf\n");
+ret = 1;
+}
+memset(out, 0x12, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-__builtin_infl()) :
+  "st");
+if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+printf("FAIL: fbstp -inf\n");
+ret = 1;
+}
+memset(out, 0x12, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (__builtin_nanl("")) :
+  "st");
+if (memcmp(out, "\0\0\0\0\0\0\0\xc0\xff\xff", sizeof out) != 0) {
+

[Bug 1878501] [NEW] qemu-i386 does not define AT_SYSINFO

[easyaspi314]

Public bug reported:

qemu-i386 does not define the AT_SYSINFO auxval when running i386 Linux
binaries.

On most libcs, this is properly handled, but this is mandatory for the
i686 Bionic (Android) libc or it will segfault.

This is due to a blind assumption that getauxval(AT_SYSINFO) will return
a valid function pointer:

The code varies from version to version, but it looks like this:

void *__libc_sysinfo;
// mangled as _Z19__libc_init_sysinfov
void __libc_init_sysinfo() {
  bool dummy;
  // __bionic_getauxval = getauxval
  __libc_sysinfo = reinterpret_cast(__bionic_getauxval(AT_SYSINFO, 
dummy));
}

A simple way to reproduce is to compile a basic C program against the
NDK:

int main(void) { return 0; }

$ i686-linux-android-clang -static empty.c -o empty
$ qemu-i386 -cpu max ./empty
qemu: uncaught target signal 11 (Segmentation fault) - core dumped
Segmentation fault

The place where it segfaults is misleading: It will, at least on the
current NDK, crash on __set_thread_area, this is due to it calling a
function pointer to __libc_sysinfo returned by __kernel_syscall.

QEMU 4.1.1 (aarch64)
Pixel 2 XL via Termux

** Affects: qemu
 Importance: Undecided
 Status: New


** Tags: i386 linux-user

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1878501

Title:
  qemu-i386 does not define AT_SYSINFO

Status in QEMU:
  New

Bug description:
  qemu-i386 does not define the AT_SYSINFO auxval when running i386
  Linux binaries.

  On most libcs, this is properly handled, but this is mandatory for the
  i686 Bionic (Android) libc or it will segfault.

  This is due to a blind assumption that getauxval(AT_SYSINFO) will
  return a valid function pointer:

  The code varies from version to version, but it looks like this:

  void *__libc_sysinfo;
  // mangled as _Z19__libc_init_sysinfov
  void __libc_init_sysinfo() {
bool dummy;
// __bionic_getauxval = getauxval
__libc_sysinfo = reinterpret_cast(__bionic_getauxval(AT_SYSINFO, 
dummy));
  }

  A simple way to reproduce is to compile a basic C program against the
  NDK:

  int main(void) { return 0; }

  $ i686-linux-android-clang -static empty.c -o empty
  $ qemu-i386 -cpu max ./empty
  qemu: uncaught target signal 11 (Segmentation fault) - core dumped
  Segmentation fault

  The place where it segfaults is misleading: It will, at least on the
  current NDK, crash on __set_thread_area, this is due to it calling a
  function pointer to __libc_sysinfo returned by __kernel_syscall.

  QEMU 4.1.1 (aarch64)
  Pixel 2 XL via Termux

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1878501/+subscriptions

[PATCH 3/4] target/i386: fix fbstp handling of negative zero

[Joseph Myers]

The fbstp implementation stores +0 when the rounded result should be
-0 because it compares an integer value with 0 to determine the sign.
Fix this by checking the sign bit of the operand instead.

Signed-off-by: Joseph Myers 
---
 target/i386/fpu_helper.c |  5 -
 tests/tcg/i386/test-i386-fbstp.c | 25 +
 2 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 tests/tcg/i386/test-i386-fbstp.c

diff --git a/target/i386/fpu_helper.c b/target/i386/fpu_helper.c
index 51372c371b..e1872b3fc0 100644
--- a/target/i386/fpu_helper.c
+++ b/target/i386/fpu_helper.c
@@ -726,11 +726,14 @@ void helper_fbst_ST0(CPUX86State *env, target_ulong ptr)
 int v;
 target_ulong mem_ref, mem_end;
 int64_t val;
+CPU_LDoubleU temp;
+
+temp.d = ST0;
 
 val = floatx80_to_int64(ST0, >fp_status);
 mem_ref = ptr;
 mem_end = mem_ref + 9;
-if (val < 0) {
+if (SIGND(temp)) {
 cpu_stb_data_ra(env, mem_end, 0x80, GETPC());
 val = -val;
 } else {
diff --git a/tests/tcg/i386/test-i386-fbstp.c b/tests/tcg/i386/test-i386-fbstp.c
new file mode 100644
index 00..d368949188
--- /dev/null
+++ b/tests/tcg/i386/test-i386-fbstp.c
@@ -0,0 +1,25 @@
+/* Test fbstp instruction.  */
+
+#include 
+#include 
+
+int main(void)
+{
+int ret = 0;
+unsigned char out[10];
+memset(out, 0xfe, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-0.0L) : "st");
+out[9] &= 0x80;
+if (memcmp(out, "\0\0\0\0\0\0\0\0\0\x80", sizeof out) != 0) {
+printf("FAIL: fbstp -0\n");
+ret = 1;
+}
+memset(out, 0x12, sizeof out);
+__asm__ volatile ("fbstp %0" : "=m" (out) : "t" (-0.1L) : "st");
+out[9] &= 0x80;
+if (memcmp(out, "\0\0\0\0\0\0\0\0\0\x80", sizeof out) != 0) {
+printf("FAIL: fbstp -0.1\n");
+ret = 1;
+}
+return ret;
+}
-- 
2.17.1


-- 
Joseph S. Myers
jos...@codesourcery.com

[PATCH 2/4] target/i386: fix fxam handling of invalid encodings

[Joseph Myers]

The fxam implementation does not check for invalid encodings, instead
treating them like NaN or normal numbers depending on the exponent.
Fix it to check that the high bit of the significand is set before
treating an encoding as NaN or normal, thus resulting in correct
handling (all of C0, C2 and C3 cleared) for invalid encodings.

Signed-off-by: Joseph Myers 
---
 target/i386/fpu_helper.c|   4 +-
 tests/tcg/i386/test-i386-fxam.c | 143 
 2 files changed, 145 insertions(+), 2 deletions(-)
 create mode 100644 tests/tcg/i386/test-i386-fxam.c

diff --git a/target/i386/fpu_helper.c b/target/i386/fpu_helper.c
index 38968b2ec7..51372c371b 100644
--- a/target/i386/fpu_helper.c
+++ b/target/i386/fpu_helper.c
@@ -1099,7 +1099,7 @@ void helper_fxam_ST0(CPUX86State *env)
 if (expdif == MAXEXPD) {
 if (MANTD(temp) == 0x8000ULL) {
 env->fpus |= 0x500; /* Infinity */
-} else {
+} else if (MANTD(temp) & 0x8000ULL) {
 env->fpus |= 0x100; /* NaN */
 }
 } else if (expdif == 0) {
@@ -1108,7 +1108,7 @@ void helper_fxam_ST0(CPUX86State *env)
 } else {
 env->fpus |= 0x4400; /* Denormal */
 }
-} else {
+} else if (MANTD(temp) & 0x8000ULL) {
 env->fpus |= 0x400;
 }
 }
diff --git a/tests/tcg/i386/test-i386-fxam.c b/tests/tcg/i386/test-i386-fxam.c
new file mode 100644
index 00..ddd76ca42d
--- /dev/null
+++ b/tests/tcg/i386/test-i386-fxam.c
@@ -0,0 +1,143 @@
+/* Test fxam instruction.  */
+
+#include 
+#include 
+
+union u {
+struct { uint64_t sig; uint16_t sign_exp; } s;
+long double ld;
+};
+
+volatile union u ld_pseudo_m16382 = { .s = { UINT64_C(1) << 63, 0 } };
+volatile union u ld_pseudo_nm16382 = { .s = { UINT64_C(1) << 63, 0x8000 } };
+volatile union u ld_invalid_1 = { .s = { 1, 1234 } };
+volatile union u ld_invalid_2 = { .s = { 0, 1234 } };
+volatile union u ld_invalid_3 = { .s = { 0, 0x7fff } };
+volatile union u ld_invalid_4 = { .s = { (UINT64_C(1) << 63) - 1, 0x7fff } };
+volatile union u ld_invalid_n1 = { .s = { 1, 0x8123 } };
+volatile union u ld_invalid_n2 = { .s = { 0, 0x8123 } };
+volatile union u ld_invalid_n3 = { .s = { 0, 0x } };
+volatile union u ld_invalid_n4 = { .s = { (UINT64_C(1) << 63) - 1, 0x } };
+
+#define C0 (1 << 8)
+#define C1 (1 << 9)
+#define C2 (1 << 10)
+#define C3 (1 << 14)
+#define FLAGS (C0 | C1 | C2 | C3)
+
+int main(void)
+{
+short sw;
+int ret = 0;
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (0.0L));
+if ((sw & FLAGS) != C3) {
+printf("FAIL: +0\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (-0.0L));
+if ((sw & FLAGS) != (C3 | C1)) {
+printf("FAIL: -0\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (1.0L));
+if ((sw & FLAGS) != C2) {
+printf("FAIL: +normal\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (-1.0L));
+if ((sw & FLAGS) != (C2 | C1)) {
+printf("FAIL: -normal\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (__builtin_infl()));
+if ((sw & FLAGS) != (C2 | C0)) {
+printf("FAIL: +inf\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (-__builtin_infl()));
+if ((sw & FLAGS) != (C2 | C1 | C0)) {
+printf("FAIL: -inf\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (__builtin_nanl("")));
+if ((sw & FLAGS) != C0) {
+printf("FAIL: +nan\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (-__builtin_nanl("")));
+if ((sw & FLAGS) != (C1 | C0)) {
+printf("FAIL: -nan\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (__builtin_nansl("")));
+if ((sw & FLAGS) != C0) {
+printf("FAIL: +snan\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (-__builtin_nansl("")));
+if ((sw & FLAGS) != (C1 | C0)) {
+printf("FAIL: -snan\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (0x1p-16445L));
+if ((sw & FLAGS) != (C3 | C2)) {
+printf("FAIL: +denormal\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (-0x1p-16445L));
+if ((sw & FLAGS) != (C3 | C2 | C1)) {
+printf("FAIL: -denormal\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (ld_pseudo_m16382.ld));
+if ((sw & FLAGS) != (C3 | C2)) {
+printf("FAIL: +pseudo-denormal\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" (ld_pseudo_nm16382.ld));
+if ((sw & FLAGS) != (C3 | C2 | C1)) {
+printf("FAIL: -pseudo-denormal\n");
+ret = 1;
+}
+__asm__ volatile ("fxam\nfnstsw" : "=a" (sw) : "t" 

[PATCH 1/4] target/i386: fix floating-point load-constant rounding

[Joseph Myers]

The implementations of the fldl2t, fldl2e, fldpi, fldlg2 and fldln2
instructions load fixed constants independent of the rounding mode.
Fix them to load a value correctly rounded for the current rounding
mode (but always rounded to 64-bit precision independent of the
precision control, and without setting "inexact") as specified.

Signed-off-by: Joseph Myers 
---
 target/i386/fpu_helper.c  |  54 +++-
 tests/tcg/i386/test-i386-fldcst.c | 199 ++
 2 files changed, 248 insertions(+), 5 deletions(-)
 create mode 100644 tests/tcg/i386/test-i386-fldcst.c

diff --git a/target/i386/fpu_helper.c b/target/i386/fpu_helper.c
index 0c3fce933c..38968b2ec7 100644
--- a/target/i386/fpu_helper.c
+++ b/target/i386/fpu_helper.c
@@ -59,8 +59,13 @@
 #define FPUC_EM 0x3f
 
 #define floatx80_lg2 make_floatx80(0x3ffd, 0x9a209a84fbcff799LL)
+#define floatx80_lg2_d make_floatx80(0x3ffd, 0x9a209a84fbcff798LL)
 #define floatx80_l2e make_floatx80(0x3fff, 0xb8aa3b295c17f0bcLL)
+#define floatx80_l2e_d make_floatx80(0x3fff, 0xb8aa3b295c17f0bbLL)
 #define floatx80_l2t make_floatx80(0x4000, 0xd49a784bcd1b8afeLL)
+#define floatx80_l2t_u make_floatx80(0x4000, 0xd49a784bcd1b8affLL)
+#define floatx80_ln2_d make_floatx80(0x3ffe, 0xb17217f7d1cf79abLL)
+#define floatx80_pi_d make_floatx80(0x4000, 0xc90fdaa22168c234LL)
 
 #if !defined(CONFIG_USER_ONLY)
 static qemu_irq ferr_irq;
@@ -544,27 +549,66 @@ void helper_fld1_ST0(CPUX86State *env)
 
 void helper_fldl2t_ST0(CPUX86State *env)
 {
-ST0 = floatx80_l2t;
+switch (env->fpuc & FPU_RC_MASK) {
+case FPU_RC_UP:
+ST0 = floatx80_l2t_u;
+break;
+default:
+ST0 = floatx80_l2t;
+break;
+}
 }
 
 void helper_fldl2e_ST0(CPUX86State *env)
 {
-ST0 = floatx80_l2e;
+switch (env->fpuc & FPU_RC_MASK) {
+case FPU_RC_DOWN:
+case FPU_RC_CHOP:
+ST0 = floatx80_l2e_d;
+break;
+default:
+ST0 = floatx80_l2e;
+break;
+}
 }
 
 void helper_fldpi_ST0(CPUX86State *env)
 {
-ST0 = floatx80_pi;
+switch (env->fpuc & FPU_RC_MASK) {
+case FPU_RC_DOWN:
+case FPU_RC_CHOP:
+ST0 = floatx80_pi_d;
+break;
+default:
+ST0 = floatx80_pi;
+break;
+}
 }
 
 void helper_fldlg2_ST0(CPUX86State *env)
 {
-ST0 = floatx80_lg2;
+switch (env->fpuc & FPU_RC_MASK) {
+case FPU_RC_DOWN:
+case FPU_RC_CHOP:
+ST0 = floatx80_lg2_d;
+break;
+default:
+ST0 = floatx80_lg2;
+break;
+}
 }
 
 void helper_fldln2_ST0(CPUX86State *env)
 {
-ST0 = floatx80_ln2;
+switch (env->fpuc & FPU_RC_MASK) {
+case FPU_RC_DOWN:
+case FPU_RC_CHOP:
+ST0 = floatx80_ln2_d;
+break;
+default:
+ST0 = floatx80_ln2;
+break;
+}
 }
 
 void helper_fldz_ST0(CPUX86State *env)
diff --git a/tests/tcg/i386/test-i386-fldcst.c 
b/tests/tcg/i386/test-i386-fldcst.c
new file mode 100644
index 00..e635432ccf
--- /dev/null
+++ b/tests/tcg/i386/test-i386-fldcst.c
@@ -0,0 +1,199 @@
+/* Test instructions loading floating-point constants.  */
+
+#include 
+#include 
+
+volatile long double ld_res;
+
+int main(void)
+{
+short cw;
+int ret = 0;
+
+/* Round to nearest.  */
+__asm__ volatile ("fnstcw %0" : "=m" (cw));
+cw = (cw & ~0xc00) | 0x000;
+__asm__ volatile ("fldcw %0" : : "m" (cw));
+__asm__ volatile ("fldl2t" : "=t" (ld_res));
+if (ld_res != 0x3.5269e12f346e2bf8p+0L) {
+printf("FAIL: fldl2t N\n");
+ret = 1;
+}
+/* Round downward.  */
+__asm__ volatile ("fnstcw %0" : "=m" (cw));
+cw = (cw & ~0xc00) | 0x400;
+__asm__ volatile ("fldcw %0" : : "m" (cw));
+__asm__ volatile ("fldl2t" : "=t" (ld_res));
+if (ld_res != 0x3.5269e12f346e2bf8p+0L) {
+printf("FAIL: fldl2t D\n");
+ret = 1;
+}
+/* Round toward zero.  */
+__asm__ volatile ("fnstcw %0" : "=m" (cw));
+cw = (cw & ~0xc00) | 0xc00;
+__asm__ volatile ("fldcw %0" : : "m" (cw));
+__asm__ volatile ("fldl2t" : "=t" (ld_res));
+if (ld_res != 0x3.5269e12f346e2bf8p+0L) {
+printf("FAIL: fldl2t Z\n");
+ret = 1;
+}
+/* Round upward.  */
+__asm__ volatile ("fnstcw %0" : "=m" (cw));
+cw = (cw & ~0xc00) | 0x800;
+__asm__ volatile ("fldcw %0" : : "m" (cw));
+__asm__ volatile ("fldl2t" : "=t" (ld_res));
+if (ld_res != 0x3.5269e12f346e2bfcp+0L) {
+printf("FAIL: fldl2t U\n");
+ret = 1;
+}
+
+/* Round to nearest.  */
+__asm__ volatile ("fnstcw %0" : "=m" (cw));
+cw = (cw & ~0xc00) | 0x000;
+__asm__ volatile ("fldcw %0" : : "m" (cw));
+__asm__ volatile ("fldl2e" : "=t" (ld_res));
+if (ld_res != 0x1.71547652b82fe178p+0L) {
+printf("FAIL: fldl2e N\n");
+ret = 1;
+}
+/* Round downward.  */
+__asm__ volatile ("fnstcw %0" : "=m" (cw));
+cw = (cw & ~0xc00) | 0x400;
+__asm__ volatile ("fldcw %0" : : "m" (cw));
+__asm__ volatile 

[PATCH 0/4] target/i386: miscellaneous x87 fixes

[Joseph Myers]

Following my previous patch series

and

for problems found in the x87 floating-point emulation, this patch
series fixes further miscellaneous bugs in that emulation.

There are further problems with x87 emulation that I am aware of and
intend to address in future patch series.  Those other problems, not
addressed by the first three patch series, generally relate to
exceptions, flag setting and those instructions for which the
emulation currently converts to host double (so losing range and
precision) and then works on host double for the rest of the emulation
process before converting back to floatx80 at the end.  Thus, the same
comments as for the previous patch series apply about this patch
series not fixing missing propagation of exceptions even when it adds
code to set exceptions in the softfloat state.

Joseph Myers (4):
  target/i386: fix floating-point load-constant rounding
  target/i386: fix fxam handling of invalid encodings
  target/i386: fix fbstp handling of negative zero
  target/i386: fix fbstp handling of out-of-range values

 target/i386/fpu_helper.c  |  73 +--
 tests/tcg/i386/test-i386-fbstp.c  | 140 +
 tests/tcg/i386/test-i386-fldcst.c | 199 ++
 tests/tcg/i386/test-i386-fxam.c   | 143 +
 4 files changed, 547 insertions(+), 8 deletions(-)
 create mode 100644 tests/tcg/i386/test-i386-fbstp.c
 create mode 100644 tests/tcg/i386/test-i386-fldcst.c
 create mode 100644 tests/tcg/i386/test-i386-fxam.c

-- 
2.17.1


-- 
Joseph S. Myers
jos...@codesourcery.com

Re: [PATCH 0/2] 9pfs: regression init_in_iov_from_pdu truncating size

[Stefano Stabellini]

On Thu, 14 May 2020, Greg Kurz wrote:
> On Tue, 12 May 2020 11:38:23 +0200
> Greg Kurz  wrote:
> 
> > On Sun, 10 May 2020 19:41:52 +0200
> > Christian Schoenebeck  wrote:
> > 
> > > Stefano, looks like your original patch needs some more fine tuning:
> > > 
> > > https://bugs.launchpad.net/bugs/1877688
> > > 
> > > Please check if the assumptions I made about Xen are correct, and please
> > > also test whether these changes still work for you with Xen as intended by
> > > you.
> > > 
> > > Christian Schoenebeck (2):
> > >   xen-9pfs: Fix log messages of reply errors
> > >   9pfs: fix init_in_iov_from_pdu truncating size
> > > 
> > >  hw/9pfs/virtio-9p-device.c | 35 
> > >  hw/9pfs/xen-9p-backend.c   | 41 --
> > >  2 files changed, 58 insertions(+), 18 deletions(-)
> > > 
> > 
> > Sorry, I'm off this week, not sure I'll have time to review.
> > So I've only applied patch 1 for now and I'll let Stefano
> > and you sort out what should be done for patch 2.
> > 
> 
> IIUC this requires more thinking and may end up in a complete rewrite of
> the truncating logic. I intend to send a PR soon : is it worth keeping
> patch 1 anyway ? 

Patch 1 is fine. For patch 2 we might need a little more time.

Re: [PATCH 2/2] 9pfs: fix init_in_iov_from_pdu truncating size

[Stefano Stabellini]

On Wed, 13 May 2020, Christian Schoenebeck wrote:
> On Mittwoch, 13. Mai 2020 01:24:16 CEST Stefano Stabellini wrote:
> > Let me premise that this patch fixes an important bug, so I am not
> > asking you to do any more work to get this patch committed right now :-)
> 
> Actually, I realized that this overall issue is much more delicate to handle 
> than I thought. My patch would still miss other important things, like e.g. 
> it 
> does not roll back the read / readdir streaming positions of the respective 
> FID on 9P server side. -> data loss / undefined behaviour on subsequent read 
> / 
> readdir
> 
> Also the 'count' parameter in the Rread response would need to be updated as 
> well. -> undefined behaviour
> 
> For these and other reasons already described, I start to think that it is 
> actually not a good idea to truncate the response at all, at least not at 
> that 
> final transport stage (as my or your patch do).
> 
> Question: is there really no way to handle this overall issue exlusively on 
> Xen side, i.e. delaying response delivery until large enough transport buffer 
> would be available or acquiring on-demand the required buffer size? Or by 
> looking ahead at msize in advance (see below)?
> 
> >From what I see now, I would recommend completely reverting response 
> truncation on virtio transport side at least. Because so far I have not seen 
> a 
> single case with virtio where the transport buffer was too small for 
> delivering any response. And as you can see, response truncation at late 
> transport level opens all kinds of severe issues on other ends. It is not 
> easy 
> to do and requires a lot of code!
> 
> > But I think it would be good to test what would happen if the client did
> > a read on a directory with hundreds of entries, such as a maildir
> > directory. There has to be point where the number of directory entries
> > is larger than the shared buffer. What happens then?
> 
> I'm talking about virtio only now: that scenario does not happen. When client 
> opens a 9P session, client transmits 'msize' as parameter for the 9P sesssion 
> which is the "maximum message size" ever to be assumed on both sides for  
> client requests and server responses. By default msize is 4 kiB (4096 bytes) 
> with Linux clients, and our server would immediately error out if any client 
> tries to open a session with 'msize' < 4 kiB (for similar reasons discussed; 
> because some 9P response types cannot be truncated at all):
> 
> https://github.com/qemu/qemu/commit/
> e16453a31a00c1c0a199cab0617e8aa888f6419a#diff-f3e98ed0a65e27743b14785fa85b7d79
> 
> The Rread and Rreaddir responses are already truncated (in a sane and correct 
> way) in 9p.c by server's request handler in case the response would exceed 
> 'msize'. For that reason it is actually unexpected that transport would not 
> be 
> able to provide the required transport buffer size for any response.
>
> > I am guessing that we'll have to implement the "truncate at the
> > boundaries of individual entries" to get it right in all cases.
> > 
> > Given that it doesn't look like truncation would work right with
> > Rreaddir anyway today, I think it would be OK to fix it in a separate
> > patch.
> 
> If there is really no way with Xen to ensure there's always a buffer size 
> according to msize, 

Thank you for the long writeup. Let's see if we can come up with a
good plan.

The Xen transport
(https://xenbits.xenproject.org/docs/4.13-testing/misc/9pfs.html) has a
shared memory area with 64 pages, so 256KB. It is split in half: 128KB
for requests and 128KB for responses. 

The original error message is the following:

  xen be: 9pfs-0: xen be: 9pfs-0: Xen 9pfs request type 116 needs 126987 bytes, 
buffer has 126965
  Xen 9pfs request type 116needs 126987 bytes, buffer has 126965

and msize was set to 131072 in the Linux client.


So transport_size == msize. However, there can be multiple requests and
responses inflight at any given time. It means that even with
transport_size=128KB and msize=4KB we could still have so many outstanding
requests that the available memory is less then 4KB.

I admit that with transport_size=128KB and msize=4KB it would be so
unlikely to happen that it would be close to impossible to reproduce the
error. But theoretically it is possible.

If we can't come up with any better ideas, we could simply limit the
msize in the Linux client to something like transport_size/2 or
transport_size/4, but it doesn't feel right.


> > I think we have to return here because the connection gets closed by
> > xen_9pfs_disconnect. I wonder if we should return -EAGAIN like you
> > suggested instead of calling xen_9pfs_disconnect() which is
> > irrecoverable. But that could be done later in a separate patch.
> 
> Even -EAGAIN would require a bunch of code, since the precise error response 
> type depends a) on the protocol dialect (9p2000.L vs. 9p2000.u) and b) even 
> depending on the precise request type. For some 9P request types 

Re: Assertion failure through vring_split_desc_read

[John Snow]

On 5/10/20 11:51 PM, Alexander Bulekov wrote:
> Hello,
> While fuzzing, I found an input that triggers an assertion failure
> through virtio-rng -> vring_split_desc_read. Maybe this is related to:
> Message-ID: <20200511033001.dzvtbdhl3oz5p...@mozz.bu.edu> 
> Assertion failure through virtio_lduw_phys_cached
> 
> #8 0x7fe6a9acf091 in __assert_fail 
> /build/glibc-GwnBeO/glibc-2.30/assert/assert.c:101:3
> #9 0x564cbe7d96fd in address_space_read_cached include/exec/memory.h:2423:5
> #10 0x564cbe7e79c5 in vring_split_desc_read hw/virtio/virtio.c:236:5
> #11 0x564cbe7e84ce in virtqueue_split_read_next_desc hw/virtio/virtio.c:929:5
> #12 0x564cbe78f86b in virtqueue_split_get_avail_bytes 
> hw/virtio/virtio.c:1009:18
> #13 0x564cbe78ab22 in virtqueue_get_avail_bytes hw/virtio/virtio.c:1208:9
> #14 0x564cc08aade1 in get_request_size hw/virtio/virtio-rng.c:40:5
> #15 0x564cc08aa20b in virtio_rng_process hw/virtio/virtio-rng.c:115:12
> #16 0x564cc08a8c48 in virtio_rng_set_status hw/virtio/virtio-rng.c:172:5
> #17 0x564cbe7a50be in virtio_set_status hw/virtio/virtio.c:1876:9
> #18 0x564cc08d1b8f in virtio_pci_common_write hw/virtio/virtio-pci.c:1245:9
> 
> I can reproduce it in a qemu 5.0 build using these qtest commands:
> https://paste.debian.net/plain/1146089
> (not including them here, as some are quite long)
> 
> wget https://paste.debian.net/plain/1146089 -O qtest-trace; 
> ~/Development/qemu/build/i386-softmmu/qemu-system-i386 -M pc-q35-5.0  -device 
> virtio-rng-pci,addr=04.0 -display none -nodefaults -nographic -qtest stdio < 
> qtest-trace
> 
> Please let me know if I can provide any further info.
> -Alex
> 

Do you have a writeup somewhere of how you are approaching fuzzing and
how you've found this pile of bugs so far?

Might make for a good blog post.

--js

Re: Is qemu could be a "FSM" state machine if running on a "quiet and clean" host pc without random event input?

[John Snow]

On 5/11/20 10:07 AM, tugouxp wrote:
> Hi folks:
>   i want to know about whether therr are limitations during qemu
> emulation systems, for exampe, did the regular bugs corener case cant be
> duplicated on qeme but exist on real boads?
> 

It's possible, yes. QEMU emulates instead of simulates. We do not try to
reproduce accurate cycle timings. There may be bugs that exist in real
hardware but don't reproduce with QEMU, or, of course, the other way around.

> why thing this is that , i have ever use hdl simulator (modsim and
> iverilog) and openrisc processor to emulate the linux and ucos running,
> and see the waveform of the simulateion process of the operations systems.
> i found an interesting things, if i take just the tick interrupt as the
> only testbech event source,the  kernel simulation waveform is identical
> duplicated again and again, which means i can predicate future behavior.
> 

Yeah, we are not operating on the verilog level for any of the hardware
we emulate.

> i think this something like qemu work principle and so want to know,
> whether the qemu has this limitation? is the simulation proces a "FSM" 
> that with definition output if the input event are all regular and
> without random?
> 

Well, if you only want determinism and not accuracy, it might be
possible, but I have to admit to you that I've never tried to instrument
or measure this. I assume there are many places where cycle
indeterminancy comes in from the host system.

And I trust x86 about as far as I can throw it.

I assume there are many barriers to this, but maybe folks who worked on
the replay debugger could give some more authoritative idea.


> thank you
> 

Re: [PATCH 0/2] 9pfs: regression init_in_iov_from_pdu truncating size

[Greg Kurz]

On Tue, 12 May 2020 11:38:23 +0200
Greg Kurz  wrote:

> On Sun, 10 May 2020 19:41:52 +0200
> Christian Schoenebeck  wrote:
> 
> > Stefano, looks like your original patch needs some more fine tuning:
> > 
> > https://bugs.launchpad.net/bugs/1877688
> > 
> > Please check if the assumptions I made about Xen are correct, and please
> > also test whether these changes still work for you with Xen as intended by
> > you.
> > 
> > Christian Schoenebeck (2):
> >   xen-9pfs: Fix log messages of reply errors
> >   9pfs: fix init_in_iov_from_pdu truncating size
> > 
> >  hw/9pfs/virtio-9p-device.c | 35 
> >  hw/9pfs/xen-9p-backend.c   | 41 --
> >  2 files changed, 58 insertions(+), 18 deletions(-)
> > 
> 
> Sorry, I'm off this week, not sure I'll have time to review.
> So I've only applied patch 1 for now and I'll let Stefano
> and you sort out what should be done for patch 2.
> 

IIUC this requires more thinking and may end up in a complete rewrite of
the truncating logic. I intend to send a PR soon : is it worth keeping
patch 1 anyway ? 

> Cheers,
> 
> --
> Greg
> 

[PATCH v2 6/6] target/ppc: Don't update radix PTE R/C bits with gdbstub

[Greg Kurz]

gdbstub shouldn't silently change guest visible state when doing address
translation. Since the R/C bits can only be updated when handling a MMU
fault, let's reuse the cause_excp flag and rename it to guest_visible.
While here drop a not very useful comment.

This was found while reading the code. I could verify that this affects
both powernv and pseries, but I failed to observe any actual bug.

Fixes: d04ea940c597 "target/ppc: Add support for Radix partition-scoped 
translation"
Signed-off-by: Greg Kurz 
---
 target/ppc/mmu-radix64.c |   39 +--
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/target/ppc/mmu-radix64.c b/target/ppc/mmu-radix64.c
index 7ce37cb778db..0d3922537c4c 100644
--- a/target/ppc/mmu-radix64.c
+++ b/target/ppc/mmu-radix64.c
@@ -274,7 +274,7 @@ static int ppc_radix64_partition_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
   ppc_v3_pate_t pate,
   hwaddr *h_raddr, int *h_prot,
   int *h_page_size, bool pde_addr,
-  bool cause_excp)
+  bool guest_visible)
 {
 int fault_cause = 0;
 hwaddr pte_addr;
@@ -289,14 +289,15 @@ static int ppc_radix64_partition_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
 if (pde_addr) { /* address being translated was that of a guest pde */
 fault_cause |= DSISR_PRTABLE_FAULT;
 }
-if (cause_excp) {
+if (guest_visible) {
 ppc_radix64_raise_hsi(cpu, rwx, eaddr, g_raddr, fault_cause);
 }
 return 1;
 }
 
-/* Update Reference and Change Bits */
-ppc_radix64_set_rc(cpu, rwx, pte, pte_addr, h_prot);
+if (guest_visible) {
+ppc_radix64_set_rc(cpu, rwx, pte, pte_addr, h_prot);
+}
 
 return 0;
 }
@@ -305,7 +306,7 @@ static int ppc_radix64_process_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
 vaddr eaddr, uint64_t pid,
 ppc_v3_pate_t pate, hwaddr 
*g_raddr,
 int *g_prot, int *g_page_size,
-bool cause_excp)
+bool guest_visible)
 {
 CPUState *cs = CPU(cpu);
 CPUPPCState *env = >env;
@@ -319,7 +320,7 @@ static int ppc_radix64_process_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
 size = 1ULL << ((pate.dw1 & PATE1_R_PRTS) + 12);
 if (offset >= size) {
 /* offset exceeds size of the process table */
-if (cause_excp) {
+if (guest_visible) {
 ppc_radix64_raise_si(cpu, rwx, eaddr, DSISR_NOPTE);
 }
 return 1;
@@ -340,7 +341,7 @@ static int ppc_radix64_process_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
 ret = ppc_radix64_partition_scoped_xlate(cpu, 0, eaddr, prtbe_addr,
  pate, _raddr, _prot,
  _page_size, true,
- cause_excp);
+ guest_visible);
 if (ret) {
 return ret;
 }
@@ -360,7 +361,7 @@ static int ppc_radix64_process_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
 _cause, _addr);
 if (ret) {
 /* No valid PTE */
-if (cause_excp) {
+if (guest_visible) {
 ppc_radix64_raise_si(cpu, rwx, eaddr, fault_cause);
 }
 return ret;
@@ -380,7 +381,7 @@ static int ppc_radix64_process_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
 ret = ppc_radix64_partition_scoped_xlate(cpu, 0, eaddr, pte_addr,
  pate, _raddr, _prot,
  _page_size, true,
- cause_excp);
+ guest_visible);
 if (ret) {
 return ret;
 }
@@ -389,7 +390,7 @@ static int ppc_radix64_process_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
  , g_page_size, , 
_cause);
 if (ret) {
 /* No valid pte */
-if (cause_excp) {
+if (guest_visible) {
 ppc_radix64_raise_si(cpu, rwx, eaddr, fault_cause);
 }
 return ret;
@@ -406,13 +407,15 @@ static int ppc_radix64_process_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
 
 if (ppc_radix64_check_prot(cpu, rwx, pte, _cause, g_prot, false)) {
 /* Access denied due to protection */
-if (cause_excp) {
+if (guest_visible) {
 ppc_radix64_raise_si(cpu, rwx, eaddr, fault_cause);
 }
 return 1;
 }
 
-   

[PATCH v2 4/6] target/ppc: Add missing braces in ppc_radix64_partition_scoped_xlate()

[Greg Kurz]

As per CODING_STYLE.

Fixes: d04ea940c597 "target/ppc: Add support for Radix partition-scoped 
translation"
Signed-off-by: Greg Kurz 
---
 target/ppc/mmu-radix64.c |3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/target/ppc/mmu-radix64.c b/target/ppc/mmu-radix64.c
index 07f956c9864f..fb7dfe25ba6f 100644
--- a/target/ppc/mmu-radix64.c
+++ b/target/ppc/mmu-radix64.c
@@ -286,8 +286,9 @@ static int ppc_radix64_partition_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
   pate.dw0 & PRTBE_R_RPDS, h_raddr, h_page_size,
   , _cause, _addr) ||
 ppc_radix64_check_prot(cpu, rwx, pte, _cause, h_prot, true)) {
-if (pde_addr) /* address being translated was that of a guest pde */
+if (pde_addr) { /* address being translated was that of a guest pde */
 fault_cause |= DSISR_PRTABLE_FAULT;
+}
 if (cause_excp) {
 ppc_radix64_raise_hsi(cpu, rwx, eaddr, g_raddr, fault_cause);
 }

[PATCH v2 3/6] target/ppc: Don't initialize some local variables in ppc_radix64_xlate()

[Greg Kurz]

It is the job of the ppc_radix64_get_fully_qualified_addr() function
which is called at the beginning of ppc_radix64_xlate() to set both
lpid *and* pid. It doesn't buy us anything to initialize them first.

Worse, a bug in ppc_radix64_get_fully_qualified_addr(), eg. failing to
set either lpid or pid, would be undetectable by static analysis tools
like coverity.

Some recent versions of gcc (eg. gcc-9.3.1-2.fc30) may still think
that lpid or pid is used uninitialized though, so this also adds
default cases in the switch statements to make it clear this cannot
happen.

Signed-off-by: Greg Kurz 
---
 target/ppc/mmu-radix64.c |6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/target/ppc/mmu-radix64.c b/target/ppc/mmu-radix64.c
index c76879f65b78..07f956c9864f 100644
--- a/target/ppc/mmu-radix64.c
+++ b/target/ppc/mmu-radix64.c
@@ -50,6 +50,8 @@ static bool ppc_radix64_get_fully_qualified_addr(const 
CPUPPCState *env,
 *lpid = 0;
 *pid = 0;
 break;
+default:
+g_assert_not_reached();
 }
 } else {  /* !MSR[HV] -> Guest */
 switch (eaddr & R_EADDR_QUADRANT) {
@@ -64,6 +66,8 @@ static bool ppc_radix64_get_fully_qualified_addr(const 
CPUPPCState *env,
 *lpid = env->spr[SPR_LPIDR];
 *pid = 0; /* pid set to 0 -> addresses guest operating system */
 break;
+default:
+g_assert_not_reached();
 }
 }
 
@@ -433,7 +437,7 @@ static int ppc_radix64_xlate(PowerPCCPU *cpu, vaddr eaddr, 
int rwx,
  bool cause_excp)
 {
 CPUPPCState *env = >env;
-uint64_t lpid = 0, pid = 0;
+uint64_t lpid, pid;
 ppc_v3_pate_t pate;
 int psize, prot;
 hwaddr g_raddr;

[PATCH v2 5/6] target/ppc: Fix arguments to ppc_radix64_partition_scoped_xlate()

[Greg Kurz]

The last two arguments have the bool type. Also, we shouldn't raise an
exception when using gdbstub.

This was found while reading the code. Since it only affects the powernv
machine, I didn't dig further to find an actual bug.

Fixes: d04ea940c597 "target/ppc: Add support for Radix partition-scoped 
translation"
Signed-off-by: Greg Kurz 
---
 target/ppc/mmu-radix64.c |6 --
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/target/ppc/mmu-radix64.c b/target/ppc/mmu-radix64.c
index fb7dfe25ba6f..7ce37cb778db 100644
--- a/target/ppc/mmu-radix64.c
+++ b/target/ppc/mmu-radix64.c
@@ -339,7 +339,8 @@ static int ppc_radix64_process_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
  */
 ret = ppc_radix64_partition_scoped_xlate(cpu, 0, eaddr, prtbe_addr,
  pate, _raddr, _prot,
- _page_size, 1, 1);
+ _page_size, true,
+ cause_excp);
 if (ret) {
 return ret;
 }
@@ -378,7 +379,8 @@ static int ppc_radix64_process_scoped_xlate(PowerPCCPU 
*cpu, int rwx,
 do {
 ret = ppc_radix64_partition_scoped_xlate(cpu, 0, eaddr, pte_addr,
  pate, _raddr, _prot,
- _page_size, 1, 1);
+ _page_size, true,
+ cause_excp);
 if (ret) {
 return ret;
 }

[PATCH v2 1/6] target/ppc: Pass const pointer to ppc_radix64_get_prot_amr()

[Greg Kurz]

This doesn't require write access to the CPU structure.

Signed-off-by: Greg Kurz 
---
 target/ppc/mmu-radix64.h |4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/target/ppc/mmu-radix64.h b/target/ppc/mmu-radix64.h
index 96228546aa85..f28c5794d071 100644
--- a/target/ppc/mmu-radix64.h
+++ b/target/ppc/mmu-radix64.h
@@ -55,9 +55,9 @@ static inline int ppc_radix64_get_prot_eaa(uint64_t pte)
(pte & R_PTE_EAA_X ? PAGE_EXEC : 0);
 }
 
-static inline int ppc_radix64_get_prot_amr(PowerPCCPU *cpu)
+static inline int ppc_radix64_get_prot_amr(const PowerPCCPU *cpu)
 {
-CPUPPCState *env = >env;
+const CPUPPCState *env = >env;
 int amr = env->spr[SPR_AMR] >> 62; /* We only care about key0 AMR63:62 */
 int iamr = env->spr[SPR_IAMR] >> 62; /* We only care about key0 IAMR63:62 
*/
 

[PATCH v2 0/6] target/ppc: Various clean-up and fixes for radix64

[Greg Kurz]

First three patches of this series are simple cleanups. The other
ones fix some regressions introduced by Cedric's recent addition
of partition-scoped translation.

Changes since v1:
- fix build break in patch 3
- introduce guest_visible argument in patch 6

--
Greg

---

Greg Kurz (6):
  target/ppc: Pass const pointer to ppc_radix64_get_prot_amr()
  target/ppc: Pass const pointer to ppc_radix64_get_fully_qualified_addr()
  target/ppc: Don't initialize some local variables in ppc_radix64_xlate()
  target/ppc: Add missing braces in ppc_radix64_partition_scoped_xlate()
  target/ppc: Fix arguments to ppc_radix64_partition_scoped_xlate()
  target/ppc: Don't update radix PTE R/C bits with gdbstub


 target/ppc/mmu-radix64.c |   53 --
 target/ppc/mmu-radix64.h |4 ++-
 2 files changed, 34 insertions(+), 23 deletions(-)

[PATCH v2 2/6] target/ppc: Pass const pointer to ppc_radix64_get_fully_qualified_addr()

[Greg Kurz]

This doesn't require write access to the CPU registers.

Signed-off-by: Greg Kurz 
---
 target/ppc/mmu-radix64.c |3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/target/ppc/mmu-radix64.c b/target/ppc/mmu-radix64.c
index 1404e53deca8..c76879f65b78 100644
--- a/target/ppc/mmu-radix64.c
+++ b/target/ppc/mmu-radix64.c
@@ -28,7 +28,8 @@
 #include "mmu-radix64.h"
 #include "mmu-book3s-v3.h"
 
-static bool ppc_radix64_get_fully_qualified_addr(CPUPPCState *env, vaddr eaddr,
+static bool ppc_radix64_get_fully_qualified_addr(const CPUPPCState *env,
+ vaddr eaddr,
  uint64_t *lpid, uint64_t *pid)
 {
 if (msr_hv) { /* MSR[HV] -> Hypervisor/bare metal */

Re: [PATCH v1 7/8] s390/kvm: header sync for diag318

[Collin Walling]

On 5/13/20 3:05 AM, Cornelia Huck wrote:
> On Fri,  8 May 2020 19:08:22 -0400
> Collin Walling  wrote:
> 
>> Signed-off-by: Collin Walling 
>> ---
>>  linux-headers/asm-s390/kvm.h | 5 +
>>  1 file changed, 5 insertions(+)
>>
>> diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
>> index 0138ccb0d8..b661feafdc 100644
>> --- a/linux-headers/asm-s390/kvm.h
>> +++ b/linux-headers/asm-s390/kvm.h
>> @@ -74,6 +74,7 @@ struct kvm_s390_io_adapter_req {
>>  #define KVM_S390_VM_CRYPTO  2
>>  #define KVM_S390_VM_CPU_MODEL   3
>>  #define KVM_S390_VM_MIGRATION   4
>> +#define KVM_S390_VM_MISC5
>>  
>>  /* kvm attributes for mem_ctrl */
>>  #define KVM_S390_VM_MEM_ENABLE_CMMA 0
>> @@ -171,6 +172,10 @@ struct kvm_s390_vm_cpu_subfunc {
>>  #define KVM_S390_VM_MIGRATION_START 1
>>  #define KVM_S390_VM_MIGRATION_STATUS2
>>  
>> +/* kvm attributes for KVM_S390_VM_MISC */
>> +#define KVM_S390_VM_MISC_ENABLE_DIAG318 0
>> +#define KVM_S390_VM_MISC_DIAG3181
>> +
>>  /* for KVM_GET_REGS and KVM_SET_REGS */
>>  struct kvm_regs {
>>  /* general purpose regs for s390 */
> 
> Hm... remind me what the state of the kernel part is?
> 

Kernel code to execute the instruction is in place today. A kernel
running on real hardware (or at least LPAR) can successfully execute the
instruction if the hardware supports it.

The Linux commit is: 4ad78b8651aacf26b3ab6d1e784952eb70469c43

The KVM code is still under review, since most of it needs to align with
the QEMU changes as well. Guest kernels executing the instruction rely
on both QEMU and KVM support. KVM handles the execution and interception
of the instruction, while QEMU handles enabling the instruction (by
negotiating with KVM to allow it and then setting the appropriate SCLP
bit) and migration of the data.

The latest changes to the KVM code is a new IOCTL that allows QEMU to
negotiate with KVM to actually enable the diag318 feature. By default,
KVM treats the feature as "disabled" unless userspace explicitly asks
for it.

The KVM discussion is: https://www.spinics.net/lists/kvm/msg215758.html

-- 
--
Regards,
Collin

Stay safe and stay healthy

Re: [PATCH v3 0/4] Additional parameters for qemu_img map

[no-reply]

Patchew URL: 
https://patchew.org/QEMU/20200513133629.18508-1-eyal.moscov...@oracle.com/



Hi,

This series failed the docker-mingw@fedora build test. Please find the testing 
commands and
their output below. If you have Docker installed, you can probably reproduce it
locally.

=== TEST SCRIPT BEGIN ===
#! /bin/bash
export ARCH=x86_64
make docker-image-fedora V=1 NETWORK=1
time make docker-test-mingw@fedora J=14 NETWORK=1
=== TEST SCRIPT END ===

  CC  x86_64-softmmu/ioport.o
  CC  x86_64-softmmu/qtest.o
/tmp/qemu-test/src/qemu-img.c: In function 'cvtnum_full':
/tmp/qemu-test/src/qemu-img.c:488:63: error: format '%ld' expects argument of 
type 'long int', but argument 3 has type 'int64_t' {aka 'long long int'} 
[-Werror=format=]
 error_report("Invalid %s specified. Must be between %ld bytes "
 ~~^
 %lld
  "to %ld bytes.", name, min, max);
 ~~~
/tmp/qemu-test/src/qemu-img.c:488:22: error: format '%ld' expects argument of 
type 'long int', but argument 4 has type 'int64_t' {aka 'long long int'} 
[-Werror=format=]
 error_report("Invalid %s specified. Must be between %ld bytes "
  ^~
  "to %ld bytes.", name, min, max);
---
  ~~^
  %lld
cc1: all warnings being treated as errors
make: *** [/tmp/qemu-test/src/rules.mak:69: qemu-img.o] Error 1
make: *** Waiting for unfinished jobs
  CC  aarch64-softmmu/dump/dump.o
  CC  x86_64-softmmu/memory.o
---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', 
'--label', 'com.qemu.instance.uuid=33f83b7d61224ad79355fee02312ee9c', '-u', 
'1003', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=', 
'-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 
'SHOW_ENV=', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', 
'/home/patchew2/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', 
'/var/tmp/patchew-tester-tmp-z4fvcbiq/src/docker-src.2020-05-13-18.10.43.324:/var/tmp/qemu:z,ro',
 'qemu:fedora', '/var/tmp/qemu/run', 'test-mingw']' returned non-zero exit 
status 2.
filter=--filter=label=com.qemu.instance.uuid=33f83b7d61224ad79355fee02312ee9c
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-z4fvcbiq/src'
make: *** [docker-run-test-mingw@fedora] Error 2

real2m36.638s
user0m7.720s


The full log is available at
http://patchew.org/logs/20200513133629.18508-1-eyal.moscov...@oracle.com/testing.docker-mingw@fedora/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-de...@redhat.com

Re: [PATCH RFC] configure: prefer sphinx-build to sphinx-build-3

[John Snow]

On 4/16/20 3:16 PM, Peter Maydell wrote:
> On Thu, 16 Apr 2020 at 19:22, John Snow  wrote:
>> My goal is to make virtual environments work out of the box.
>>
>> I.e., if you run ./configure from inside a VENV, it should "just work."
> 
> Yeah, this seems reasonable to me. If I understand your
> patch correctly it ought to work without breaking
> the setup Markus describes, because in that case
> 'sphinx-build' exists but will fail the test_sphinx_build
> step (because it's a Python 2 sphinx-build) and we'll
> then move on and use sphinx-build-3.
> 
> Patch looks good to me, but you'll need to rebase and update it
> to take account of commits 516e8b7d4a and 988ae6c3a7
> now in master.
> 
> thanks
> -- PMM
> 

OK, Done.

--js

Re: [PATCH v3 06/10] iotests: add testfinder.py

[John Snow]

On 5/7/20 1:43 PM, Vladimir Sementsov-Ogievskiy wrote:
> 21.04.2020 19:56, Kevin Wolf wrote:
>> Am 21.04.2020 um 09:35 hat Vladimir Sementsov-Ogievskiy geschrieben:
>>> Add python script with new logic of searching for tests:
>>>
>>> Current ./check behavior:
>>>   - tests are named [0-9][0-9][0-9]
>>>   - tests must be registered in group file (even if test doesn't belong
>>>     to any group, like 142)
>>>
>>> Behavior of new test:
>>>   - group file is dropped
>>>   - tests are searched by file-name instead of group file, so it's not
>>>     needed more to "register the test", just create it with name
>>>     *-test. Old names like [0-9][0-9][0-9] are supported too, but not
>>>     recommended for new tests
>>
>> I wonder if a tests/ subdirectory instead of the -test suffix would
>> organise things a bit better.
>>
> 
> It will make more difficult to import iotests.py.. Calling common.rc from
> bash tests will need to be modified too.
> 
> So, we'll need additional line in all python tests:
> 
> sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
> 
> which doesn't seem to be good practice.. So, instead we'd better call
> tests with PYTHONPATH set appropriately..
> 

Just chiming in to say that it's largely bad practice because it
confuses pylint, mypy and friends -- if we want to keep pushing our CI
code analysis for python in that direction, this will be a barrier.

Using PYTHONPATH is better, because it isolates the script itself from
the environment, but requires you to now always set PYTHONPATH to
execute any of the individual iotests.

Not actually a big deal, because iotests already expect a large number
of environment variables to be set. It's not really a huge net loss in
convenience, I think.

looks like that's the direction you're headed in anyway based on
discussion, so that's good.

--js

[PATCH v2] configure: prefer sphinx-build to sphinx-build-3

[John Snow]

sphinx-build is the name of the script entry point from the sphinx
package itself. sphinx-build-3 is a pacakging convention in the Fedora
distribution of Sphinx. Prefer, where possible, the canonical package
name.

In the event that this resolves to a python2 version, test the
suitability of the binary early in the configuration process, and
continue looking for sphinx-build-3 if necessary.

This prioritizes a virtual environment version of sphinx above any
distribution versions, if attempting to build out of a virtual python
environment, which can ease dependency difficulties on older
distributions, as well as allowing easy testing of specific sphinx
versions.

Signed-off-by: John Snow 
---
 configure | 66 +--
 1 file changed, 40 insertions(+), 26 deletions(-)

diff --git a/configure b/configure
index 0d69c360c0..cda4b022e7 100755
--- a/configure
+++ b/configure
@@ -929,13 +929,42 @@ do
 fi
 done
 
+# If we're making warnings fatal, apply this to Sphinx runs as well
+sphinx_werror=""
+if test "$werror" = "yes"; then
+sphinx_werror="-W"
+fi
+
+# Check we have a new enough version of sphinx-build
+test_sphinx_build() {
+sphinx=$1
+# This is a bit awkward but works: create a trivial document and
+# try to run it with our configuration file (which enforces a
+# version requirement). This will fail if either
+# sphinx-build doesn't exist at all or if it is too old.
+mkdir -p "$TMPDIR1/sphinx"
+touch "$TMPDIR1/sphinx/index.rst"
+"$sphinx" $sphinx_werror -c "$source_path/docs" \
+  -b html "$TMPDIR1/sphinx" \
+  "$TMPDIR1/sphinx/out" >> config.log 2>&1
+}
+
+# We require the python3 version of sphinx, but sphinx-build-3 is a
+# distro package. prefer 'sphinx-build' to find the venv version, if
+# any, but ensure it is a suitable version.
 sphinx_build=
-for binary in sphinx-build-3 sphinx-build
+sphinx_ok=
+for binary in sphinx-build sphinx-build-3
 do
 if has "$binary"
 then
-sphinx_build=$(command -v "$binary")
-break
+sphinx_candidate=$(command -v "$binary")
+if test_sphinx_build "$sphinx_candidate"
+then
+sphinx_build=$sphinx_candidate
+sphinx_ok=yes
+break
+fi
 fi
 done
 
@@ -4929,32 +4958,17 @@ if check_include sys/kcov.h ; then
 kcov=yes
 fi
 
-# If we're making warnings fatal, apply this to Sphinx runs as well
-sphinx_werror=""
-if test "$werror" = "yes"; then
-sphinx_werror="-W"
-fi
-
-# Check we have a new enough version of sphinx-build
-has_sphinx_build() {
-# This is a bit awkward but works: create a trivial document and
-# try to run it with our configuration file (which enforces a
-# version requirement). This will fail if either
-# sphinx-build doesn't exist at all or if it is too old.
-mkdir -p "$TMPDIR1/sphinx"
-touch "$TMPDIR1/sphinx/index.rst"
-"$sphinx_build" $sphinx_werror -c "$source_path/docs" \
--b html "$TMPDIR1/sphinx" \
-"$TMPDIR1/sphinx/out"  >> config.log 2>&1
-}
-
 # Check if tools are available to build documentation.
 if test "$docs" != "no" ; then
-  if has_sphinx_build; then
-sphinx_ok=yes
-  else
-sphinx_ok=no
+
+  if [ "$sphinx_ok" != "yes" ]; then
+if test_sphinx_build "$sphinx_build"; then
+  sphinx_ok=yes
+else
+  sphinx_ok=no
+fi
   fi
+
   if has makeinfo && has pod2man && test "$sphinx_ok" = "yes"; then
 docs=yes
   else
-- 
2.21.1

[PATCH RFC WIP 0/6] iotests: delinting work-in-progress

[John Snow]

I ran out of time, but I was briefly entertaining the idea of hitting
everything else in the iotests folder with the pylint and mypy beam.

This is just a draft of what I had at the time, in case someone gets
around to it before I do.

I forced all of the python scripts in this directory to Python3 and pass
the formatter check imposed by the pylintrc that we make iotests.py
adhere to, but there's no guarantee these scripts actually work. I was a
little rough around the edges.

qed.py and qcow2.py in particular abuse some python features that pylint
really doesn't like to see at all, and forcing these to pass a pylint
check was clearly fighting against the idioms in-use in those files,
with more than a few unsatisfactory hacks employed just to make pylint
shut up. They are not necessarily improvements to those scripts.

I created a Makefile check to hit the iotests themselves with the pylint
checker so we can identify when refactors to the testing infrastructure
invalidate tests we don't get to run that often; but didn't get around
to improving the quality of the tests themselves.

(Whether or not this is even worth our time is debatable, it depends on
how much effort it would take to bring them up to par. Maybe it's a lot,
I don't know.)

--js

John Snow (6):
  iotests: type hint wip
  Makefile: add delint WIP
  nbd-fault-injector: delint
  qed.py: delint
  qcow2.py: delint
  WIP: delint test files

 tests/qemu-iotests/Makefile  |   6 +
 tests/qemu-iotests/iotests.py|  28 ++--
 tests/qemu-iotests/nbd-fault-injector.py |  34 +++--
 tests/qemu-iotests/pylintrc  |   1 +
 tests/qemu-iotests/qcow2.py  | 156 +++
 tests/qemu-iotests/qed.py|  46 ---
 6 files changed, 173 insertions(+), 98 deletions(-)

-- 
2.21.1

[PATCH RFC WIP 6/6] WIP: delint test files

[John Snow]

Signed-off-by: John Snow 
---
 tests/qemu-iotests/Makefile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/qemu-iotests/Makefile b/tests/qemu-iotests/Makefile
index 77da9fd96d..fd85eb5bae 100644
--- a/tests/qemu-iotests/Makefile
+++ b/tests/qemu-iotests/Makefile
@@ -1,4 +1,6 @@
 PYMODULES = $(wildcard *.py)
+PYTESTS := $(shell grep -rl '^\#!/usr/bin/env python3' *)
+PYTESTS := $(filter-out $(PYMODULES), $(PYTESTS))
 
 CLEANFILES= *.out.bad *.notrun check.log check.time*
 
@@ -10,3 +12,4 @@ clean:
 
 delint:
pylint $(PYMODULES)
+   pylint --disable=R,C,W $(PYTESTS)
-- 
2.21.1

[PATCH RFC WIP 4/6] qed.py: delint

[John Snow]

Signed-off-by: John Snow 
---
 tests/qemu-iotests/Makefile |  2 +-
 tests/qemu-iotests/qed.py   | 46 -
 2 files changed, 31 insertions(+), 17 deletions(-)

diff --git a/tests/qemu-iotests/Makefile b/tests/qemu-iotests/Makefile
index 64db48342f..5a3a1e8092 100644
--- a/tests/qemu-iotests/Makefile
+++ b/tests/qemu-iotests/Makefile
@@ -1,6 +1,6 @@
 PYMODULES = $(wildcard *.py)
 
-KNOWN_GOOD = iotests.py nbd-fault-injector.py
+KNOWN_GOOD = iotests.py nbd-fault-injector.py qed.py
 
 CLEANFILES= *.out.bad *.notrun check.log check.time*
 
diff --git a/tests/qemu-iotests/qed.py b/tests/qemu-iotests/qed.py
index d6bec96069..9e016cc2a2 100755
--- a/tests/qemu-iotests/qed.py
+++ b/tests/qemu-iotests/qed.py
@@ -13,7 +13,6 @@
 import sys
 import struct
 import random
-import optparse
 
 # This can be used as a module
 __all__ = ['QED_F_NEED_CHECK', 'QED']
@@ -47,7 +46,7 @@ def unpack_table_elem(s):
 def pack_table_elem(elem):
 return struct.pack(table_elem_fmt, elem)
 
-class QED(object):
+class QED:
 def __init__(self, f):
 self.f = f
 
@@ -74,19 +73,22 @@ def store_header(self):
 def read_table(self, offset):
 size = self.header['table_size'] * self.header['cluster_size']
 s = self.raw_pread(offset, size)
-table = [unpack_table_elem(s[i:i + table_elem_size]) for i in 
xrange(0, size, table_elem_size)]
+table = [unpack_table_elem(s[i:i + table_elem_size])
+ for i in range(0, size, table_elem_size)]
 return table
 
 def load_l1_table(self):
 self.l1_table = self.read_table(self.header['l1_table_offset'])
-self.table_nelems = self.header['table_size'] * 
self.header['cluster_size'] // table_elem_size
+self.table_nelems = (self.header['table_size']
+ * self.header['cluster_size'] // table_elem_size)
 
 def write_table(self, offset, table):
 s = ''.join(pack_table_elem(x) for x in table)
 self.raw_pwrite(offset, s)
 
 def random_table_item(table):
-vals = [(index, offset) for index, offset in enumerate(table) if offset != 
0]
+vals = [(index, offset) for index, offset
+in enumerate(table) if offset != 0]
 if not vals:
 err('cannot pick random item because table is empty')
 return random.choice(vals)
@@ -103,7 +105,8 @@ def corrupt_table_duplicate(table):
 def corrupt_table_invalidate(qed, table):
 '''Corrupt a table by introducing an invalid offset'''
 index, _ = random_table_item(table)
-table[index] = qed.filesize + random.randint(0, 100 * 1024 * 1024 * 1024 * 
1024)
+table[index] = (qed.filesize
++ random.randint(0, 100 * 1024 * 1024 * 1024 * 1024))
 
 def cmd_show(qed, *args):
 '''show [header|l1|l2 ]- Show header or l1/l2 tables'''
@@ -144,7 +147,11 @@ def cmd_invalidate(qed, table_level):
 qed.write_table(offset, table)
 
 def cmd_need_check(qed, *args):
-'''need-check [on|off] - Test, set, or clear the QED_F_NEED_CHECK header 
bit'''
+"""
+need-check [on|off]
+
+Test, set, or clear the QED_F_NEED_CHECK header bit
+"""
 if not args:
 print(bool(qed.header['features'] & QED_F_NEED_CHECK))
 return
@@ -165,7 +172,7 @@ def cmd_zero_cluster(qed, pos, *args):
 err('expected one argument')
 n = int(args[0])
 
-for i in xrange(n):
+for _ in range(n):
 l1_index = pos // qed.header['cluster_size'] // len(qed.l1_table)
 if qed.l1_table[l1_index] == 0:
 err('no l2 table allocated')
@@ -179,7 +186,11 @@ def cmd_zero_cluster(qed, pos, *args):
 pos += qed.header['cluster_size']
 
 def cmd_copy_metadata(qed, outfile):
-'''copy-metadata  - Copy metadata only (for scrubbing corrupted 
images)'''
+"""
+copy-metadata 
+
+Copy metadata only (for scrubbing corrupted images)
+"""
 out = open(outfile, 'wb')
 
 # Match file size
@@ -213,23 +224,26 @@ def usage():
 print('Supported commands:')
 for cmd in sorted(x for x in globals() if x.startswith('cmd_')):
 print(globals()[cmd].__doc__)
-sys.exit(1)
+return 1
 
 def main():
 if len(sys.argv) < 3:
-usage()
-filename, cmd = sys.argv[1:3]
+return usage()
+filename = sys.argv[1]
+cmd = sys.argv[2]
 
 cmd = 'cmd_' + cmd.replace('-', '_')
 if cmd not in globals():
-usage()
+return usage()
 
 qed = QED(open(filename, 'r+b'))
 try:
 globals()[cmd](qed, *sys.argv[3:])
-except TypeError as e:
+except TypeError:
 sys.stderr.write(globals()[cmd].__doc__ + '\n')
-sys.exit(1)
+return 1
+
+return 0
 
 if __name__ == '__main__':
-main()
+sys.exit(main())
-- 
2.21.1

[PATCH RFC WIP 5/6] qcow2.py: delint

[John Snow]

Signed-off-by: John Snow 
---
 tests/qemu-iotests/Makefile |   5 +-
 tests/qemu-iotests/pylintrc |   1 +
 tests/qemu-iotests/qcow2.py | 156 +++-
 3 files changed, 104 insertions(+), 58 deletions(-)

diff --git a/tests/qemu-iotests/Makefile b/tests/qemu-iotests/Makefile
index 5a3a1e8092..77da9fd96d 100644
--- a/tests/qemu-iotests/Makefile
+++ b/tests/qemu-iotests/Makefile
@@ -1,7 +1,5 @@
 PYMODULES = $(wildcard *.py)
 
-KNOWN_GOOD = iotests.py nbd-fault-injector.py qed.py
-
 CLEANFILES= *.out.bad *.notrun check.log check.time*
 
 # no default target
@@ -11,5 +9,4 @@ clean:
rm -f $(CLEANFILES)
 
 delint:
-   pylint $(KNOWN_GOOD)
-   pylint --disable=R,C,W $(filter-out $(KNOWN_GOOD), $(PYMODULES))
+   pylint $(PYMODULES)
diff --git a/tests/qemu-iotests/pylintrc b/tests/qemu-iotests/pylintrc
index 5481afe528..a334e242b6 100644
--- a/tests/qemu-iotests/pylintrc
+++ b/tests/qemu-iotests/pylintrc
@@ -17,6 +17,7 @@ disable=invalid-name,
 too-many-lines,
 too-many-locals,
 too-many-public-methods,
+too-many-instance-attributes,
 # These are temporary, and should be removed:
 missing-docstring,
 
diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py
index 94a07b2f6f..2840f4b661 100755
--- a/tests/qemu-iotests/qcow2.py
+++ b/tests/qemu-iotests/qcow2.py
@@ -11,14 +11,15 @@ def __init__(self, magic, length, data):
 padding = 8 - (length % 8)
 data += b"\0" * padding
 
-self.magic  = magic
+self.magic = magic
 self.length = length
-self.data   = data
+self.data = data
 
 @classmethod
 def create(cls, magic, data):
 return QcowHeaderExtension(magic, len(data), data)
 
+
 class QcowHeader:
 
 uint32_t = 'I'
@@ -26,30 +27,41 @@ class QcowHeader:
 
 fields = [
 # Version 2 header fields
-[ uint32_t, '%#x',  'magic' ],
-[ uint32_t, '%d',   'version' ],
-[ uint64_t, '%#x',  'backing_file_offset' ],
-[ uint32_t, '%#x',  'backing_file_size' ],
-[ uint32_t, '%d',   'cluster_bits' ],
-[ uint64_t, '%d',   'size' ],
-[ uint32_t, '%d',   'crypt_method' ],
-[ uint32_t, '%d',   'l1_size' ],
-[ uint64_t, '%#x',  'l1_table_offset' ],
-[ uint64_t, '%#x',  'refcount_table_offset' ],
-[ uint32_t, '%d',   'refcount_table_clusters' ],
-[ uint32_t, '%d',   'nb_snapshots' ],
-[ uint64_t, '%#x',  'snapshot_offset' ],
+# pylint: disable=bad-whitespace
+[uint32_t, '%#x',  'magic'],
+[uint32_t, '%d',   'version'],
+[uint64_t, '%#x',  'backing_file_offset'],
+[uint32_t, '%#x',  'backing_file_size'],
+[uint32_t, '%d',   'cluster_bits'],
+[uint64_t, '%d',   'size'],
+[uint32_t, '%d',   'crypt_method'],
+[uint32_t, '%d',   'l1_size'],
+[uint64_t, '%#x',  'l1_table_offset'],
+[uint64_t, '%#x',  'refcount_table_offset'],
+[uint32_t, '%d',   'refcount_table_clusters'],
+[uint32_t, '%d',   'nb_snapshots'],
+[uint64_t, '%#x',  'snapshot_offset'],
 
 # Version 3 header fields
-[ uint64_t, 'mask', 'incompatible_features' ],
-[ uint64_t, 'mask', 'compatible_features' ],
-[ uint64_t, 'mask', 'autoclear_features' ],
-[ uint32_t, '%d',   'refcount_order' ],
-[ uint32_t, '%d',   'header_length' ],
-];
+[uint64_t, 'mask', 'incompatible_features'],
+[uint64_t, 'mask', 'compatible_features'],
+[uint64_t, 'mask', 'autoclear_features'],
+[uint32_t, '%d',   'refcount_order'],
+[uint32_t, '%d',   'header_length'],
+]
 
 fmt = '>' + ''.join(field[0] for field in fields)
 
+@property
+def backing_file_offset(self):
+# Pylint struggles to verify dynamic properties.
+# Dataclasses in 3.7 would make this easy.
+return self.__dict__['backing_file_offset']
+
+@backing_file_offset.setter
+def backing_file_offset(self, val):
+self.__dict__['backing_file_offset'] = val
+
 def __init__(self, fd):
 
 buf_size = struct.calcsize(QcowHeader.fmt)
@@ -59,22 +71,22 @@ def __init__(self, fd):
 
 header = struct.unpack(QcowHeader.fmt, buf)
 self.__dict__ = dict((field[2], header[i])
-for i, field in enumerate(QcowHeader.fields))
+ for i, field in enumerate(QcowHeader.fields))
 
 self.set_defaults()
-self.cluster_size = 1 << self.cluster_bits
+self.cluster_size = 1 << self.__dict__['cluster_bits']
 
 fd.seek(self.header_length)
 self.load_extensions(fd)
 
 if self.backing_file_offset:
 fd.seek(self.backing_file_offset)
-self.backing_file = fd.read(self.backing_file_size)
+self.backing_file = fd.read(self.__dict__['backing_file_size'])
 else:
 

[PATCH RFC WIP 1/6] iotests: type hint wip

[John Snow]

Signed-off-by: John Snow 
---
 tests/qemu-iotests/iotests.py | 28 ++--
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index 6c0e781af7..27c477c8a7 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -90,7 +90,7 @@
 luks_default_key_secret_opt = 'key-secret=keysec0'
 
 
-def qemu_img(*args):
+def qemu_img(*args) -> int:
 '''Run qemu-img and return the exit code'''
 devnull = open('/dev/null', 'r+')
 exitcode = subprocess.call(qemu_img_args + list(args),
@@ -113,24 +113,24 @@ def ordered_qmp(qmsg, conv_keys=True):
 return od
 return qmsg
 
-def qemu_img_create(*args):
-args = list(args)
+def qemu_img_create(*args: str) -> int:
+qargs = list(args)
 
 # default luks support
-if '-f' in args and args[args.index('-f') + 1] == 'luks':
-if '-o' in args:
-i = args.index('-o')
-if 'key-secret' not in args[i + 1]:
-args[i + 1].append(luks_default_key_secret_opt)
-args.insert(i + 2, '--object')
-args.insert(i + 3, luks_default_secret_object)
+if '-f' in qargs and qargs[qargs.index('-f') + 1] == 'luks':
+if '-o' in qargs:
+i = qargs.index('-o')
+if 'key-secret' not in qargs[i + 1]:
+qargs[i + 1].append(luks_default_key_secret_opt)
+qargs.insert(i + 2, '--object')
+qargs.insert(i + 3, luks_default_secret_object)
 else:
-args = ['-o', luks_default_key_secret_opt,
-'--object', luks_default_secret_object] + args
+qargs = ['-o', luks_default_key_secret_opt,
+ '--object', luks_default_secret_object] + qargs
 
-args.insert(0, 'create')
+qargs.insert(0, 'create')
 
-return qemu_img(*args)
+return qemu_img(*qargs)
 
 def qemu_img_verbose(*args):
 '''Run qemu-img without suppressing its output and return the exit code'''
-- 
2.21.1

[PATCH RFC WIP 3/6] nbd-fault-injector: delint

[John Snow]

Signed-off-by: John Snow 
---
 tests/qemu-iotests/Makefile  |  2 +-
 tests/qemu-iotests/nbd-fault-injector.py | 34 ++--
 2 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/tests/qemu-iotests/Makefile b/tests/qemu-iotests/Makefile
index 7dbb7f0fff..64db48342f 100644
--- a/tests/qemu-iotests/Makefile
+++ b/tests/qemu-iotests/Makefile
@@ -1,6 +1,6 @@
 PYMODULES = $(wildcard *.py)
 
-KNOWN_GOOD = iotests.py
+KNOWN_GOOD = iotests.py nbd-fault-injector.py
 
 CLEANFILES= *.out.bad *.notrun check.log check.time*
 
diff --git a/tests/qemu-iotests/nbd-fault-injector.py 
b/tests/qemu-iotests/nbd-fault-injector.py
index 588d62aebf..0de6bc643e 100755
--- a/tests/qemu-iotests/nbd-fault-injector.py
+++ b/tests/qemu-iotests/nbd-fault-injector.py
@@ -47,10 +47,7 @@
 import socket
 import struct
 import collections
-if sys.version_info.major >= 3:
-import configparser
-else:
-import ConfigParser as configparser
+import configparser
 
 FAKE_DISK_SIZE = 8 * 1024 * 1024 * 1024 # 8 GB
 
@@ -71,7 +68,9 @@
 export_tuple = collections.namedtuple('Export', 'reserved magic opt len')
 export_struct = struct.Struct('>IQII')
 neg2_struct = struct.Struct('>QH124x')
-request_tuple = collections.namedtuple('Request', 'magic type handle from_ 
len')
+request_tuple = collections.namedtuple(
+'Request', 'magic type handle from_ len'
+)
 request_struct = struct.Struct('>IIQQI')
 reply_struct = struct.Struct('>IIQ')
 
@@ -84,13 +83,13 @@ def recvall(sock, bufsize):
 chunks = []
 while received < bufsize:
 chunk = sock.recv(bufsize - received)
-if len(chunk) == 0:
+if not chunk:
 raise Exception('unexpected disconnect')
 chunks.append(chunk)
 received += len(chunk)
 return b''.join(chunks)
 
-class Rule(object):
+class Rule:
 def __init__(self, name, event, io, when):
 self.name = name
 self.event = event
@@ -104,7 +103,7 @@ def match(self, event, io):
 return False
 return True
 
-class FaultInjectionSocket(object):
+class FaultInjectionSocket:
 def __init__(self, sock, rules):
 self.sock = sock
 self.rules = rules
@@ -150,7 +149,7 @@ def negotiate_export(conn):
 export = export_tuple._make(export_struct.unpack(buf))
 assert export.magic == NBD_OPTS_MAGIC
 assert export.opt == NBD_OPT_EXPORT_NAME
-name = conn.recv(export.len, event='export-name')
+_name = conn.recv(export.len, event='export-name')
 
 # Send negotiation part 2
 buf = neg2_struct.pack(FAKE_DISK_SIZE, 0)
@@ -200,7 +199,8 @@ def parse_inject_error(name, options):
 if 'event' not in options:
 err('missing \"event\" option in %s' % name)
 event = options['event']
-if event not in ('neg-classic', 'neg1', 'export', 'neg2', 'request', 
'reply', 'data'):
+if event not in ('neg-classic', 'neg1', 'export',
+ 'neg2', 'request', 'reply', 'data'):
 err('invalid \"event\" option value \"%s\" in %s' % (event, name))
 io = options.get('io', 'readwrite')
 if io not in ('read', 'write', 'readwrite'):
@@ -229,8 +229,8 @@ def parse_config(config):
 
 def load_rules(filename):
 config = configparser.RawConfigParser()
-with open(filename, 'rt') as f:
-config.readfp(f, filename)
+with open(filename, 'rt') as infile:
+config.read_file(infile)
 return parse_config(config)
 
 def open_socket(path):
@@ -252,8 +252,14 @@ def open_socket(path):
 return sock
 
 def usage(args):
-sys.stderr.write('usage: %s [--classic-negotiation] | 
\n' % args[0])
-sys.stderr.write('Run an fault injector NBD server with rules defined in a 
config file.\n')
+name = args[0]
+sys.stderr.write(
+f'usage: {name} [--classic-negotiation] '
+'| \n'
+)
+sys.stderr.write(
+'Run a fault injector NBD server with '
+'rules defined in a config file.\n')
 sys.exit(1)
 
 def main(args):
-- 
2.21.1

[PATCH RFC WIP 2/6] Makefile: add delint WIP

[John Snow]

Signed-off-by: John Snow 
---
 tests/qemu-iotests/Makefile | 6 ++
 1 file changed, 6 insertions(+)

diff --git a/tests/qemu-iotests/Makefile b/tests/qemu-iotests/Makefile
index 27380e60c1..7dbb7f0fff 100644
--- a/tests/qemu-iotests/Makefile
+++ b/tests/qemu-iotests/Makefile
@@ -1,3 +1,6 @@
+PYMODULES = $(wildcard *.py)
+
+KNOWN_GOOD = iotests.py
 
 CLEANFILES= *.out.bad *.notrun check.log check.time*
 
@@ -7,3 +10,6 @@ default:
 clean:
rm -f $(CLEANFILES)
 
+delint:
+   pylint $(KNOWN_GOOD)
+   pylint --disable=R,C,W $(filter-out $(KNOWN_GOOD), $(PYMODULES))
-- 
2.21.1

Re: [PATCH v4 34/34] block: Drop @child_class from bdrv_child_perm()

[Eric Blake]

On 5/13/20 6:05 AM, Max Reitz wrote:

Implementations should decide the necessary permissions based on @role.

Signed-off-by: Max Reitz 
---



+++ b/block.c
@@ -1947,13 +1947,13 @@ bool bdrv_is_writable(BlockDriverState *bs)
  }
  
  static void bdrv_child_perm(BlockDriverState *bs, BlockDriverState *child_bs,

-BdrvChild *c, const BdrvChildClass *child_class,
-BdrvChildRole role, BlockReopenQueue *reopen_queue,
+BdrvChild *c, BdrvChildRole role,
+BlockReopenQueue *reopen_queue,
  uint64_t parent_perm, uint64_t parent_shared,
  uint64_t *nperm, uint64_t *nshared)
  {
  assert(bs->drv && bs->drv->bdrv_child_perm);
-bs->drv->bdrv_child_perm(bs, c, child_class, role, reopen_queue,
+bs->drv->bdrv_child_perm(bs, c, role, reopen_queue,
   parent_perm, parent_shared,
   nperm, nshared);


Is it worth reflowing this call into two lines rather than three?  But 
that's cosmetic.


Reviewed-by: Eric Blake 

--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3226
Virtualization:  qemu.org | libvirt.org

Re: [PATCH v4 33/34] block: Pass BdrvChildRole in remaining cases

[Eric Blake]

On 5/13/20 6:05 AM, Max Reitz wrote:

These calls have no real use for the child role yet, but it will not
harm to give one.

Notably, the bdrv_root_attach_child() call in blockjob.c is left
unmodified because there is not much the generic BlockJob object wants
from its children.

Signed-off-by: Max Reitz 
---
  block/block-backend.c | 11 +++
  block/vvfat.c |  4 +++-
  2 files changed, 10 insertions(+), 5 deletions(-)



Reviewed-by: Eric Blake 

--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3226
Virtualization:  qemu.org | libvirt.org

Re: [PATCH 4/4] scripts/qmp: Fix QEMU Python scripts path

[John Snow]

On 4/30/20 1:04 AM, Markus Armbruster wrote:
> John Snow  writes:
> 
>> On 4/21/20 5:42 AM, Philippe Mathieu-Daudé wrote:
>>> QEMU Python scripts have been moved in commit 8f8fd9edba4 ("Introduce
>>> Python module structure"). Use the same sys.path modification used
>>> in the referenced commit to be able to use these scripts again.
>>>
>>> Signed-off-by: Philippe Mathieu-Daudé 
>>> ---
>>>  scripts/qmp/qmp  | 4 +++-
>>>  scripts/qmp/qom-fuse | 4 +++-
>>>  scripts/qmp/qom-get  | 4 +++-
>>>  scripts/qmp/qom-list | 4 +++-
>>>  scripts/qmp/qom-set  | 4 +++-
>>>  scripts/qmp/qom-tree | 4 +++-
>>>  6 files changed, 18 insertions(+), 6 deletions(-)
>>>
>>> diff --git a/scripts/qmp/qmp b/scripts/qmp/qmp
>>> index 0625fc2aba..8e52e4a54d 100755
>>> --- a/scripts/qmp/qmp
>>> +++ b/scripts/qmp/qmp
>>> @@ -11,7 +11,9 @@
>>>  # See the COPYING file in the top-level directory.
>>>  
>>>  import sys, os
>>> -from qmp import QEMUMonitorProtocol
>>> +
>>> +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 
>>> 'python'))
>>> +from qemu.qmp import QEMUMonitorProtocol
>>>  
>>
>> Try to avoid using sys.path hacks; they don't work in pylint or mypy and
>> it provides an active barrier to CQA work here.
>> (They also tend to be quite fragile.)
>>
>> We can discuss the right way to do this; one of those ways is to create
>> an installable package that we can install locally in a virtual environment.
>>
>> Another way is perhaps to set PYTHONPATH in the calling environment so
>> that standard "import" directives will work.
>>
>> Both ultimately involve changing the environment of the user to
>> accommodate the script.
> 
> For what it's worth, tests/Makefile.involve does the latter for
> tests/qapi-schema/test-qapi.py.  Simple enough, but makes manual
> invocation inconvenient.
> 
> Not necessary for scripts/qapi-gen.py, because its "import qmp.FOO"
> finds qmp right in scripts/qmp/.
> 

Yeah, I should be clear here: this is actually kind of a hard thing to
fix tree-wide right now.

What I didn't realize when I was reviewing this patch is that these
imports are *already broken* and these sys.path hacks actually make it
*work again*.

Under that premise, I rescind my feedback, and offer instead:

Reviewed-by: John Snow 

But we should look into how to fix the sys.path problems more long-term;
as part of the packaging work we should investigate this. I believe
similar problems are cropping up in iotests.

Sorry for the noise.

--js

Re: [PATCH v4 32/34] block: Drop child_file

[Eric Blake]

On 5/13/20 6:05 AM, Max Reitz wrote:

Signed-off-by: Max Reitz 
---
  include/block/block_int.h |  1 -
  block.c   | 39 ++-
  tests/test-bdrv-drain.c   |  8 +++-
  3 files changed, 5 insertions(+), 43 deletions(-)



Reviewed-by: Eric Blake 

--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3226
Virtualization:  qemu.org | libvirt.org

Re: [PATCH v4 27/34] block: Use child_of_bds in remaining places

[Eric Blake]

On 5/13/20 6:05 AM, Max Reitz wrote:

Replace child_file by child_of_bds in all remaining places (excluding
tests).

Signed-off-by: Max Reitz 
---
  block.c  |  3 ++-
  block/backup-top.c   |  4 ++--
  block/blklogwrites.c |  4 ++--
  block/raw-format.c   | 15 +--
  4 files changed, 19 insertions(+), 7 deletions(-)




@@ -448,8 +449,18 @@ static int raw_open(BlockDriverState *bs, QDict *options, 
int flags,
  return ret;
  }
  
-bs->file = bdrv_open_child(NULL, options, "file", bs, _file, 0,

-   false, errp);
+/*
+ * Without offset and a size limit, this driver behaves very much
+ * like a filter.  With any such limit, it does not.
+ */
+if (offset || has_size) {
+file_role = BDRV_CHILD_DATA | BDRV_CHILD_PRIMARY;
+} else {
+file_role = BDRV_CHILD_FILTERED | BDRV_CHILD_PRIMARY;
+}


I wonder if we'll hit subtle bugs where one but not the other use of raw 
breaks because of the difference in roles.  I know we have _some_ iotest 
coverage of raw with offsets, but wonder if we might need more (or a way 
to add it on top of existing images, similar to how we can easily toggle 
qcow2 options like refcount size for interesting non-default test runs).



+
+bs->file = bdrv_open_child(NULL, options, "file", bs, _of_bds,
+   file_role, false, errp);
  if (!bs->file) {
  return -EINVAL;
  }



At any rate, if we have corner-case bugs, I don't think this is making 
them worse.

Reviewed-by: Eric Blake 

--
Eric Blake, Principal Software Engineer
Red Hat, Inc.   +1-919-301-3226
Virtualization:  qemu.org | libvirt.org