Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package qemu for openSUSE:Factory checked in
at 2022-04-08 00:27:13
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/qemu (Old)
and /work/SRC/openSUSE:Factory/.qemu.new.1900 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "qemu"
Fri Apr 8 00:27:13 2022 rev:225 rq:967219 version:unknown
Changes:
--------
--- /work/SRC/openSUSE:Factory/qemu/qemu.changes 2022-04-02
18:20:53.378039995 +0200
+++ /work/SRC/openSUSE:Factory/.qemu.new.1900/qemu.changes 2022-04-08
00:27:39.682360915 +0200
@@ -1,0 +2,44 @@
+Tue Apr 5 08:54:51 UTC 2022 - Li Zhang <li.zh...@suse.com>
+
+- Backport aqmp patches from upstream which can fix iotest issues
+* Patches added:
+ python-aqmp-add-__del__-method-to-legacy.patch
+ python-aqmp-add-_session_guard.patch
+ python-aqmp-add-SocketAddrT-to-package-r.patch
+ python-aqmp-add-socket-bind-step-to-lega.patch
+ python-aqmp-add-start_server-and-accept-.patch
+ python-aqmp-copy-type-definitions-from-q.patch
+ python-aqmp-drop-_bind_hack.patch
+ python-aqmp-fix-docstring-typo.patch
+ python-aqmp-Fix-negotiation-with-pre-oob.patch
+ python-aqmp-fix-race-condition-in-legacy.patch
+ Python-aqmp-fix-type-definitions-for-myp.patch
+ python-aqmp-handle-asyncio.TimeoutError-.patch
+ python-aqmp-refactor-_do_accept-into-two.patch
+ python-aqmp-remove-_new_session-and-_est.patch
+ python-aqmp-rename-accept-to-start_serve.patch
+ python-aqmp-rename-AQMPError-to-QMPError.patch
+ python-aqmp-split-_client_connected_cb-o.patch
+ python-aqmp-squelch-pylint-warning-for-t.patch
+ python-aqmp-stop-the-server-during-disco.patch
+ python-introduce-qmp-shell-wrap-convenie.patch
+ python-machine-raise-VMLaunchFailure-exc.patch
+ python-move-qmp-shell-under-the-AQMP-pac.patch
+ python-move-qmp-utilities-to-python-qemu.patch
+ python-qmp-switch-qmp-shell-to-AQMP.patch
+ python-support-recording-QMP-session-to-.patch
+ python-upgrade-mypy-to-0.780.patch
+
+-------------------------------------------------------------------
+Tue Apr 5 08:24:58 UTC 2022 - Li Zhang <li.zh...@suse.com>
+
+- Drop the patches which are workaround to fix iotest issues
+* Patches dropped:
+ Revert-python-iotests-replace-qmp-with-a.patch
+ Revert-python-machine-add-instance-disam.patch
+ Revert-python-machine-add-sock_dir-prope.patch
+ Revert-python-machine-handle-fast-QEMU-t.patch
+ Revert-python-machine-move-more-variable.patch
+ Revert-python-machine-remove-_remove_mon.patch
+
+-------------------------------------------------------------------
Old:
----
Revert-python-iotests-replace-qmp-with-a.patch
Revert-python-machine-add-instance-disam.patch
Revert-python-machine-add-sock_dir-prope.patch
Revert-python-machine-handle-fast-QEMU-t.patch
Revert-python-machine-move-more-variable.patch
Revert-python-machine-remove-_remove_mon.patch
New:
----
Python-aqmp-fix-type-definitions-for-myp.patch
python-aqmp-Fix-negotiation-with-pre-oob.patch
python-aqmp-add-SocketAddrT-to-package-r.patch
python-aqmp-add-__del__-method-to-legacy.patch
python-aqmp-add-_session_guard.patch
python-aqmp-add-socket-bind-step-to-lega.patch
python-aqmp-add-start_server-and-accept-.patch
python-aqmp-copy-type-definitions-from-q.patch
python-aqmp-drop-_bind_hack.patch
python-aqmp-fix-docstring-typo.patch
python-aqmp-fix-race-condition-in-legacy.patch
python-aqmp-handle-asyncio.TimeoutError-.patch
python-aqmp-refactor-_do_accept-into-two.patch
python-aqmp-remove-_new_session-and-_est.patch
python-aqmp-rename-AQMPError-to-QMPError.patch
python-aqmp-rename-accept-to-start_serve.patch
python-aqmp-split-_client_connected_cb-o.patch
python-aqmp-squelch-pylint-warning-for-t.patch
python-aqmp-stop-the-server-during-disco.patch
python-introduce-qmp-shell-wrap-convenie.patch
python-machine-raise-VMLaunchFailure-exc.patch
python-move-qmp-shell-under-the-AQMP-pac.patch
python-move-qmp-utilities-to-python-qemu.patch
python-qmp-switch-qmp-shell-to-AQMP.patch
python-support-recording-QMP-session-to-.patch
python-upgrade-mypy-to-0.780.patch
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ qemu.spec ++++++
--- /var/tmp/diff_new_pack.MdzzZM/_old 2022-04-08 00:27:42.730326706 +0200
+++ /var/tmp/diff_new_pack.MdzzZM/_new 2022-04-08 00:27:42.730326706 +0200
@@ -220,16 +220,36 @@
Patch00074: tests-qemu-iotests-testrunner-Quote-case.patch
Patch00075: Fix-the-module-building-problem-for-s390.patch
Patch00076: scsi-generic-check-for-additional-SG_IO-.patch
-Patch00077: Revert-python-machine-handle-fast-QEMU-t.patch
-Patch00078: Revert-python-machine-move-more-variable.patch
-Patch00079: Revert-python-machine-add-instance-disam.patch
-Patch00080: Revert-python-machine-remove-_remove_mon.patch
-Patch00081: Revert-python-machine-add-sock_dir-prope.patch
-Patch00082: Revert-python-iotests-replace-qmp-with-a.patch
-Patch00083: hw-nvme-fix-CVE-2021-3929.patch
-Patch00084: numa-Enable-numa-for-SGX-EPC-sections.patch
-Patch00085: numa-Support-SGX-numa-in-the-monitor-and.patch
-Patch00086: doc-Add-the-SGX-numa-description.patch
+Patch00077: hw-nvme-fix-CVE-2021-3929.patch
+Patch00078: numa-Enable-numa-for-SGX-EPC-sections.patch
+Patch00079: numa-Support-SGX-numa-in-the-monitor-and.patch
+Patch00080: doc-Add-the-SGX-numa-description.patch
+Patch00081: python-aqmp-Fix-negotiation-with-pre-oob.patch
+Patch00082: python-machine-raise-VMLaunchFailure-exc.patch
+Patch00083: python-upgrade-mypy-to-0.780.patch
+Patch00084: Python-aqmp-fix-type-definitions-for-myp.patch
+Patch00085: python-aqmp-add-__del__-method-to-legacy.patch
+Patch00086: python-aqmp-copy-type-definitions-from-q.patch
+Patch00087: python-aqmp-handle-asyncio.TimeoutError-.patch
+Patch00088: python-aqmp-add-SocketAddrT-to-package-r.patch
+Patch00089: python-aqmp-fix-docstring-typo.patch
+Patch00090: python-aqmp-rename-AQMPError-to-QMPError.patch
+Patch00091: python-qmp-switch-qmp-shell-to-AQMP.patch
+Patch00092: python-aqmp-add-socket-bind-step-to-lega.patch
+Patch00093: python-move-qmp-utilities-to-python-qemu.patch
+Patch00094: python-move-qmp-shell-under-the-AQMP-pac.patch
+Patch00095: python-introduce-qmp-shell-wrap-convenie.patch
+Patch00096: python-support-recording-QMP-session-to-.patch
+Patch00097: python-aqmp-add-_session_guard.patch
+Patch00098: python-aqmp-rename-accept-to-start_serve.patch
+Patch00099: python-aqmp-remove-_new_session-and-_est.patch
+Patch00100: python-aqmp-split-_client_connected_cb-o.patch
+Patch00101: python-aqmp-squelch-pylint-warning-for-t.patch
+Patch00102: python-aqmp-refactor-_do_accept-into-two.patch
+Patch00103: python-aqmp-stop-the-server-during-disco.patch
+Patch00104: python-aqmp-add-start_server-and-accept-.patch
+Patch00105: python-aqmp-fix-race-condition-in-legacy.patch
+Patch00106: python-aqmp-drop-_bind_hack.patch
# Patches applied in roms/seabios/:
Patch01000: seabios-use-python2-explicitly-as-needed.patch
Patch01001: seabios-switch-to-python3-as-needed.patch
@@ -1239,6 +1259,26 @@
%patch00084 -p1
%patch00085 -p1
%patch00086 -p1
+%patch00087 -p1
+%patch00088 -p1
+%patch00089 -p1
+%patch00090 -p1
+%patch00091 -p1
+%patch00092 -p1
+%patch00093 -p1
+%patch00094 -p1
+%patch00095 -p1
+%patch00096 -p1
+%patch00097 -p1
+%patch00098 -p1
+%patch00099 -p1
+%patch00100 -p1
+%patch00101 -p1
+%patch00102 -p1
+%patch00103 -p1
+%patch00104 -p1
+%patch00105 -p1
+%patch00106 -p1
%patch01000 -p1
%patch01001 -p1
%patch01002 -p1
++++++ Python-aqmp-fix-type-definitions-for-myp.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 10 Jan 2022 14:13:48 -0500
Subject: Python/aqmp: fix type definitions for mypy 0.920
Git-commit: 42d73f2894ea1855df5a25d58e0d9eac6023dcc3
0.920 (Released 2021-12-15) is not entirely happy with the
way that I was defining _FutureT:
qemu/aqmp/protocol.py:601: error: Item "object" of the upper bound
"Optional[Future[Any]]" of type variable "_FutureT" has no attribute
"done"
Update it with something a little mechanically simpler that works better
across a wider array of mypy versions.
Signed-off-by: John Snow <js...@redhat.com>
Message-id: 20220110191349.1841027-3-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/protocol.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index 5190b33b13df24fc2ca4aed934ed..c4fbe35a0e41c589059ec4fa37a8 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -43,8 +43,8 @@ from .util import (
T = TypeVar('T')
+_U = TypeVar('_U')
_TaskFN = Callable[[], Awaitable[None]] # aka ``async def func() -> None``
-_FutureT = TypeVar('_FutureT', bound=Optional['asyncio.Future[Any]'])
class Runstate(Enum):
@@ -591,7 +591,8 @@ class AsyncProtocol(Generic[T]):
"""
Fully reset this object to a clean state and return to `IDLE`.
"""
- def _paranoid_task_erase(task: _FutureT) -> Optional[_FutureT]:
+ def _paranoid_task_erase(task: Optional['asyncio.Future[_U]']
+ ) -> Optional['asyncio.Future[_U]']:
# Help to erase a task, ENSURING it is fully quiesced first.
assert (task is None) or task.done()
return None if (task and task.done()) else task
++++++ bundles.tar.xz ++++++
Binary files old/44f28df24767cf9dca1ddc9b23157737c4cbb645.bundle and
new/44f28df24767cf9dca1ddc9b23157737c4cbb645.bundle differ
++++++ numa-Support-SGX-numa-in-the-monitor-and.patch ++++++
--- /var/tmp/diff_new_pack.MdzzZM/_old 2022-04-08 00:27:43.274320601 +0200
+++ /var/tmp/diff_new_pack.MdzzZM/_new 2022-04-08 00:27:43.278320555 +0200
@@ -33,7 +33,6 @@
Signed-off-by: Yang Zhong <yang.zh...@intel.com>
Message-Id: <20211101162009.62161-4-yang.zh...@intel.com>
Signed-off-by: Paolo Bonzini <pbonz...@redhat.com>
-(cherry picked from commit 4755927ae12547c2e7cb22c5fa1b39038c6c11b1)
Signed-off-by: Li Zhang <lizh...@suse.de>
---
hw/i386/sgx.c | 51 +++++++++++++++++++++++++++++++++++--------
++++++ python-aqmp-Fix-negotiation-with-pre-oob.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 31 Jan 2022 23:11:31 -0500
Subject: python/aqmp: Fix negotiation with pre-"oob" QEMU
Git-commit: fa73e6e4ca1a93c5bbf9d05fb2a25736ab810b35
QEMU versions prior to the "oob" capability *also* can't accept the
"enable" keyword argument at all. Fix the handshake process with older
QEMU versions.
Signed-off-by: John Snow <js...@redhat.com>
Reviewed-by: Hanna Reitz <hre...@redhat.com>
Reviewed-by: Kevin Wolf <kw...@redhat.com>
Message-id: 20220201041134.1237016-2-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/qmp_client.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/python/qemu/aqmp/qmp_client.py b/python/qemu/aqmp/qmp_client.py
index 8105e29fa8f04297ec9390ec25ea..6b43e1dbbe38eded19fd0115e8bc 100644
--- a/python/qemu/aqmp/qmp_client.py
+++ b/python/qemu/aqmp/qmp_client.py
@@ -292,9 +292,9 @@ class QMPClient(AsyncProtocol[Message], Events):
"""
self.logger.debug("Negotiating capabilities ...")
- arguments: Dict[str, List[str]] = {'enable': []}
+ arguments: Dict[str, List[str]] = {}
if self._greeting and 'oob' in self._greeting.QMP.capabilities:
- arguments['enable'].append('oob')
+ arguments.setdefault('enable', []).append('oob')
msg = self.make_execute_msg('qmp_capabilities', arguments=arguments)
# It's not safe to use execute() here, because the reader/writers
++++++ python-aqmp-add-SocketAddrT-to-package-r.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 10 Jan 2022 18:28:48 -0500
Subject: python/aqmp: add SocketAddrT to package root
Git-commit: 728dcac5e356ce5b948943f21c0c72a1b2d96122
It's a commonly needed definition, it can be re-exported by the root.
Signed-off-by: John Snow <js...@redhat.com>
Reviewed-by: Vladimir Sementsov-Ogievskiy <vsement...@virtuozzo.com>
Reviewed-by: Beraldo Leal <bl...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/__init__.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/python/qemu/aqmp/__init__.py b/python/qemu/aqmp/__init__.py
index 880d5b6fa7f2c966542b12e25571..c6fa2dda58fdf4f1a867e677eadb 100644
--- a/python/qemu/aqmp/__init__.py
+++ b/python/qemu/aqmp/__init__.py
@@ -26,7 +26,12 @@ import logging
from .error import AQMPError
from .events import EventListener
from .message import Message
-from .protocol import ConnectError, Runstate, StateError
+from .protocol import (
+ ConnectError,
+ Runstate,
+ SocketAddrT,
+ StateError,
+)
from .qmp_client import ExecInterruptedError, ExecuteError, QMPClient
@@ -48,4 +53,7 @@ __all__ = (
'ConnectError',
'ExecuteError',
'ExecInterruptedError',
+
+ # Type aliases
+ 'SocketAddrT',
)
++++++ python-aqmp-add-__del__-method-to-legacy.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 10 Jan 2022 18:28:45 -0500
Subject: python/aqmp: add __del__ method to legacy interface
Git-commit: 3bc72e3aed76e0326703db81964b13f1da075cbf
asyncio can complain *very* loudly if you forget to back out of things
gracefully before the garbage collector starts destroying objects that
contain live references to asyncio Tasks.
The usual fix is just to remember to call aqmp.disconnect(), but for the
sake of the legacy wrapper and quick, one-off scripts where a graceful
shutdown is not necessarily of paramount imporance, add a courtesy
cleanup that will trigger prior to seeing screenfuls of confusing
asyncio tracebacks.
Note that we can't *always* save you from yourself; depending on when
the GC runs, you might just seriously be out of luck. The best we can do
in this case is to gently remind you to clean up after yourself.
(Still much better than multiple pages of incomprehensible python
warnings for the crime of forgetting to put your toys away.)
Signed-off-by: John Snow <js...@redhat.com>
Reviewed-by: Vladimir Sementsov-Ogievskiy <vsement...@virtuozzo.com>
Reviewed-by: Beraldo Leal <bl...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/legacy.py | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py
index 9e7b9fb80b95ad5d442fea0dbbab..2ccb136b02c1fcf586507400c056 100644
--- a/python/qemu/aqmp/legacy.py
+++ b/python/qemu/aqmp/legacy.py
@@ -16,6 +16,8 @@ from typing import (
import qemu.qmp
from qemu.qmp import QMPMessage, QMPReturnValue, SocketAddrT
+from .error import AQMPError
+from .protocol import Runstate
from .qmp_client import QMPClient
@@ -136,3 +138,19 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol):
def send_fd_scm(self, fd: int) -> None:
self._aqmp.send_fd_scm(fd)
+
+ def __del__(self) -> None:
+ if self._aqmp.runstate == Runstate.IDLE:
+ return
+
+ if not self._aloop.is_running():
+ self.close()
+ else:
+ # Garbage collection ran while the event loop was running.
+ # Nothing we can do about it now, but if we don't raise our
+ # own error, the user will be treated to a lot of traceback
+ # they might not understand.
+ raise AQMPError(
+ "QEMUMonitorProtocol.close()"
+ " was not called before object was garbage collected"
+ )
++++++ python-aqmp-add-_session_guard.patch ++++++
From: John Snow <js...@redhat.com>
Date: Fri, 25 Feb 2022 15:59:39 -0500
Subject: python/aqmp: add _session_guard()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 40196c23939758abc5300e85333e676196e3ba6d
In _new_session, there's a fairly complex except clause that's used to
give semantic errors to callers of accept() and connect(). We need to
create a new two-step replacement for accept(), so factoring out this
piece of logic will be useful.
Bolster the comments and docstring here to try and demystify what's
going on in this fairly delicate piece of Python magic.
(If we were using Python 3.7+, this would be an @asynccontextmanager. We
don't have that very nice piece of magic, however, so this must take an
Awaitable to manage the Exception contexts properly. We pay the price
for platform compatibility.)
Signed-off-by: John Snow <js...@redhat.com>
Acked-by: Kevin Wolf <kw...@redhat.com>
Reviewed-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220225205948.3693480-2-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/protocol.py | 89 +++++++++++++++++++++++++-----------
1 file changed, 62 insertions(+), 27 deletions(-)
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index 33358f5cd72b61bd060b8dea6091..009883f64d011e44dd003e9dcde3 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -317,6 +317,62 @@ class AsyncProtocol(Generic[T]):
# Section: Session machinery
# --------------------------
+ async def _session_guard(self, coro: Awaitable[None], emsg: str) -> None:
+ """
+ Async guard function used to roll back to `IDLE` on any error.
+
+ On any Exception, the state machine will be reset back to
+ `IDLE`. Most Exceptions will be wrapped with `ConnectError`, but
+ `BaseException` events will be left alone (This includes
+ asyncio.CancelledError, even prior to Python 3.8).
+
+ :param error_message:
+ Human-readable string describing what connection phase failed.
+
+ :raise BaseException:
+ When `BaseException` occurs in the guarded block.
+ :raise ConnectError:
+ When any other error is encountered in the guarded block.
+ """
+ # Note: After Python 3.6 support is removed, this should be an
+ # @asynccontextmanager instead of accepting a callback.
+ try:
+ await coro
+ except BaseException as err:
+ self.logger.error("%s: %s", emsg, exception_summary(err))
+ self.logger.debug("%s:\n%s\n", emsg, pretty_traceback())
+ try:
+ # Reset the runstate back to IDLE.
+ await self.disconnect()
+ except:
+ # We don't expect any Exceptions from the disconnect function
+ # here, because we failed to connect in the first place.
+ # The disconnect() function is intended to perform
+ # only cannot-fail cleanup here, but you never know.
+ emsg = (
+ "Unexpected bottom half exception. "
+ "This is a bug in the QMP library. "
+ "Please report it to <qemu-de...@nongnu.org> and "
+ "CC: John Snow <js...@redhat.com>."
+ )
+ self.logger.critical("%s:\n%s\n", emsg, pretty_traceback())
+ raise
+
+ # CancelledError is an Exception with special semantic meaning;
+ # We do NOT want to wrap it up under ConnectError.
+ # NB: CancelledError is not a BaseException before Python 3.8
+ if isinstance(err, asyncio.CancelledError):
+ raise
+
+ # Any other kind of error can be treated as some kind of connection
+ # failure broadly. Inspect the 'exc' field to explore the root
+ # cause in greater detail.
+ if isinstance(err, Exception):
+ raise ConnectError(emsg, err) from err
+
+ # Raise BaseExceptions un-wrapped, they're more important.
+ raise
+
@property
def _runstate_event(self) -> asyncio.Event:
# asyncio.Event() objects should not be created prior to entrance into
@@ -371,34 +427,13 @@ class AsyncProtocol(Generic[T]):
"""
assert self.runstate == Runstate.IDLE
- try:
- phase = "connection"
- await self._establish_connection(address, ssl, accept)
-
- phase = "session"
- await self._establish_session()
+ await self._session_guard(
+ self._establish_connection(address, ssl, accept),
+ 'Failed to establish connection')
- except BaseException as err:
- emsg = f"Failed to establish {phase}"
- self.logger.error("%s: %s", emsg, exception_summary(err))
- self.logger.debug("%s:\n%s\n", emsg, pretty_traceback())
- try:
- # Reset from CONNECTING back to IDLE.
- await self.disconnect()
- except:
- emsg = "Unexpected bottom half exception"
- self.logger.critical("%s:\n%s\n", emsg, pretty_traceback())
- raise
-
- # NB: CancelledError is not a BaseException before Python 3.8
- if isinstance(err, asyncio.CancelledError):
- raise
-
- if isinstance(err, Exception):
- raise ConnectError(emsg, err) from err
-
- # Raise BaseExceptions un-wrapped, they're more important.
- raise
+ await self._session_guard(
+ self._establish_session(),
+ 'Failed to establish session')
assert self.runstate == Runstate.RUNNING
++++++ python-aqmp-add-socket-bind-step-to-lega.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 31 Jan 2022 23:11:34 -0500
Subject: python/aqmp: add socket bind step to legacy.py
Git-commit: b0b662bb2b340d63529672b5bdae596a6243c4d0
The synchronous QMP library would bind to the server address during
__init__(). The new library delays this to the accept() call, because
binding occurs inside of the call to start_[unix_]server(), which is an
async method -- so it cannot happen during __init__ anymore.
Python 3.7+ adds the ability to create the server (and thus the bind()
call) and begin the active listening in separate steps, but we don't
have that functionality in 3.6, our current minimum.
Therefore ... Add a temporary workaround that allows the synchronous
version of the client to bind the socket in advance, guaranteeing that
there will be a UNIX socket in the filesystem ready for the QEMU client
to connect to without a race condition.
(Yes, it's a bit ugly. Fixing it more nicely will have to wait until our
minimum Python version is 3.7+.)
Signed-off-by: John Snow <js...@redhat.com>
Reviewed-by: Kevin Wolf <kw...@redhat.com>
Message-id: 20220201041134.1237016-5-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/legacy.py | 3 +++
python/qemu/aqmp/protocol.py | 41 +++++++++++++++++++++++++++++++++---
2 files changed, 41 insertions(+), 3 deletions(-)
diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py
index 0890f95b16875ecb815ed4560bc7..6baa5f3409a6b459c67097d3c2a0 100644
--- a/python/qemu/aqmp/legacy.py
+++ b/python/qemu/aqmp/legacy.py
@@ -56,6 +56,9 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol):
self._address = address
self._timeout: Optional[float] = None
+ if server:
+ self._aqmp._bind_hack(address) # pylint: disable=protected-access
+
_T = TypeVar('_T')
def _sync(
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index 50e973c2f2dc9c5fa759380ab3e9..33358f5cd72b61bd060b8dea6091 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -15,6 +15,7 @@ from asyncio import StreamReader, StreamWriter
from enum import Enum
from functools import wraps
import logging
+import socket
from ssl import SSLContext
from typing import (
Any,
@@ -238,6 +239,9 @@ class AsyncProtocol(Generic[T]):
self._runstate = Runstate.IDLE
self._runstate_changed: Optional[asyncio.Event] = None
+ # Workaround for bind()
+ self._sock: Optional[socket.socket] = None
+
def __repr__(self) -> str:
cls_name = type(self).__name__
tokens = []
@@ -427,6 +431,34 @@ class AsyncProtocol(Generic[T]):
else:
await self._do_connect(address, ssl)
+ def _bind_hack(self, address: Union[str, Tuple[str, int]]) -> None:
+ """
+ Used to create a socket in advance of accept().
+
+ This is a workaround to ensure that we can guarantee timing of
+ precisely when a socket exists to avoid a connection attempt
+ bouncing off of nothing.
+
+ Python 3.7+ adds a feature to separate the server creation and
+ listening phases instead, and should be used instead of this
+ hack.
+ """
+ if isinstance(address, tuple):
+ family = socket.AF_INET
+ else:
+ family = socket.AF_UNIX
+
+ sock = socket.socket(family, socket.SOCK_STREAM)
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+ try:
+ sock.bind(address)
+ except:
+ sock.close()
+ raise
+
+ self._sock = sock
+
@upper_half
async def _do_accept(self, address: SocketAddrT,
ssl: Optional[SSLContext] = None) -> None:
@@ -464,24 +496,27 @@ class AsyncProtocol(Generic[T]):
if isinstance(address, tuple):
coro = asyncio.start_server(
_client_connected_cb,
- host=address[0],
- port=address[1],
+ host=None if self._sock else address[0],
+ port=None if self._sock else address[1],
ssl=ssl,
backlog=1,
limit=self._limit,
+ sock=self._sock,
)
else:
coro = asyncio.start_unix_server(
_client_connected_cb,
- path=address,
+ path=None if self._sock else address,
ssl=ssl,
backlog=1,
limit=self._limit,
+ sock=self._sock,
)
server = await coro # Starts listening
await connected.wait() # Waits for the callback to fire (and finish)
assert server is None
+ self._sock = None
self.logger.debug("Connection accepted.")
++++++ python-aqmp-add-start_server-and-accept-.patch ++++++
From: John Snow <js...@redhat.com>
Date: Fri, 25 Feb 2022 15:59:46 -0500
Subject: python/aqmp: add start_server() and accept() methods
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 481607c7d35de2bc4d9bec7f4734036fc467f330
Add start_server() and accept() methods that can be used instead of
start_server_and_accept() to allow more fine-grained control over the
incoming connection process.
(Eagle-eyed reviewers will surely notice that it's a bit weird that
"CONNECTING" is a state that's shared between both the start_server()
and connect() states. That's absolutely true, and it's very true that
checking on the presence of _accepted as an indicator of state is a
hack. That's also very certainly true. But ... this keeps client code an
awful lot simpler, as it doesn't have to care exactly *how* the
connection is being made, just that it *is*. Is it worth disrupting that
simplicity in order to provide a better state guard on `accept()`? Hm.)
Signed-off-by: John Snow <js...@redhat.com>
Acked-by: Kevin Wolf <kw...@redhat.com>
Reviewed-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220225205948.3693480-9-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/protocol.py | 67 +++++++++++++++++++++++++++++++++---
python/tests/protocol.py | 7 ++++
2 files changed, 69 insertions(+), 5 deletions(-)
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index cdbc9cba0d15dea19cc1c60ca3c3..2ecba1455571a35e0e6c565e3641 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -280,6 +280,8 @@ class AsyncProtocol(Generic[T]):
Accept a connection and begin processing message queues.
If this call fails, `runstate` is guaranteed to be set back to `IDLE`.
+ This method is precisely equivalent to calling `start_server()`
+ followed by `accept()`.
:param address:
Address to listen on; UNIX socket path or TCP address/port.
@@ -294,9 +296,62 @@ class AsyncProtocol(Generic[T]):
protocol-level failure occurs while establishing a new
session, the wrapped error may also be an `QMPError`.
"""
+ await self.start_server(address, ssl)
+ await self.accept()
+ assert self.runstate == Runstate.RUNNING
+
+ @upper_half
+ @require(Runstate.IDLE)
+ async def start_server(self, address: SocketAddrT,
+ ssl: Optional[SSLContext] = None) -> None:
+ """
+ Start listening for an incoming connection, but do not wait for a peer.
+
+ This method starts listening for an incoming connection, but
+ does not block waiting for a peer. This call will return
+ immediately after binding and listening on a socket. A later
+ call to `accept()` must be made in order to finalize the
+ incoming connection.
+
+ :param address:
+ Address to listen on; UNIX socket path or TCP address/port.
+ :param ssl: SSL context to use, if any.
+
+ :raise StateError: When the `Runstate` is not `IDLE`.
+ :raise ConnectError:
+ When the server could not start listening on this address.
+
+ This exception will wrap a more concrete one. In most cases,
+ the wrapped exception will be `OSError`.
+ """
await self._session_guard(
self._do_start_server(address, ssl),
'Failed to establish connection')
+ assert self.runstate == Runstate.CONNECTING
+
+ @upper_half
+ @require(Runstate.CONNECTING)
+ async def accept(self) -> None:
+ """
+ Accept an incoming connection and begin processing message queues.
+
+ If this call fails, `runstate` is guaranteed to be set back to `IDLE`.
+
+ :raise StateError: When the `Runstate` is not `CONNECTING`.
+ :raise QMPError: When `start_server()` was not called yet.
+ :raise ConnectError:
+ When a connection or session cannot be established.
+
+ This exception will wrap a more concrete one. In most cases,
+ the wrapped exception will be `OSError` or `EOFError`. If a
+ protocol-level failure occurs while establishing a new
+ session, the wrapped error may also be an `QMPError`.
+ """
+ if self._accepted is None:
+ raise QMPError("Cannot call accept() before start_server().")
+ await self._session_guard(
+ self._do_accept(),
+ 'Failed to establish connection')
await self._session_guard(
self._establish_session(),
'Failed to establish session')
@@ -512,7 +567,12 @@ class AsyncProtocol(Generic[T]):
async def _do_start_server(self, address: SocketAddrT,
ssl: Optional[SSLContext] = None) -> None:
"""
- Acting as the transport server, accept a single connection.
+ Start listening for an incoming connection, but do not wait for a peer.
+
+ This method starts listening for an incoming connection, but does not
+ block waiting for a peer. This call will return immediately after
+ binding and listening to a socket. A later call to accept() must be
+ made in order to finalize the incoming connection.
:param address:
Address to listen on; UNIX socket path or TCP address/port.
@@ -554,10 +614,7 @@ class AsyncProtocol(Generic[T]):
# This will start the server (bind(2), listen(2)). It will also
# call accept(2) if we yield, but we don't block on that here.
self._server = await coro
-
- # Just for this one commit, wait for a peer.
- # This gets split out in the next patch.
- await self._do_accept()
+ self.logger.debug("Server listening on %s", address)
@upper_half
async def _do_accept(self) -> None:
diff --git a/python/tests/protocol.py b/python/tests/protocol.py
index 5e442e1efbd19bf95a95de371060..d6849ad3062081c62b29bf89dd2a 100644
--- a/python/tests/protocol.py
+++ b/python/tests/protocol.py
@@ -43,11 +43,18 @@ class NullProtocol(AsyncProtocol[None]):
async def _do_start_server(self, address, ssl=None):
if self.fake_session:
+ self._accepted = asyncio.Event()
self._set_state(Runstate.CONNECTING)
await asyncio.sleep(0)
else:
await super()._do_start_server(address, ssl)
+ async def _do_accept(self):
+ if self.fake_session:
+ self._accepted = None
+ else:
+ await super()._do_accept()
+
async def _do_connect(self, address, ssl=None):
if self.fake_session:
self._set_state(Runstate.CONNECTING)
++++++ python-aqmp-copy-type-definitions-from-q.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 10 Jan 2022 18:28:47 -0500
Subject: python/aqmp: copy type definitions from qmp
Git-commit: 0e6bfd8b96e407db7b0cb5e8c14cc315a7154f53
Copy the remaining type definitions from QMP into the qemu.aqmp.legacy
module. Now, users that require the legacy interface don't need to
import anything else but qemu.aqmp.legacy wrapper.
Signed-off-by: John Snow <js...@redhat.com>
Reviewed-by: Vladimir Sementsov-Ogievskiy <vsement...@virtuozzo.com>
Reviewed-by: Beraldo Leal <bl...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/legacy.py | 22 ++++++++++++++++++++--
python/qemu/aqmp/protocol.py | 16 ++++++++++------
2 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py
index 2ccb136b02c1fcf586507400c056..9431fe933019b1e1c221ea3ab7bb 100644
--- a/python/qemu/aqmp/legacy.py
+++ b/python/qemu/aqmp/legacy.py
@@ -6,7 +6,9 @@ This class pretends to be qemu.qmp.QEMUMonitorProtocol.
import asyncio
from typing import (
+ Any,
Awaitable,
+ Dict,
List,
Optional,
TypeVar,
@@ -14,13 +16,29 @@ from typing import (
)
import qemu.qmp
-from qemu.qmp import QMPMessage, QMPReturnValue, SocketAddrT
from .error import AQMPError
-from .protocol import Runstate
+from .protocol import Runstate, SocketAddrT
from .qmp_client import QMPClient
+#: QMPMessage is an entire QMP message of any kind.
+QMPMessage = Dict[str, Any]
+
+#: QMPReturnValue is the 'return' value of a command.
+QMPReturnValue = object
+
+#: QMPObject is any object in a QMP message.
+QMPObject = Dict[str, object]
+
+# QMPMessage can be outgoing commands or incoming events/returns.
+# QMPReturnValue is usually a dict/json object, but due to QAPI's
+# 'returns-whitelist', it can actually be anything.
+#
+# {'return': {}} is a QMPMessage,
+# {} is the QMPReturnValue.
+
+
# pylint: disable=missing-docstring
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index c4fbe35a0e41c589059ec4fa37a8..5b4f2f0d0a81a0d2902358e9b799 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -46,6 +46,10 @@ T = TypeVar('T')
_U = TypeVar('_U')
_TaskFN = Callable[[], Awaitable[None]] # aka ``async def func() -> None``
+InternetAddrT = Tuple[str, int]
+UnixAddrT = str
+SocketAddrT = Union[UnixAddrT, InternetAddrT]
+
class Runstate(Enum):
"""Protocol session runstate."""
@@ -257,7 +261,7 @@ class AsyncProtocol(Generic[T]):
@upper_half
@require(Runstate.IDLE)
- async def accept(self, address: Union[str, Tuple[str, int]],
+ async def accept(self, address: SocketAddrT,
ssl: Optional[SSLContext] = None) -> None:
"""
Accept a connection and begin processing message queues.
@@ -275,7 +279,7 @@ class AsyncProtocol(Generic[T]):
@upper_half
@require(Runstate.IDLE)
- async def connect(self, address: Union[str, Tuple[str, int]],
+ async def connect(self, address: SocketAddrT,
ssl: Optional[SSLContext] = None) -> None:
"""
Connect to the server and begin processing message queues.
@@ -337,7 +341,7 @@ class AsyncProtocol(Generic[T]):
@upper_half
async def _new_session(self,
- address: Union[str, Tuple[str, int]],
+ address: SocketAddrT,
ssl: Optional[SSLContext] = None,
accept: bool = False) -> None:
"""
@@ -397,7 +401,7 @@ class AsyncProtocol(Generic[T]):
@upper_half
async def _establish_connection(
self,
- address: Union[str, Tuple[str, int]],
+ address: SocketAddrT,
ssl: Optional[SSLContext] = None,
accept: bool = False
) -> None:
@@ -424,7 +428,7 @@ class AsyncProtocol(Generic[T]):
await self._do_connect(address, ssl)
@upper_half
- async def _do_accept(self, address: Union[str, Tuple[str, int]],
+ async def _do_accept(self, address: SocketAddrT,
ssl: Optional[SSLContext] = None) -> None:
"""
Acting as the transport server, accept a single connection.
@@ -482,7 +486,7 @@ class AsyncProtocol(Generic[T]):
self.logger.debug("Connection accepted.")
@upper_half
- async def _do_connect(self, address: Union[str, Tuple[str, int]],
+ async def _do_connect(self, address: SocketAddrT,
ssl: Optional[SSLContext] = None) -> None:
"""
Acting as the transport client, initiate a connection to a server.
++++++ python-aqmp-drop-_bind_hack.patch ++++++
From: John Snow <js...@redhat.com>
Date: Fri, 25 Feb 2022 15:59:48 -0500
Subject: python/aqmp: drop _bind_hack()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 4c1fe7003c9b373acb0791b4356e2285a10365c0
_bind_hack() was a quick fix to allow async QMP to call bind(2) prior to
calling listen(2) and accept(2). This wasn't sufficient to fully address
the race condition present in synchronous clients.
With the race condition in legacy.py fixed (see the previous commit),
there are no longer any users of _bind_hack(). Drop it.
Fixes: b0b662bb2b3
Signed-off-by: John Snow <js...@redhat.com>
Acked-by: Kevin Wolf <kw...@redhat.com>
Reviewed-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220225205948.3693480-11-js...@redhat.com
[Expanded commit message. --js]
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/legacy.py | 2 +-
python/qemu/aqmp/protocol.py | 41 +++---------------------------------
2 files changed, 4 insertions(+), 39 deletions(-)
diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py
index cb50e60564823fdc0aeeb194c5e3..46026e9fdc6c8859a8c944076c71 100644
--- a/python/qemu/aqmp/legacy.py
+++ b/python/qemu/aqmp/legacy.py
@@ -57,7 +57,7 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol):
self._timeout: Optional[float] = None
if server:
- self._sync(self._aqmp.start_server(address))
+ self._sync(self._aqmp.start_server(self._address))
_T = TypeVar('_T')
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index 2ecba1455571a35e0e6c565e3641..36fae57f277826d5cd4026e0d079 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -18,7 +18,6 @@ from asyncio import StreamReader, StreamWriter
from enum import Enum
from functools import wraps
import logging
-import socket
from ssl import SSLContext
from typing import (
Any,
@@ -242,9 +241,6 @@ class AsyncProtocol(Generic[T]):
self._runstate = Runstate.IDLE
self._runstate_changed: Optional[asyncio.Event] = None
- # Workaround for bind()
- self._sock: Optional[socket.socket] = None
-
# Server state for start_server() and _incoming()
self._server: Optional[asyncio.AbstractServer] = None
self._accepted: Optional[asyncio.Event] = None
@@ -535,34 +531,6 @@ class AsyncProtocol(Generic[T]):
self._reader, self._writer = (reader, writer)
self._accepted.set()
- def _bind_hack(self, address: Union[str, Tuple[str, int]]) -> None:
- """
- Used to create a socket in advance of accept().
-
- This is a workaround to ensure that we can guarantee timing of
- precisely when a socket exists to avoid a connection attempt
- bouncing off of nothing.
-
- Python 3.7+ adds a feature to separate the server creation and
- listening phases instead, and should be used instead of this
- hack.
- """
- if isinstance(address, tuple):
- family = socket.AF_INET
- else:
- family = socket.AF_UNIX
-
- sock = socket.socket(family, socket.SOCK_STREAM)
- sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-
- try:
- sock.bind(address)
- except:
- sock.close()
- raise
-
- self._sock = sock
-
@upper_half
async def _do_start_server(self, address: SocketAddrT,
ssl: Optional[SSLContext] = None) -> None:
@@ -589,21 +557,19 @@ class AsyncProtocol(Generic[T]):
if isinstance(address, tuple):
coro = asyncio.start_server(
self._incoming,
- host=None if self._sock else address[0],
- port=None if self._sock else address[1],
+ host=address[0],
+ port=address[1],
ssl=ssl,
backlog=1,
limit=self._limit,
- sock=self._sock,
)
else:
coro = asyncio.start_unix_server(
self._incoming,
- path=None if self._sock else address,
+ path=address,
ssl=ssl,
backlog=1,
limit=self._limit,
- sock=self._sock,
)
# Allow runstate watchers to witness 'CONNECTING' state; some
@@ -630,7 +596,6 @@ class AsyncProtocol(Generic[T]):
await self._accepted.wait()
assert self._server is None
self._accepted = None
- self._sock = None
self.logger.debug("Connection accepted.")
++++++ python-aqmp-fix-docstring-typo.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 10 Jan 2022 18:28:44 -0500
Subject: python/aqmp: fix docstring typo
Git-commit: dc6877bd2ea04a38700adcea2359d5d20c1082a6
Reported-by: Vladimir Sementsov-Ogievskiy <vsement...@virtuozzo.com>
Signed-off-by: John Snow <js...@redhat.com>
Reviewed-by: Beraldo Leal <bl...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/qemu/aqmp/__init__.py b/python/qemu/aqmp/__init__.py
index c6fa2dda58fdf4f1a867e677eadb..05f467c1415360169e9c66c8d27e 100644
--- a/python/qemu/aqmp/__init__.py
+++ b/python/qemu/aqmp/__init__.py
@@ -6,7 +6,7 @@ asynchronously with QMP protocol servers, as implemented by
QEMU, the
QEMU Guest Agent, and the QEMU Storage Daemon.
`QMPClient` provides the main functionality of this package. All errors
-raised by this library dervive from `AQMPError`, see `aqmp.error` for
+raised by this library derive from `AQMPError`, see `aqmp.error` for
additional detail. See `aqmp.events` for an in-depth tutorial on
managing QMP events.
"""
++++++ python-aqmp-fix-race-condition-in-legacy.patch ++++++
From: John Snow <js...@redhat.com>
Date: Fri, 25 Feb 2022 15:59:47 -0500
Subject: python/aqmp: fix race condition in legacy.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 673856f9d889dc50b6a1a7964df960c4f00c7c93
legacy.py provides a synchronous model. iotests frequently uses this
paradigm:
- create QMP client object
- start QEMU process
- await connection from QEMU process
In the switch from sync to async QMP, the QMP client object stopped
calling bind() and listen() during the QMP object creation step, which
creates a race condition if the QEMU process dials in too quickly.
With refactoring out of the way, restore the former behavior of calling
bind() and listen() during __init__() to fix this race condition.
Signed-off-by: John Snow <js...@redhat.com>
Acked-by: Kevin Wolf <kw...@redhat.com>
Reviewed-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220225205948.3693480-10-js...@redhat.com
[Expanded commit message. --js]
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/legacy.py | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py
index dca1e76ed4994959caf542031363..cb50e60564823fdc0aeeb194c5e3 100644
--- a/python/qemu/aqmp/legacy.py
+++ b/python/qemu/aqmp/legacy.py
@@ -57,7 +57,7 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol):
self._timeout: Optional[float] = None
if server:
- self._aqmp._bind_hack(address) # pylint: disable=protected-access
+ self._sync(self._aqmp.start_server(address))
_T = TypeVar('_T')
@@ -90,10 +90,7 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol):
self._aqmp.await_greeting = True
self._aqmp.negotiate = True
- self._sync(
- self._aqmp.start_server_and_accept(self._address),
- timeout
- )
+ self._sync(self._aqmp.accept(), timeout)
ret = self._get_greeting()
assert ret is not None
++++++ python-aqmp-handle-asyncio.TimeoutError-.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 10 Jan 2022 18:28:46 -0500
Subject: python/aqmp: handle asyncio.TimeoutError on execute()
Git-commit: 3b5bf136f5798a4ea2c66875d6337ca3d6b79434
This exception can be injected into any await statement. If we are
canceled via timeout, we want to clear the pending execution record on
our way out.
Signed-off-by: John Snow <js...@redhat.com>
Reviewed-by: Beraldo Leal <bl...@redhat.com>
Reviewed-by: Vladimir Sementsov-Ogievskiy <vsement...@virtuozzo.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/qmp_client.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/python/qemu/aqmp/qmp_client.py b/python/qemu/aqmp/qmp_client.py
index 6b43e1dbbe38eded19fd0115e8bc..45864f288e4fe5a7505c0022ed13 100644
--- a/python/qemu/aqmp/qmp_client.py
+++ b/python/qemu/aqmp/qmp_client.py
@@ -435,7 +435,11 @@ class QMPClient(AsyncProtocol[Message], Events):
msg_id = msg['id']
self._pending[msg_id] = asyncio.Queue(maxsize=1)
- await self._outgoing.put(msg)
+ try:
+ await self._outgoing.put(msg)
+ except:
+ del self._pending[msg_id]
+ raise
return msg_id
@@ -452,9 +456,9 @@ class QMPClient(AsyncProtocol[Message], Events):
was lost, or some other problem.
"""
queue = self._pending[msg_id]
- result = await queue.get()
try:
+ result = await queue.get()
if isinstance(result, ExecInterruptedError):
raise result
return result
++++++ python-aqmp-refactor-_do_accept-into-two.patch ++++++
From: John Snow <js...@redhat.com>
Date: Fri, 25 Feb 2022 15:59:44 -0500
Subject: python/aqmp: refactor _do_accept() into two distinct steps
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 5e9902a030ab832b0b6577764c65ce6a6f874af6
Refactor _do_accept() into _do_start_server() and _do_accept(). As of
this commit, the former calls the latter, but in subsequent commits
they'll be split apart.
(So please forgive the misnomer for _do_start_server(); it will live up
to its name shortly, and the docstring will be updated then too. I'm
just cutting down on some churn.)
Signed-off-by: John Snow <js...@redhat.com>
Acked-by: Kevin Wolf <kw...@redhat.com>
Reviewed-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220225205948.3693480-7-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/protocol.py | 29 ++++++++++++++++++++++++-----
python/tests/protocol.py | 4 ++--
2 files changed, 26 insertions(+), 7 deletions(-)
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index 631bcdaa554f4a104af4e25a3c61..e2bdad542dc0ef451dd200a1d679 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -295,7 +295,7 @@ class AsyncProtocol(Generic[T]):
session, the wrapped error may also be an `QMPError`.
"""
await self._session_guard(
- self._do_accept(address, ssl),
+ self._do_start_server(address, ssl),
'Failed to establish connection')
await self._session_guard(
self._establish_session(),
@@ -509,8 +509,8 @@ class AsyncProtocol(Generic[T]):
self._sock = sock
@upper_half
- async def _do_accept(self, address: SocketAddrT,
- ssl: Optional[SSLContext] = None) -> None:
+ async def _do_start_server(self, address: SocketAddrT,
+ ssl: Optional[SSLContext] = None) -> None:
"""
Acting as the transport server, accept a single connection.
@@ -551,9 +551,28 @@ class AsyncProtocol(Generic[T]):
# otherwise yield.
await asyncio.sleep(0)
- self._server = await coro # Starts listening
- await self._accepted.wait() # Waits for the callback to finish
+ # This will start the server (bind(2), listen(2)). It will also
+ # call accept(2) if we yield, but we don't block on that here.
+ self._server = await coro
+
+ # Just for this one commit, wait for a peer.
+ # This gets split out in the next patch.
+ await self._do_accept()
+
+ @upper_half
+ async def _do_accept(self) -> None:
+ """
+ Wait for and accept an incoming connection.
+
+ Requires that we have not yet accepted an incoming connection
+ from the upper_half, but it's OK if the server is no longer
+ running because the bottom_half has already accepted the
+ connection.
+ """
+ assert self._accepted is not None
+ await self._accepted.wait()
assert self._server is None
+ self._accepted = None
self._sock = None
self.logger.debug("Connection accepted.")
diff --git a/python/tests/protocol.py b/python/tests/protocol.py
index 8dd26c4ed1e0973b8058604c2373..5e442e1efbd19bf95a95de371060 100644
--- a/python/tests/protocol.py
+++ b/python/tests/protocol.py
@@ -41,12 +41,12 @@ class NullProtocol(AsyncProtocol[None]):
self.trigger_input = asyncio.Event()
await super()._establish_session()
- async def _do_accept(self, address, ssl=None):
+ async def _do_start_server(self, address, ssl=None):
if self.fake_session:
self._set_state(Runstate.CONNECTING)
await asyncio.sleep(0)
else:
- await super()._do_accept(address, ssl)
+ await super()._do_start_server(address, ssl)
async def _do_connect(self, address, ssl=None):
if self.fake_session:
++++++ python-aqmp-remove-_new_session-and-_est.patch ++++++
From: John Snow <js...@redhat.com>
Date: Fri, 25 Feb 2022 15:59:41 -0500
Subject: python/aqmp: remove _new_session and _establish_connection
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 68a6cf3ffe3532c0655efbbf5910bd99a1b4a3fa
These two methods attempted to entirely envelop the logic of
establishing a connection to a peer start to finish. However, we need to
break apart the incoming connection step into more granular steps. We
will no longer be able to reasonably constrain the logic inside of these
helper functions.
So, remove them - with _session_guard(), they no longer serve a real
purpose.
Although the public API doesn't change, the internal API does. Now that
there are no intermediary methods between e.g. connect() and
_do_connect(), there's no hook where the runstate is set. As a result,
the test suite changes a little to cope with the new semantics of
_do_accept() and _do_connect().
Lastly, take some pieces of the now-deleted docstrings and move
them up to the public interface level. They were a little more detailed,
and it won't hurt to keep them.
Signed-off-by: John Snow <js...@redhat.com>
Acked-by: Kevin Wolf <kw...@redhat.com>
Reviewed-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220225205948.3693480-4-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/protocol.py | 117 ++++++++++++++---------------------
python/tests/protocol.py | 10 ++-
2 files changed, 53 insertions(+), 74 deletions(-)
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index 73719257e058b7e9e4d8a281bcd9..b7e5e635d886db0efc85f829f42e 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -275,13 +275,25 @@ class AsyncProtocol(Generic[T]):
If this call fails, `runstate` is guaranteed to be set back to `IDLE`.
:param address:
- Address to listen to; UNIX socket path or TCP address/port.
+ Address to listen on; UNIX socket path or TCP address/port.
:param ssl: SSL context to use, if any.
:raise StateError: When the `Runstate` is not `IDLE`.
- :raise ConnectError: If a connection could not be accepted.
+ :raise ConnectError:
+ When a connection or session cannot be established.
+
+ This exception will wrap a more concrete one. In most cases,
+ the wrapped exception will be `OSError` or `EOFError`. If a
+ protocol-level failure occurs while establishing a new
+ session, the wrapped error may also be an `QMPError`.
"""
- await self._new_session(address, ssl, accept=True)
+ await self._session_guard(
+ self._do_accept(address, ssl),
+ 'Failed to establish connection')
+ await self._session_guard(
+ self._establish_session(),
+ 'Failed to establish session')
+ assert self.runstate == Runstate.RUNNING
@upper_half
@require(Runstate.IDLE)
@@ -297,9 +309,21 @@ class AsyncProtocol(Generic[T]):
:param ssl: SSL context to use, if any.
:raise StateError: When the `Runstate` is not `IDLE`.
- :raise ConnectError: If a connection cannot be made to the server.
+ :raise ConnectError:
+ When a connection or session cannot be established.
+
+ This exception will wrap a more concrete one. In most cases,
+ the wrapped exception will be `OSError` or `EOFError`. If a
+ protocol-level failure occurs while establishing a new
+ session, the wrapped error may also be an `QMPError`.
"""
- await self._new_session(address, ssl)
+ await self._session_guard(
+ self._do_connect(address, ssl),
+ 'Failed to establish connection')
+ await self._session_guard(
+ self._establish_session(),
+ 'Failed to establish session')
+ assert self.runstate == Runstate.RUNNING
@upper_half
async def disconnect(self) -> None:
@@ -401,73 +425,6 @@ class AsyncProtocol(Generic[T]):
self._runstate_event.set()
self._runstate_event.clear()
- @upper_half
- async def _new_session(self,
- address: SocketAddrT,
- ssl: Optional[SSLContext] = None,
- accept: bool = False) -> None:
- """
- Establish a new connection and initialize the session.
-
- Connect or accept a new connection, then begin the protocol
- session machinery. If this call fails, `runstate` is guaranteed
- to be set back to `IDLE`.
-
- :param address:
- Address to connect to/listen on;
- UNIX socket path or TCP address/port.
- :param ssl: SSL context to use, if any.
- :param accept: Accept a connection instead of connecting when `True`.
-
- :raise ConnectError:
- When a connection or session cannot be established.
-
- This exception will wrap a more concrete one. In most cases,
- the wrapped exception will be `OSError` or `EOFError`. If a
- protocol-level failure occurs while establishing a new
- session, the wrapped error may also be an `QMPError`.
- """
- assert self.runstate == Runstate.IDLE
-
- await self._session_guard(
- self._establish_connection(address, ssl, accept),
- 'Failed to establish connection')
-
- await self._session_guard(
- self._establish_session(),
- 'Failed to establish session')
-
- assert self.runstate == Runstate.RUNNING
-
- @upper_half
- async def _establish_connection(
- self,
- address: SocketAddrT,
- ssl: Optional[SSLContext] = None,
- accept: bool = False
- ) -> None:
- """
- Establish a new connection.
-
- :param address:
- Address to connect to/listen on;
- UNIX socket path or TCP address/port.
- :param ssl: SSL context to use, if any.
- :param accept: Accept a connection instead of connecting when `True`.
- """
- assert self.runstate == Runstate.IDLE
- self._set_state(Runstate.CONNECTING)
-
- # Allow runstate watchers to witness 'CONNECTING' state; some
- # failures in the streaming layer are synchronous and will not
- # otherwise yield.
- await asyncio.sleep(0)
-
- if accept:
- await self._do_accept(address, ssl)
- else:
- await self._do_connect(address, ssl)
-
def _bind_hack(self, address: Union[str, Tuple[str, int]]) -> None:
"""
Used to create a socket in advance of accept().
@@ -508,6 +465,9 @@ class AsyncProtocol(Generic[T]):
:raise OSError: For stream-related errors.
"""
+ assert self.runstate == Runstate.IDLE
+ self._set_state(Runstate.CONNECTING)
+
self.logger.debug("Awaiting connection on %s ...", address)
connected = asyncio.Event()
server: Optional[asyncio.AbstractServer] = None
@@ -550,6 +510,11 @@ class AsyncProtocol(Generic[T]):
sock=self._sock,
)
+ # Allow runstate watchers to witness 'CONNECTING' state; some
+ # failures in the streaming layer are synchronous and will not
+ # otherwise yield.
+ await asyncio.sleep(0)
+
server = await coro # Starts listening
await connected.wait() # Waits for the callback to fire (and finish)
assert server is None
@@ -569,6 +534,14 @@ class AsyncProtocol(Generic[T]):
:raise OSError: For stream-related errors.
"""
+ assert self.runstate == Runstate.IDLE
+ self._set_state(Runstate.CONNECTING)
+
+ # Allow runstate watchers to witness 'CONNECTING' state; some
+ # failures in the streaming layer are synchronous and will not
+ # otherwise yield.
+ await asyncio.sleep(0)
+
self.logger.debug("Connecting to %s ...", address)
if isinstance(address, tuple):
diff --git a/python/tests/protocol.py b/python/tests/protocol.py
index 354d6559b9d1e3dc3ad29598af3c..8dd26c4ed1e0973b8058604c2373 100644
--- a/python/tests/protocol.py
+++ b/python/tests/protocol.py
@@ -42,11 +42,17 @@ class NullProtocol(AsyncProtocol[None]):
await super()._establish_session()
async def _do_accept(self, address, ssl=None):
- if not self.fake_session:
+ if self.fake_session:
+ self._set_state(Runstate.CONNECTING)
+ await asyncio.sleep(0)
+ else:
await super()._do_accept(address, ssl)
async def _do_connect(self, address, ssl=None):
- if not self.fake_session:
+ if self.fake_session:
+ self._set_state(Runstate.CONNECTING)
+ await asyncio.sleep(0)
+ else:
await super()._do_connect(address, ssl)
async def _do_recv(self) -> None:
++++++ python-aqmp-rename-AQMPError-to-QMPError.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 10 Jan 2022 18:28:49 -0500
Subject: python/aqmp: rename AQMPError to QMPError
Git-commit: 6e7751dc388df6daf425db0e245d4d3a10859803
This is in preparation for renaming qemu.aqmp to qemu.qmp. I should have
done this from this from the very beginning, but it's a convenient time
to make sure this churn is taken care of.
Signed-off-by: John Snow <js...@redhat.com>
Reviewed-by: Vladimir Sementsov-Ogievskiy <vsement...@virtuozzo.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/__init__.py | 6 +++---
python/qemu/aqmp/error.py | 12 ++++++------
python/qemu/aqmp/events.py | 4 ++--
python/qemu/aqmp/legacy.py | 4 ++--
python/qemu/aqmp/protocol.py | 8 ++++----
python/qemu/aqmp/qmp_client.py | 8 ++++----
6 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/python/qemu/aqmp/__init__.py b/python/qemu/aqmp/__init__.py
index 05f467c1415360169e9c66c8d27e..4c22c380790fd0f86402e402628b 100644
--- a/python/qemu/aqmp/__init__.py
+++ b/python/qemu/aqmp/__init__.py
@@ -6,7 +6,7 @@ asynchronously with QMP protocol servers, as implemented by
QEMU, the
QEMU Guest Agent, and the QEMU Storage Daemon.
`QMPClient` provides the main functionality of this package. All errors
-raised by this library derive from `AQMPError`, see `aqmp.error` for
+raised by this library derive from `QMPError`, see `aqmp.error` for
additional detail. See `aqmp.events` for an in-depth tutorial on
managing QMP events.
"""
@@ -23,7 +23,7 @@ managing QMP events.
import logging
-from .error import AQMPError
+from .error import QMPError
from .events import EventListener
from .message import Message
from .protocol import (
@@ -48,7 +48,7 @@ __all__ = (
'Runstate',
# Exceptions, most generic to most explicit
- 'AQMPError',
+ 'QMPError',
'StateError',
'ConnectError',
'ExecuteError',
diff --git a/python/qemu/aqmp/error.py b/python/qemu/aqmp/error.py
index 781f49b00877893d7a88f755c67f..24ba4d505410b5fe56390e3d4e02 100644
--- a/python/qemu/aqmp/error.py
+++ b/python/qemu/aqmp/error.py
@@ -1,21 +1,21 @@
"""
-AQMP Error Classes
+QMP Error Classes
This package seeks to provide semantic error classes that are intended
to be used directly by clients when they would like to handle particular
semantic failures (e.g. "failed to connect") without needing to know the
enumeration of possible reasons for that failure.
-AQMPError serves as the ancestor for all exceptions raised by this
+QMPError serves as the ancestor for all exceptions raised by this
package, and is suitable for use in handling semantic errors from this
library. In most cases, individual public methods will attempt to catch
and re-encapsulate various exceptions to provide a semantic
error-handling interface.
-.. admonition:: AQMP Exception Hierarchy Reference
+.. admonition:: QMP Exception Hierarchy Reference
| `Exception`
- | +-- `AQMPError`
+ | +-- `QMPError`
| +-- `ConnectError`
| +-- `StateError`
| +-- `ExecInterruptedError`
@@ -31,11 +31,11 @@ error-handling interface.
"""
-class AQMPError(Exception):
+class QMPError(Exception):
"""Abstract error class for all errors originating from this package."""
-class ProtocolError(AQMPError):
+class ProtocolError(QMPError):
"""
Abstract error class for protocol failures.
diff --git a/python/qemu/aqmp/events.py b/python/qemu/aqmp/events.py
index 5f7150c78d49d9513978103dc9a7..f3d4e2b5e853c39db9e016009db0 100644
--- a/python/qemu/aqmp/events.py
+++ b/python/qemu/aqmp/events.py
@@ -443,7 +443,7 @@ from typing import (
Union,
)
-from .error import AQMPError
+from .error import QMPError
from .message import Message
@@ -451,7 +451,7 @@ EventNames = Union[str, Iterable[str], None]
EventFilter = Callable[[Message], bool]
-class ListenerError(AQMPError):
+class ListenerError(QMPError):
"""
Generic error class for `EventListener`-related problems.
"""
diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py
index 9431fe933019b1e1c221ea3ab7bb..27df22818a76190e872f08c0852e 100644
--- a/python/qemu/aqmp/legacy.py
+++ b/python/qemu/aqmp/legacy.py
@@ -17,7 +17,7 @@ from typing import (
import qemu.qmp
-from .error import AQMPError
+from .error import QMPError
from .protocol import Runstate, SocketAddrT
from .qmp_client import QMPClient
@@ -168,7 +168,7 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol):
# Nothing we can do about it now, but if we don't raise our
# own error, the user will be treated to a lot of traceback
# they might not understand.
- raise AQMPError(
+ raise QMPError(
"QEMUMonitorProtocol.close()"
" was not called before object was garbage collected"
)
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index 5b4f2f0d0a81a0d2902358e9b799..50e973c2f2dc9c5fa759380ab3e9 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -29,7 +29,7 @@ from typing import (
cast,
)
-from .error import AQMPError
+from .error import QMPError
from .util import (
bottom_half,
create_task,
@@ -65,7 +65,7 @@ class Runstate(Enum):
DISCONNECTING = 3
-class ConnectError(AQMPError):
+class ConnectError(QMPError):
"""
Raised when the initial connection process has failed.
@@ -90,7 +90,7 @@ class ConnectError(AQMPError):
return f"{self.error_message}: {cause}"
-class StateError(AQMPError):
+class StateError(QMPError):
"""
An API command (connect, execute, etc) was issued at an inappropriate time.
@@ -363,7 +363,7 @@ class AsyncProtocol(Generic[T]):
This exception will wrap a more concrete one. In most cases,
the wrapped exception will be `OSError` or `EOFError`. If a
protocol-level failure occurs while establishing a new
- session, the wrapped error may also be an `AQMPError`.
+ session, the wrapped error may also be an `QMPError`.
"""
assert self.runstate == Runstate.IDLE
diff --git a/python/qemu/aqmp/qmp_client.py b/python/qemu/aqmp/qmp_client.py
index 45864f288e4fe5a7505c0022ed13..90a8737f03a997f6813ee7cbcaac 100644
--- a/python/qemu/aqmp/qmp_client.py
+++ b/python/qemu/aqmp/qmp_client.py
@@ -20,7 +20,7 @@ from typing import (
cast,
)
-from .error import AQMPError, ProtocolError
+from .error import ProtocolError, QMPError
from .events import Events
from .message import Message
from .models import ErrorResponse, Greeting
@@ -66,7 +66,7 @@ class NegotiationError(_WrappedProtocolError):
"""
-class ExecuteError(AQMPError):
+class ExecuteError(QMPError):
"""
Exception raised by `QMPClient.execute()` on RPC failure.
@@ -87,7 +87,7 @@ class ExecuteError(AQMPError):
self.error_class: str = error_response.error.class_
-class ExecInterruptedError(AQMPError):
+class ExecInterruptedError(QMPError):
"""
Exception raised by `execute()` (et al) when an RPC is interrupted.
@@ -641,7 +641,7 @@ class QMPClient(AsyncProtocol[Message], Events):
sock = self._writer.transport.get_extra_info('socket')
if sock.family != socket.AF_UNIX:
- raise AQMPError("Sending file descriptors requires a UNIX socket.")
+ raise QMPError("Sending file descriptors requires a UNIX socket.")
if not hasattr(sock, 'sendmsg'):
# We need to void the warranty sticker.
++++++ python-aqmp-rename-accept-to-start_serve.patch ++++++
From: John Snow <js...@redhat.com>
Date: Fri, 25 Feb 2022 15:59:40 -0500
Subject: python/aqmp: rename 'accept()' to 'start_server_and_accept()'
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 0ba4e76b23fed77d09be7f56da783ab3f0b2d497
Previously, I had a method named "accept()" that under-the-hood calls
bind(2), listen(2) *and* accept(2). I meant this as a simplification and
counterpart to the one-shot "connect()" method.
This is confusing to readers who expect accept() to mean *just*
accept(2). Since I need to split apart the "accept()" method into
multiple methods anyway (one of which strongly resembling accept(2)), it
feels pertinent to rename this method *now*.
Rename this all-in-one method "start_server_and_accept()" instead.
Signed-off-by: John Snow <js...@redhat.com>
Acked-by: Kevin Wolf <kw...@redhat.com>
Reviewed-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220225205948.3693480-3-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/legacy.py | 2 +-
python/qemu/aqmp/protocol.py | 6 ++++--
python/tests/protocol.py | 24 ++++++++++++------------
3 files changed, 17 insertions(+), 15 deletions(-)
diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py
index 6baa5f3409a6b459c67097d3c2a0..dca1e76ed4994959caf542031363 100644
--- a/python/qemu/aqmp/legacy.py
+++ b/python/qemu/aqmp/legacy.py
@@ -91,7 +91,7 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol):
self._aqmp.negotiate = True
self._sync(
- self._aqmp.accept(self._address),
+ self._aqmp.start_server_and_accept(self._address),
timeout
)
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index 009883f64d011e44dd003e9dcde3..73719257e058b7e9e4d8a281bcd9 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -265,8 +265,10 @@ class AsyncProtocol(Generic[T]):
@upper_half
@require(Runstate.IDLE)
- async def accept(self, address: SocketAddrT,
- ssl: Optional[SSLContext] = None) -> None:
+ async def start_server_and_accept(
+ self, address: SocketAddrT,
+ ssl: Optional[SSLContext] = None
+ ) -> None:
"""
Accept a connection and begin processing message queues.
diff --git a/python/tests/protocol.py b/python/tests/protocol.py
index 5cd7938be35ec61a1d412728f64e..354d6559b9d1e3dc3ad29598af3c 100644
--- a/python/tests/protocol.py
+++ b/python/tests/protocol.py
@@ -413,14 +413,14 @@ class Accept(Connect):
assert family in ('INET', 'UNIX')
if family == 'INET':
- await self.proto.accept(('example.com', 1))
+ await self.proto.start_server_and_accept(('example.com', 1))
elif family == 'UNIX':
- await self.proto.accept('/dev/null')
+ await self.proto.start_server_and_accept('/dev/null')
async def _hanging_connection(self):
with TemporaryDirectory(suffix='.aqmp') as tmpdir:
sock = os.path.join(tmpdir, type(self.proto).__name__ + ".sock")
- await self.proto.accept(sock)
+ await self.proto.start_server_and_accept(sock)
class FakeSession(TestBase):
@@ -449,13 +449,13 @@ class FakeSession(TestBase):
@TestBase.async_test
async def testFakeAccept(self):
"""Test the full state lifecycle (via accept) with a no-op session."""
- await self.proto.accept('/not/a/real/path')
+ await self.proto.start_server_and_accept('/not/a/real/path')
self.assertEqual(self.proto.runstate, Runstate.RUNNING)
@TestBase.async_test
async def testFakeRecv(self):
"""Test receiving a fake/null message."""
- await self.proto.accept('/not/a/real/path')
+ await self.proto.start_server_and_accept('/not/a/real/path')
logname = self.proto.logger.name
with self.assertLogs(logname, level='DEBUG') as context:
@@ -471,7 +471,7 @@ class FakeSession(TestBase):
@TestBase.async_test
async def testFakeSend(self):
"""Test sending a fake/null message."""
- await self.proto.accept('/not/a/real/path')
+ await self.proto.start_server_and_accept('/not/a/real/path')
logname = self.proto.logger.name
with self.assertLogs(logname, level='DEBUG') as context:
@@ -493,7 +493,7 @@ class FakeSession(TestBase):
):
with self.assertRaises(StateError) as context:
if accept:
- await self.proto.accept('/not/a/real/path')
+ await self.proto.start_server_and_accept('/not/a/real/path')
else:
await self.proto.connect('/not/a/real/path')
@@ -504,7 +504,7 @@ class FakeSession(TestBase):
@TestBase.async_test
async def testAcceptRequireRunning(self):
"""Test that accept() cannot be called when Runstate=RUNNING"""
- await self.proto.accept('/not/a/real/path')
+ await self.proto.start_server_and_accept('/not/a/real/path')
await self._prod_session_api(
Runstate.RUNNING,
@@ -515,7 +515,7 @@ class FakeSession(TestBase):
@TestBase.async_test
async def testConnectRequireRunning(self):
"""Test that connect() cannot be called when Runstate=RUNNING"""
- await self.proto.accept('/not/a/real/path')
+ await self.proto.start_server_and_accept('/not/a/real/path')
await self._prod_session_api(
Runstate.RUNNING,
@@ -526,7 +526,7 @@ class FakeSession(TestBase):
@TestBase.async_test
async def testAcceptRequireDisconnecting(self):
"""Test that accept() cannot be called when Runstate=DISCONNECTING"""
- await self.proto.accept('/not/a/real/path')
+ await self.proto.start_server_and_accept('/not/a/real/path')
# Cheat: force a disconnect.
await self.proto.simulate_disconnect()
@@ -541,7 +541,7 @@ class FakeSession(TestBase):
@TestBase.async_test
async def testConnectRequireDisconnecting(self):
"""Test that connect() cannot be called when Runstate=DISCONNECTING"""
- await self.proto.accept('/not/a/real/path')
+ await self.proto.start_server_and_accept('/not/a/real/path')
# Cheat: force a disconnect.
await self.proto.simulate_disconnect()
@@ -576,7 +576,7 @@ class SimpleSession(TestBase):
async def testSmoke(self):
with TemporaryDirectory(suffix='.aqmp') as tmpdir:
sock = os.path.join(tmpdir, type(self.proto).__name__ + ".sock")
- server_task = create_task(self.server.accept(sock))
+ server_task =
create_task(self.server.start_server_and_accept(sock))
# give the server a chance to start listening [...]
await asyncio.sleep(0)
++++++ python-aqmp-split-_client_connected_cb-o.patch ++++++
From: John Snow <js...@redhat.com>
Date: Fri, 25 Feb 2022 15:59:42 -0500
Subject: python/aqmp: split _client_connected_cb() out as _incoming()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 830e6fd36e2aef37b158a10dea6c3853ce43b20c
As part of disentangling the monolithic nature of _do_accept(), split
out the incoming callback to prepare for factoring out the "wait for a
peer" step. Namely, this means using an event signal we can wait on from
outside of this method.
Signed-off-by: John Snow <js...@redhat.com>
Acked-by: Kevin Wolf <kw...@redhat.com>
Reviewed-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220225205948.3693480-5-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/protocol.py | 83 +++++++++++++++++++++++++-----------
1 file changed, 58 insertions(+), 25 deletions(-)
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index b7e5e635d886db0efc85f829f42e..56f05b90308c44a86d0978fd2ce6 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -242,6 +242,10 @@ class AsyncProtocol(Generic[T]):
# Workaround for bind()
self._sock: Optional[socket.socket] = None
+ # Server state for start_server() and _incoming()
+ self._server: Optional[asyncio.AbstractServer] = None
+ self._accepted: Optional[asyncio.Event] = None
+
def __repr__(self) -> str:
cls_name = type(self).__name__
tokens = []
@@ -425,6 +429,54 @@ class AsyncProtocol(Generic[T]):
self._runstate_event.set()
self._runstate_event.clear()
+ @bottom_half # However, it does not run from the R/W tasks.
+ async def _stop_server(self) -> None:
+ """
+ Stop listening for / accepting new incoming connections.
+ """
+ if self._server is None:
+ return
+
+ try:
+ self.logger.debug("Stopping server.")
+ self._server.close()
+ await self._server.wait_closed()
+ self.logger.debug("Server stopped.")
+ finally:
+ self._server = None
+
+ @bottom_half # However, it does not run from the R/W tasks.
+ async def _incoming(self,
+ reader: asyncio.StreamReader,
+ writer: asyncio.StreamWriter) -> None:
+ """
+ Accept an incoming connection and signal the upper_half.
+
+ This method does the minimum necessary to accept a single
+ incoming connection. It signals back to the upper_half ASAP so
+ that any errors during session initialization can occur
+ naturally in the caller's stack.
+
+ :param reader: Incoming `asyncio.StreamReader`
+ :param writer: Incoming `asyncio.StreamWriter`
+ """
+ peer = writer.get_extra_info('peername', 'Unknown peer')
+ self.logger.debug("Incoming connection from %s", peer)
+
+ if self._reader or self._writer:
+ # Sadly, we can have more than one pending connection
+ # because of https://bugs.python.org/issue46715
+ # Close any extra connections we don't actually want.
+ self.logger.warning("Extraneous connection inadvertently accepted")
+ writer.close()
+ return
+
+ # A connection has been accepted; stop listening for new ones.
+ assert self._accepted is not None
+ await self._stop_server()
+ self._reader, self._writer = (reader, writer)
+ self._accepted.set()
+
def _bind_hack(self, address: Union[str, Tuple[str, int]]) -> None:
"""
Used to create a socket in advance of accept().
@@ -469,30 +521,11 @@ class AsyncProtocol(Generic[T]):
self._set_state(Runstate.CONNECTING)
self.logger.debug("Awaiting connection on %s ...", address)
- connected = asyncio.Event()
- server: Optional[asyncio.AbstractServer] = None
-
- async def _client_connected_cb(reader: asyncio.StreamReader,
- writer: asyncio.StreamWriter) -> None:
- """Used to accept a single incoming connection, see below."""
- nonlocal server
- nonlocal connected
-
- # A connection has been accepted; stop listening for new ones.
- assert server is not None
- server.close()
- await server.wait_closed()
- server = None
-
- # Register this client as being connected
- self._reader, self._writer = (reader, writer)
-
- # Signal back: We've accepted a client!
- connected.set()
+ self._accepted = asyncio.Event()
if isinstance(address, tuple):
coro = asyncio.start_server(
- _client_connected_cb,
+ self._incoming,
host=None if self._sock else address[0],
port=None if self._sock else address[1],
ssl=ssl,
@@ -502,7 +535,7 @@ class AsyncProtocol(Generic[T]):
)
else:
coro = asyncio.start_unix_server(
- _client_connected_cb,
+ self._incoming,
path=None if self._sock else address,
ssl=ssl,
backlog=1,
@@ -515,9 +548,9 @@ class AsyncProtocol(Generic[T]):
# otherwise yield.
await asyncio.sleep(0)
- server = await coro # Starts listening
- await connected.wait() # Waits for the callback to fire (and finish)
- assert server is None
+ self._server = await coro # Starts listening
+ await self._accepted.wait() # Waits for the callback to finish
+ assert self._server is None
self._sock = None
self.logger.debug("Connection accepted.")
++++++ python-aqmp-squelch-pylint-warning-for-t.patch ++++++
From: John Snow <js...@redhat.com>
Date: Fri, 25 Feb 2022 15:59:43 -0500
Subject: python/aqmp: squelch pylint warning for too many lines
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 1b9c8cb6ce6b5c5911eb715b2d5b0a2671999dde
I would really like to keep this under 1000 lines, I promise. Doesn't
look like it's gonna happen.
Signed-off-by: John Snow <js...@redhat.com>
Acked-by: Kevin Wolf <kw...@redhat.com>
Reviewed-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220225205948.3693480-6-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
(cherry picked from commit 1b9c8cb6ce6b5c5911eb715b2d5b0a2671999dde)
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/protocol.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index 56f05b90308c44a86d0978fd2ce6..631bcdaa554f4a104af4e25a3c61 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -10,6 +10,9 @@ In this package, it is used as the implementation for the
`QMPClient`
class.
"""
+# It's all the docstrings ... ! It's long for a good reason ^_^;
+# pylint: disable=too-many-lines
+
import asyncio
from asyncio import StreamReader, StreamWriter
from enum import Enum
++++++ python-aqmp-stop-the-server-during-disco.patch ++++++
From: John Snow <js...@redhat.com>
Date: Fri, 25 Feb 2022 15:59:45 -0500
Subject: python/aqmp: stop the server during disconnect()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 32c5abf051d06ff103d9d30eb6a7f3e8bf582334
Before we allow the full separation of starting the server and accepting
new connections, make sure that the disconnect cleans up the server and
its new state, too.
Signed-off-by: John Snow <js...@redhat.com>
Acked-by: Kevin Wolf <kw...@redhat.com>
Reviewed-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220225205948.3693480-8-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/protocol.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py
index e2bdad542dc0ef451dd200a1d679..cdbc9cba0d15dea19cc1c60ca3c3 100644
--- a/python/qemu/aqmp/protocol.py
+++ b/python/qemu/aqmp/protocol.py
@@ -432,7 +432,7 @@ class AsyncProtocol(Generic[T]):
self._runstate_event.set()
self._runstate_event.clear()
- @bottom_half # However, it does not run from the R/W tasks.
+ @bottom_half
async def _stop_server(self) -> None:
"""
Stop listening for / accepting new incoming connections.
@@ -709,6 +709,7 @@ class AsyncProtocol(Generic[T]):
self._reader = None
self._writer = None
+ self._accepted = None
# NB: _runstate_changed cannot be cleared because we still need it to
# send the final runstate changed event ...!
@@ -732,6 +733,9 @@ class AsyncProtocol(Generic[T]):
def _done(task: Optional['asyncio.Future[Any]']) -> bool:
return task is not None and task.done()
+ # If the server is running, stop it.
+ await self._stop_server()
+
# Are we already in an error pathway? If either of the tasks are
# already done, or if we have no tasks but a reader/writer; we
# must be.
++++++ python-introduce-qmp-shell-wrap-convenie.patch ++++++
From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= <berra...@redhat.com>
Date: Fri, 28 Jan 2022 16:11:56 +0000
Subject: python: introduce qmp-shell-wrap convenience tool
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 439125293cc9cfb684eb4db23db04199f5f435a2
With the current 'qmp-shell' tool developers must first spawn QEMU with
a suitable -qmp arg and then spawn qmp-shell in a separate terminal
pointing to the right socket.
With 'qmp-shell-wrap' developers can ignore QMP sockets entirely and
just pass the QEMU command and arguments they want. The program will
listen on a UNIX socket and tell QEMU to connect QMP to that.
For example, this:
# qmp-shell-wrap -- qemu-system-x86_64 -display none
Is roughly equivalent of running:
# qemu-system-x86_64 -display none -qmp qmp-shell-1234 &
# qmp-shell qmp-shell-1234
Except that 'qmp-shell-wrap' switches the socket peers around so that
it is the UNIX socket server and QEMU is the socket client. This makes
QEMU reliably go away when qmp-shell-wrap exits, closing the server
socket.
Signed-off-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220128161157.36261-2-berra...@redhat.com
[Edited for rebase. --js]
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/qmp_shell.py | 65 ++++++++++++++++++++++++++++++++---
python/setup.cfg | 1 +
scripts/qmp/qmp-shell-wrap | 11 ++++++
3 files changed, 73 insertions(+), 4 deletions(-)
diff --git a/python/qemu/aqmp/qmp_shell.py b/python/qemu/aqmp/qmp_shell.py
index d11bf54b00e5d56616ae57be0006..c60df787fcd50bf8a0109e5f5cd3 100644
--- a/python/qemu/aqmp/qmp_shell.py
+++ b/python/qemu/aqmp/qmp_shell.py
@@ -86,6 +86,7 @@ import logging
import os
import re
import readline
+from subprocess import Popen
import sys
from typing import (
Iterator,
@@ -167,8 +168,10 @@ class QMPShell(QEMUMonitorProtocol):
:param verbose: Echo outgoing QMP messages to console.
"""
def __init__(self, address: SocketAddrT,
- pretty: bool = False, verbose: bool = False):
- super().__init__(address)
+ pretty: bool = False,
+ verbose: bool = False,
+ server: bool = False):
+ super().__init__(address, server=server)
self._greeting: Optional[QMPMessage] = None
self._completer = QMPCompleter()
self._transmode = False
@@ -409,8 +412,10 @@ class HMPShell(QMPShell):
:param verbose: Echo outgoing QMP messages to console.
"""
def __init__(self, address: SocketAddrT,
- pretty: bool = False, verbose: bool = False):
- super().__init__(address, pretty, verbose)
+ pretty: bool = False,
+ verbose: bool = False,
+ server: bool = False):
+ super().__init__(address, pretty, verbose, server)
self._cpu_index = 0
def _cmd_completion(self) -> None:
@@ -533,5 +538,57 @@ def main() -> None:
pass
+def main_wrap() -> None:
+ """
+ qmp-shell-wrap entry point: parse command line arguments and
+ start the REPL.
+ """
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-H', '--hmp', action='store_true',
+ help='Use HMP interface')
+ parser.add_argument('-v', '--verbose', action='store_true',
+ help='Verbose (echo commands sent and received)')
+ parser.add_argument('-p', '--pretty', action='store_true',
+ help='Pretty-print JSON')
+
+ parser.add_argument('command', nargs=argparse.REMAINDER,
+ help='QEMU command line to invoke')
+
+ args = parser.parse_args()
+
+ cmd = args.command
+ if len(cmd) != 0 and cmd[0] == '--':
+ cmd = cmd[1:]
+ if len(cmd) == 0:
+ cmd = ["qemu-system-x86_64"]
+
+ sockpath = "qmp-shell-wrap-%d" % os.getpid()
+ cmd += ["-qmp", "unix:%s" % sockpath]
+
+ shell_class = HMPShell if args.hmp else QMPShell
+
+ try:
+ address = shell_class.parse_address(sockpath)
+ except QMPBadPortError:
+ parser.error(f"Bad port number: {sockpath}")
+ return # pycharm doesn't know error() is noreturn
+
+ try:
+ with shell_class(address, args.pretty, args.verbose, True) as qemu:
+ with Popen(cmd):
+
+ try:
+ qemu.accept()
+ except ConnectError as err:
+ if isinstance(err.exc, OSError):
+ die(f"Couldn't connect to {args.qmp_server}: {err!s}")
+ die(str(err))
+
+ for _ in qemu.repl():
+ pass
+ finally:
+ os.unlink(sockpath)
+
+
if __name__ == '__main__':
main()
diff --git a/python/setup.cfg b/python/setup.cfg
index 0063c757b78638ef651a362af338..bec54e8b0d663191e2b7afbfa350 100644
--- a/python/setup.cfg
+++ b/python/setup.cfg
@@ -68,6 +68,7 @@ console_scripts =
qom-fuse = qemu.utils.qom_fuse:QOMFuse.entry_point [fuse]
qemu-ga-client = qemu.utils.qemu_ga_client:main
qmp-shell = qemu.aqmp.qmp_shell:main
+ qmp-shell-wrap = qemu.aqmp.qmp_shell:main_wrap
aqmp-tui = qemu.aqmp.aqmp_tui:main [tui]
[flake8]
diff --git a/scripts/qmp/qmp-shell-wrap b/scripts/qmp/qmp-shell-wrap
new file mode 100755
index
0000000000000000000000000000000000000000..9e94da114f5f87588639f6b2cc636391e80c3864
--- /dev/null
+++ b/scripts/qmp/qmp-shell-wrap
@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python'))
+from qemu.qmp import qmp_shell
+
+
+if __name__ == '__main__':
+ qmp_shell.main_wrap()
++++++ python-machine-raise-VMLaunchFailure-exc.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 31 Jan 2022 23:11:32 -0500
Subject: python/machine: raise VMLaunchFailure exception from launch()
Git-commit: 50465f94d211beabfbfc80e4f85ec4fad0757570
This allows us to pack in some extra information about the failure,
which guarantees that if the caller did not *intentionally* cause a
failure (by capturing this Exception), some pretty good clues will be
printed at the bottom of the traceback information.
This will help make failures in the event of a non-negative return code
more obvious when they go unhandled; the current behavior in
_post_shutdown() is to print a warning message only in the event of
signal-based terminations (for negative return codes).
(Note: In Python, catching BaseException instead of Exception catches a
broader array of Exception events, including SystemExit and
KeyboardInterrupt. We do not want to "wrap" such exceptions as a
VMLaunchFailure, because that will 'downgrade' the exception from a
BaseException to a regular Exception. We do, however, want to perform
cleanup in either case, so catch on the broadest scope and
wrap-and-re-raise only in the more targeted scope.)
Signed-off-by: John Snow <js...@redhat.com>
Reviewed-by: Hanna Reitz <hre...@redhat.com>
Reviewed-by: Kevin Wolf <kw...@redhat.com>
Message-id: 20220201041134.1237016-3-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/machine/machine.py | 45 ++++++++++++++++++++---
tests/qemu-iotests/tests/mirror-top-perms | 3 +-
2 files changed, 40 insertions(+), 8 deletions(-)
diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py
index 67ab06ca2b6daa531b7c0ad9f7c2..a5972fab4d2b1fddfa2d5a7db882 100644
--- a/python/qemu/machine/machine.py
+++ b/python/qemu/machine/machine.py
@@ -74,6 +74,35 @@ class QEMUMachineAddDeviceError(QEMUMachineError):
"""
+class VMLaunchFailure(QEMUMachineError):
+ """
+ Exception raised when a VM launch was attempted, but failed.
+ """
+ def __init__(self, exitcode: Optional[int],
+ command: str, output: Optional[str]):
+ super().__init__(exitcode, command, output)
+ self.exitcode = exitcode
+ self.command = command
+ self.output = output
+
+ def __str__(self) -> str:
+ ret = ''
+ if self.__cause__ is not None:
+ name = type(self.__cause__).__name__
+ reason = str(self.__cause__)
+ if reason:
+ ret += f"{name}: {reason}"
+ else:
+ ret += f"{name}"
+ ret += '\n'
+
+ if self.exitcode is not None:
+ ret += f"\tExit code: {self.exitcode}\n"
+ ret += f"\tCommand: {self.command}\n"
+ ret += f"\tOutput: {self.output}\n"
+ return ret
+
+
class AbnormalShutdown(QEMUMachineError):
"""
Exception raised when a graceful shutdown was requested, but not performed.
@@ -397,7 +426,7 @@ class QEMUMachine:
try:
self._launch()
- except:
+ except BaseException as exc:
# We may have launched the process but it may
# have exited before we could connect via QMP.
# Assume the VM didn't launch or is exiting.
@@ -408,11 +437,15 @@ class QEMUMachine:
else:
self._post_shutdown()
- LOG.debug('Error launching VM')
- if self._qemu_full_args:
- LOG.debug('Command: %r', ' '.join(self._qemu_full_args))
- if self._iolog:
- LOG.debug('Output: %r', self._iolog)
+ if isinstance(exc, Exception):
+ raise VMLaunchFailure(
+ exitcode=self.exitcode(),
+ command=' '.join(self._qemu_full_args),
+ output=self._iolog
+ ) from exc
+
+ # Don't wrap 'BaseException'; doing so would downgrade
+ # that exception. However, we still want to clean up.
raise
def _launch(self) -> None:
diff --git a/tests/qemu-iotests/tests/mirror-top-perms
b/tests/qemu-iotests/tests/mirror-top-perms
index 0a51a613f39764b2b3ab3fa460ef..b5849978c4158c35e18480186ea2 100755
--- a/tests/qemu-iotests/tests/mirror-top-perms
+++ b/tests/qemu-iotests/tests/mirror-top-perms
@@ -21,7 +21,6 @@
import os
-from qemu.aqmp import ConnectError
from qemu.machine import machine
from qemu.qmp import QMPConnectError
@@ -107,7 +106,7 @@ class TestMirrorTopPerms(iotests.QMPTestCase):
self.vm_b.launch()
print('ERROR: VM B launched successfully, '
'this should not have happened')
- except (QMPConnectError, ConnectError):
+ except (QMPConnectError, machine.VMLaunchFailure):
assert 'Is another process using the image' in self.vm_b.get_log()
result = self.vm.qmp('block-job-cancel',
++++++ python-move-qmp-shell-under-the-AQMP-pac.patch ++++++
++++ 1144 lines (skipped)
++++++ python-move-qmp-utilities-to-python-qemu.patch ++++++
++++ 2154 lines (skipped)
++++++ python-qmp-switch-qmp-shell-to-AQMP.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 10 Jan 2022 18:28:53 -0500
Subject: python/qmp: switch qmp-shell to AQMP
Git-commit: f3efd12930f34b9724e15d8fd2ff56a97b67219d
We have a replacement for async QMP, but it doesn't have feature parity
yet. For now, then, port the old tool onto the new backend.
Signed-off-by: John Snow <js...@redhat.com>
Reviewed-by: Vladimir Sementsov-Ogievskiy <vsement...@virtuozzo.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/legacy.py | 3 +++
python/qemu/qmp/qmp_shell.py | 31 +++++++++++++++++--------------
2 files changed, 20 insertions(+), 14 deletions(-)
diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py
index 27df22818a76190e872f08c0852e..0890f95b16875ecb815ed4560bc7 100644
--- a/python/qemu/aqmp/legacy.py
+++ b/python/qemu/aqmp/legacy.py
@@ -22,6 +22,9 @@ from .protocol import Runstate, SocketAddrT
from .qmp_client import QMPClient
+# (Temporarily) Re-export QMPBadPortError
+QMPBadPortError = qemu.qmp.QMPBadPortError
+
#: QMPMessage is an entire QMP message of any kind.
QMPMessage = Dict[str, Any]
diff --git a/python/qemu/qmp/qmp_shell.py b/python/qemu/qmp/qmp_shell.py
index e7d7eb18f19cae7ac185b333013e..d11bf54b00e5d56616ae57be0006 100644
--- a/python/qemu/qmp/qmp_shell.py
+++ b/python/qemu/qmp/qmp_shell.py
@@ -95,8 +95,13 @@ from typing import (
Sequence,
)
-from qemu import qmp
-from qemu.qmp import QMPMessage
+from qemu.aqmp import ConnectError, QMPError, SocketAddrT
+from qemu.aqmp.legacy import (
+ QEMUMonitorProtocol,
+ QMPBadPortError,
+ QMPMessage,
+ QMPObject,
+)
LOG = logging.getLogger(__name__)
@@ -125,7 +130,7 @@ class QMPCompleter:
return None
-class QMPShellError(qmp.QMPError):
+class QMPShellError(QMPError):
"""
QMP Shell Base error class.
"""
@@ -153,7 +158,7 @@ class FuzzyJSON(ast.NodeTransformer):
return node
-class QMPShell(qmp.QEMUMonitorProtocol):
+class QMPShell(QEMUMonitorProtocol):
"""
QMPShell provides a basic readline-based QMP shell.
@@ -161,7 +166,7 @@ class QMPShell(qmp.QEMUMonitorProtocol):
:param pretty: Pretty-print QMP messages.
:param verbose: Echo outgoing QMP messages to console.
"""
- def __init__(self, address: qmp.SocketAddrT,
+ def __init__(self, address: SocketAddrT,
pretty: bool = False, verbose: bool = False):
super().__init__(address)
self._greeting: Optional[QMPMessage] = None
@@ -237,7 +242,7 @@ class QMPShell(qmp.QEMUMonitorProtocol):
def _cli_expr(self,
tokens: Sequence[str],
- parent: qmp.QMPObject) -> None:
+ parent: QMPObject) -> None:
for arg in tokens:
(key, sep, val) = arg.partition('=')
if sep != '=':
@@ -403,7 +408,7 @@ class HMPShell(QMPShell):
:param pretty: Pretty-print QMP messages.
:param verbose: Echo outgoing QMP messages to console.
"""
- def __init__(self, address: qmp.SocketAddrT,
+ def __init__(self, address: SocketAddrT,
pretty: bool = False, verbose: bool = False):
super().__init__(address, pretty, verbose)
self._cpu_index = 0
@@ -512,19 +517,17 @@ def main() -> None:
try:
address = shell_class.parse_address(args.qmp_server)
- except qmp.QMPBadPortError:
+ except QMPBadPortError:
parser.error(f"Bad port number: {args.qmp_server}")
return # pycharm doesn't know error() is noreturn
with shell_class(address, args.pretty, args.verbose) as qemu:
try:
qemu.connect(negotiate=not args.skip_negotiation)
- except qmp.QMPConnectError:
- die("Didn't get QMP greeting message")
- except qmp.QMPCapabilitiesError:
- die("Couldn't negotiate capabilities")
- except OSError as err:
- die(f"Couldn't connect to {args.qmp_server}: {err!s}")
+ except ConnectError as err:
+ if isinstance(err.exc, OSError):
+ die(f"Couldn't connect to {args.qmp_server}: {err!s}")
+ die(str(err))
for _ in qemu.repl():
pass
++++++ python-support-recording-QMP-session-to-.patch ++++++
From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= <berra...@redhat.com>
Date: Fri, 28 Jan 2022 16:11:57 +0000
Subject: python: support recording QMP session to a file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Git-commit: 5c66d7d8de9a00460199669d26cd83fba135bee5
When running QMP commands with very large response payloads, it is often
not easy to spot the info you want. If we can save the response to a
file then tools like 'grep' or 'jq' can be used to extract information.
For convenience of processing, we merge the QMP command and response
dictionaries together:
{
"arguments": {},
"execute": "query-kvm",
"return": {
"enabled": false,
"present": true
}
}
Example usage
$ ./scripts/qmp/qmp-shell-wrap -l q.log -p -- ./build/qemu-system-x86_64
-display none
Welcome to the QMP low-level shell!
Connected
(QEMU) query-kvm
{
"return": {
"enabled": false,
"present": true
}
}
(QEMU) query-mice
{
"return": [
{
"absolute": false,
"current": true,
"index": 2,
"name": "QEMU PS/2 Mouse"
}
]
}
$ jq --slurp '. | to_entries[] | select(.value.execute == "query-kvm") |
.value.return.enabled' < q.log
false
Reviewed-by: Philippe Mathieu-Daud?? <f4...@amsat.org>
Signed-off-by: Daniel P. Berrang?? <berra...@redhat.com>
Message-id: 20220128161157.36261-3-berra...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/qemu/aqmp/qmp_shell.py | 29 ++++++++++++++++++++++-------
python/setup.cfg | 3 +++
2 files changed, 25 insertions(+), 7 deletions(-)
diff --git a/python/qemu/aqmp/qmp_shell.py b/python/qemu/aqmp/qmp_shell.py
index c60df787fcd50bf8a0109e5f5cd3..35691494d0a88070bcf1ad691699 100644
--- a/python/qemu/aqmp/qmp_shell.py
+++ b/python/qemu/aqmp/qmp_shell.py
@@ -89,6 +89,7 @@ import readline
from subprocess import Popen
import sys
from typing import (
+ IO,
Iterator,
List,
NoReturn,
@@ -170,7 +171,8 @@ class QMPShell(QEMUMonitorProtocol):
def __init__(self, address: SocketAddrT,
pretty: bool = False,
verbose: bool = False,
- server: bool = False):
+ server: bool = False,
+ logfile: Optional[str] = None):
super().__init__(address, server=server)
self._greeting: Optional[QMPMessage] = None
self._completer = QMPCompleter()
@@ -180,6 +182,10 @@ class QMPShell(QEMUMonitorProtocol):
'.qmp-shell_history')
self.pretty = pretty
self.verbose = verbose
+ self.logfile = None
+
+ if logfile is not None:
+ self.logfile = open(logfile, "w", encoding='utf-8')
def close(self) -> None:
# Hook into context manager of parent to save shell history.
@@ -320,11 +326,11 @@ class QMPShell(QEMUMonitorProtocol):
self._cli_expr(cmdargs[1:], qmpcmd['arguments'])
return qmpcmd
- def _print(self, qmp_message: object) -> None:
+ def _print(self, qmp_message: object, fh: IO[str] = sys.stdout) -> None:
jsobj = json.dumps(qmp_message,
indent=4 if self.pretty else None,
sort_keys=self.pretty)
- print(str(jsobj))
+ print(str(jsobj), file=fh)
def _execute_cmd(self, cmdline: str) -> bool:
try:
@@ -347,6 +353,9 @@ class QMPShell(QEMUMonitorProtocol):
print('Disconnected')
return False
self._print(resp)
+ if self.logfile is not None:
+ cmd = {**qmpcmd, **resp}
+ self._print(cmd, fh=self.logfile)
return True
def connect(self, negotiate: bool = True) -> None:
@@ -414,8 +423,9 @@ class HMPShell(QMPShell):
def __init__(self, address: SocketAddrT,
pretty: bool = False,
verbose: bool = False,
- server: bool = False):
- super().__init__(address, pretty, verbose, server)
+ server: bool = False,
+ logfile: Optional[str] = None):
+ super().__init__(address, pretty, verbose, server, logfile)
self._cpu_index = 0
def _cmd_completion(self) -> None:
@@ -508,6 +518,8 @@ def main() -> None:
help='Verbose (echo commands sent and received)')
parser.add_argument('-p', '--pretty', action='store_true',
help='Pretty-print JSON')
+ parser.add_argument('-l', '--logfile',
+ help='Save log of all QMP messages to PATH')
default_server = os.environ.get('QMP_SOCKET')
parser.add_argument('qmp_server', action='store',
@@ -526,7 +538,7 @@ def main() -> None:
parser.error(f"Bad port number: {args.qmp_server}")
return # pycharm doesn't know error() is noreturn
- with shell_class(address, args.pretty, args.verbose) as qemu:
+ with shell_class(address, args.pretty, args.verbose, args.logfile) as qemu:
try:
qemu.connect(negotiate=not args.skip_negotiation)
except ConnectError as err:
@@ -550,6 +562,8 @@ def main_wrap() -> None:
help='Verbose (echo commands sent and received)')
parser.add_argument('-p', '--pretty', action='store_true',
help='Pretty-print JSON')
+ parser.add_argument('-l', '--logfile',
+ help='Save log of all QMP messages to PATH')
parser.add_argument('command', nargs=argparse.REMAINDER,
help='QEMU command line to invoke')
@@ -574,7 +588,8 @@ def main_wrap() -> None:
return # pycharm doesn't know error() is noreturn
try:
- with shell_class(address, args.pretty, args.verbose, True) as qemu:
+ with shell_class(address, args.pretty, args.verbose,
+ True, args.logfile) as qemu:
with Popen(cmd):
try:
diff --git a/python/setup.cfg b/python/setup.cfg
index bec54e8b0d663191e2b7afbfa350..241f243e8b94417f9b032c41576b 100644
--- a/python/setup.cfg
+++ b/python/setup.cfg
@@ -114,7 +114,10 @@ ignore_missing_imports = True
# no Warning level messages displayed, use "--disable=all --enable=classes
# --disable=W".
disable=consider-using-f-string,
+ consider-using-with,
+ too-many-arguments,
too-many-function-args, # mypy handles this with less false positives.
+ too-many-instance-attributes,
no-member, # mypy also handles this better.
[pylint.basic]
++++++ python-upgrade-mypy-to-0.780.patch ++++++
From: John Snow <js...@redhat.com>
Date: Mon, 31 Jan 2022 23:11:33 -0500
Subject: python: upgrade mypy to 0.780
Git-commit: 74a1505d279897d2a448c876820a33cbe1f0f22e
We need a slightly newer version of mypy in order to use some features
of the asyncio server functions in the next commit.
(Note: pipenv is not really suited to upgrading individual packages; I
need to replace this tool with something better for the task. For now,
the miscellaneous updates not related to the mypy upgrade are simply
beyond my control. It's on my list to take care of soon.)
Signed-off-by: John Snow <js...@redhat.com>
Reviewed-by: Kevin Wolf <kw...@redhat.com>
Message-id: 20220201041134.1237016-4-js...@redhat.com
Signed-off-by: John Snow <js...@redhat.com>
Signed-off-by: Li Zhang <lizh...@suse.de>
---
python/Pipfile.lock | 66 ++++++++++++++++++++++++++-------------------
python/setup.cfg | 2 +-
2 files changed, 40 insertions(+), 28 deletions(-)
diff --git a/python/Pipfile.lock b/python/Pipfile.lock
index d2a7dbd88be19fd6db0baa083d8a..ce46404ce0840c693d3c982674ac 100644
--- a/python/Pipfile.lock
+++ b/python/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256":
"784b327272db32403d5a488507853b5afba850ba26a5948e5b6a90c1baef2d9c"
+ "sha256":
"f1a25654d884a5b450e38d78b1f2e3ebb9073e421cc4358d4bbb83ac251a5670"
},
"pipfile-spec": 6,
"requires": {
@@ -34,7 +34,7 @@
"sha256:09bdb456e02564731f8b5957cdd0c98a7f01d2db5e90eb1d794c353c28bfd705",
"sha256:6a8a51f64dae307f6e0c9db752b66a7951e282389d8362cc1d39a56f3feeb31d"
],
- "markers": "python_version ~= '3.6'",
+ "index": "pypi",
"version": "==2.6.0"
},
"avocado-framework": {
@@ -50,6 +50,7 @@
"sha256:106fef6dc37dd8c0e2c0a60d3fca3e77460a48907f335fa28420463a6f799736",
"sha256:23e223426b28491b1ced97dc3bbe183027419dfc7982b4fa2f05d5f3ff10711c"
],
+ "index": "pypi",
"version": "==0.3.2"
},
"filelock": {
@@ -57,6 +58,7 @@
"sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59",
"sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"
],
+ "index": "pypi",
"version": "==3.0.12"
},
"flake8": {
@@ -88,7 +90,7 @@
"sha256:54161657e8ffc76596c4ede7080ca68cb02962a2e074a2586b695a93a925d36e",
"sha256:e962bff7440364183203d179d7ae9ad90cb1f2b74dcb84300e88ecc42dca3351"
],
- "markers": "python_version < '3.7'",
+ "index": "pypi",
"version": "==5.1.4"
},
"isort": {
@@ -124,7 +126,7 @@
"sha256:ed361bb83436f117f9917d282a456f9e5009ea12fd6de8742d1a4752c3017e93",
"sha256:f5144c75445ae3ca2057faac03fda5a902eff196702b0a24daf1d6ce0650514b"
],
- "markers": "python_version >= '2.7' and python_version not in
'3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+ "index": "pypi",
"version": "==1.6.0"
},
"mccabe": {
@@ -136,23 +138,23 @@
},
"mypy": {
"hashes": [
-
"sha256:15b948e1302682e3682f11f50208b726a246ab4e6c1b39f9264a8796bb416aa2",
-
"sha256:219a3116ecd015f8dca7b5d2c366c973509dfb9a8fc97ef044a36e3da66144a1",
-
"sha256:3b1fc683fb204c6b4403a1ef23f0b1fac8e4477091585e0c8c54cbdf7d7bb164",
-
"sha256:3beff56b453b6ef94ecb2996bea101a08f1f8a9771d3cbf4988a61e4d9973761",
-
"sha256:7687f6455ec3ed7649d1ae574136835a4272b65b3ddcf01ab8704ac65616c5ce",
-
"sha256:7ec45a70d40ede1ec7ad7f95b3c94c9cf4c186a32f6bacb1795b60abd2f9ef27",
-
"sha256:86c857510a9b7c3104cf4cde1568f4921762c8f9842e987bc03ed4f160925754",
-
"sha256:8a627507ef9b307b46a1fea9513d5c98680ba09591253082b4c48697ba05a4ae",
-
"sha256:8dfb69fbf9f3aeed18afffb15e319ca7f8da9642336348ddd6cab2713ddcf8f9",
-
"sha256:a34b577cdf6313bf24755f7a0e3f3c326d5c1f4fe7422d1d06498eb25ad0c600",
-
"sha256:a8ffcd53cb5dfc131850851cc09f1c44689c2812d0beb954d8138d4f5fc17f65",
-
"sha256:b90928f2d9eb2f33162405f32dde9f6dcead63a0971ca8a1b50eb4ca3e35ceb8",
-
"sha256:c56ffe22faa2e51054c5f7a3bc70a370939c2ed4de308c690e7949230c995913",
-
"sha256:f91c7ae919bbc3f96cd5e5b2e786b2b108343d1d7972ea130f7de27fdd547cf3"
+
"sha256:00cb1964a7476e871d6108341ac9c1a857d6bd20bf5877f4773ac5e9d92cd3cd",
+
"sha256:127de5a9b817a03a98c5ae8a0c46a20dc44442af6dcfa2ae7f96cb519b312efa",
+
"sha256:1f3976a945ad7f0a0727aafdc5651c2d3278e3c88dee94e2bf75cd3386b7b2f4",
+
"sha256:2f8c098f12b402c19b735aec724cc9105cc1a9eea405d08814eb4b14a6fb1a41",
+
"sha256:4ef13b619a289aa025f2273e05e755f8049bb4eaba6d703a425de37d495d178d",
+
"sha256:5d142f219bf8c7894dfa79ebfb7d352c4c63a325e75f10dfb4c3db9417dcd135",
+
"sha256:62eb5dd4ea86bda8ce386f26684f7f26e4bfe6283c9f2b6ca6d17faf704dcfad",
+
"sha256:64c36eb0936d0bfb7d8da49f92c18e312ad2e3ed46e5548ae4ca997b0d33bd59",
+
"sha256:75eed74d2faf2759f79c5f56f17388defd2fc994222312ec54ee921e37b31ad4",
+
"sha256:974bebe3699b9b46278a7f076635d219183da26e1a675c1f8243a69221758273",
+
"sha256:a5e5bb12b7982b179af513dddb06fca12285f0316d74f3964078acbfcf4c68f2",
+
"sha256:d31291df31bafb997952dc0a17ebb2737f802c754aed31dd155a8bfe75112c57",
+
"sha256:d3b4941de44341227ece1caaf5b08b23e42ad4eeb8b603219afb11e9d4cfb437",
+
"sha256:eadb865126da4e3c4c95bdb47fe1bb087a3e3ea14d39a3b13224b8a4d9f9a102"
],
"index": "pypi",
- "version": "==0.770"
+ "version": "==0.780"
},
"mypy-extensions": {
"hashes": [
@@ -166,7 +168,7 @@
"sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5",
"sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"
],
- "markers": "python_version >= '2.7' and python_version not in
'3.0, 3.1, 3.2, 3.3'",
+ "index": "pypi",
"version": "==20.9"
},
"pluggy": {
@@ -174,7 +176,7 @@
"sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
"sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
],
- "markers": "python_version >= '2.7' and python_version not in
'3.0, 3.1, 3.2, 3.3'",
+ "index": "pypi",
"version": "==0.13.1"
},
"py": {
@@ -182,7 +184,7 @@
"sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
"sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
],
- "markers": "python_version >= '2.7' and python_version not in
'3.0, 3.1, 3.2, 3.3'",
+ "index": "pypi",
"version": "==1.10.0"
},
"pycodestyle": {
@@ -205,7 +207,7 @@
"sha256:a18f47b506a429f6f4b9df81bb02beab9ca21d0a5fee38ed15aef65f0545519f",
"sha256:d66e804411278594d764fc69ec36ec13d9ae9147193a1740cd34d272ca383b8e"
],
- "markers": "python_version >= '3.5'",
+ "index": "pypi",
"version": "==2.9.0"
},
"pylint": {
@@ -221,13 +223,21 @@
"sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
"sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
],
- "markers": "python_version >= '2.6' and python_version not in
'3.0, 3.1, 3.2, 3.3'",
+ "index": "pypi",
"version": "==2.4.7"
},
"qemu": {
"editable": true,
"path": "."
},
+ "setuptools": {
+ "hashes": [
+
"sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373",
+
"sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==59.6.0"
+ },
"six": {
"hashes": [
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
@@ -294,19 +304,21 @@
"sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342",
"sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"
],
- "markers": "python_version < '3.8'",
+ "index": "pypi",
"version": "==3.10.0.0"
},
"urwid": {
"hashes": [
"sha256:588bee9c1cb208d0906a9f73c613d2bd32c3ed3702012f51efe318a3f2127eae"
],
+ "index": "pypi",
"version": "==2.1.2"
},
"urwid-readline": {
"hashes": [
"sha256:018020cbc864bb5ed87be17dc26b069eae2755cb29f3a9c569aac3bded1efaf4"
],
+ "index": "pypi",
"version": "==0.13"
},
"virtualenv": {
@@ -314,7 +326,7 @@
"sha256:14fdf849f80dbb29a4eb6caa9875d476ee2a5cf76a5f5415fa2f1606010ab467",
"sha256:2b0126166ea7c9c3661f5b8e06773d28f83322de7a3ff7d06f0aed18c9de6a76"
],
- "markers": "python_version >= '2.7' and python_version not in
'3.0, 3.1, 3.2, 3.3'",
+ "index": "pypi",
"version": "==20.4.7"
},
"wrapt": {
@@ -328,7 +340,7 @@
"sha256:3607921face881ba3e026887d8150cca609d517579abe052ac81fc5aeffdbd76",
"sha256:51cb66cc54621609dd593d1787f286ee42a5c0adbb4b29abea5a63edc3e03098"
],
- "markers": "python_version < '3.10'",
+ "index": "pypi",
"version": "==3.4.1"
}
}
diff --git a/python/setup.cfg b/python/setup.cfg
index 417e937839b85eecd752b29ad7df..4f4f20571f304507e20ce16cee66 100644
--- a/python/setup.cfg
+++ b/python/setup.cfg
@@ -41,7 +41,7 @@ devel =
flake8 >= 3.6.0
fusepy >= 2.0.4
isort >= 5.1.2
- mypy >= 0.770
+ mypy >= 0.780
pylint >= 2.8.0
tox >= 3.18.0
urwid >= 2.1.2
