Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-rpyc for openSUSE:Factory checked in at 2022-12-07 17:35:07 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-rpyc (Old) and /work/SRC/openSUSE:Factory/.python-rpyc.new.1835 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-rpyc" Wed Dec 7 17:35:07 2022 rev:11 rq:1040778 version:5.3.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-rpyc/python-rpyc.changes 2022-09-26 18:47:53.940019703 +0200 +++ /work/SRC/openSUSE:Factory/.python-rpyc.new.1835/python-rpyc.changes 2022-12-07 17:36:27.049027264 +0100 @@ -1,0 +2,9 @@ +Tue Dec 6 15:32:20 UTC 2022 - Yogalakshmi Arunachalam <yarunacha...@suse.com> + +- Update to version 5.3.0 + #515 Support for Python 3.11 is available after teleportation bug fix + #507 Experimental support for threading is added (default is disabled for now) + #516 Resolved server-side exceptions due to the logic for checking if a name is in ModuleNamespace + #511 Improved documentation on the life-cycle of a netref/proxy-object + +------------------------------------------------------------------- Old: ---- 5.2.3.tar.gz New: ---- 5.3.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-rpyc.spec ++++++ --- /var/tmp/diff_new_pack.0abjiV/_old 2022-12-07 17:36:27.521029849 +0100 +++ /var/tmp/diff_new_pack.0abjiV/_new 2022-12-07 17:36:27.525029871 +0100 @@ -27,7 +27,7 @@ %endif %global skip_python2 1 Name: python-rpyc%{psuffix} -Version: 5.2.3 +Version: 5.3.0 Release: 0 Summary: Remote Python Call (RPyC), a RPC library License: MIT ++++++ 5.2.3.tar.gz -> 5.3.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/.github/workflows/python-app.yml new/rpyc-5.3.0/.github/workflows/python-app.yml --- old/rpyc-5.2.3/.github/workflows/python-app.yml 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/.github/workflows/python-app.yml 2022-11-26 07:09:01.000000000 +0100 @@ -10,36 +10,42 @@ branches: [ master ] jobs: - unittest-3-10: + python-unittest-all-versions: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12.0-alpha.1"] + steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.10 - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools flake8 - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - echo "PYTHONPATH=${PYTHONPATH}:/home/runner/work/rpyc" >> $GITHUB_ENV - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Init ssh settings - run: | - mkdir -pv ~/.ssh - chmod 700 ~/.ssh - echo NoHostAuthenticationForLocalhost yes >> ~/.ssh/config - echo StrictHostKeyChecking no >> ~/.ssh/config - ssh-keygen -q -f ~/.ssh/id_rsa -N '' - cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys - uname -a - - name: Test with unittest - run: | - python -m unittest discover -s ./rpyc ./tests + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools flake8 + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + echo "PYTHONPATH=${PYTHONPATH}:/home/runner/work/rpyc" >> $GITHUB_ENV + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Init ssh settings + run: | + mkdir -pv ~/.ssh + chmod 700 ~/.ssh + echo NoHostAuthenticationForLocalhost yes >> ~/.ssh/config + echo StrictHostKeyChecking no >> ~/.ssh/config + ssh-keygen -q -f ~/.ssh/id_rsa -N '' + cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys + uname -a + - name: Test with unittest + run: | + python -m unittest discover -v -s ./rpyc ./tests diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/CHANGELOG.rst new/rpyc-5.3.0/CHANGELOG.rst --- old/rpyc-5.2.3/CHANGELOG.rst 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/CHANGELOG.rst 2022-11-26 07:09:01.000000000 +0100 @@ -1,3 +1,18 @@ +5.3.0 +===== +Date: 2022-11-25 + +- `#515`_ Support for Python 3.11 is available after teleportation bug fix +- `#507`_ Experimental support for threading is added (default is disabled for now) +- `#516`_ Resolved server-side exceptions due to the logic for checking if a name is in `ModuleNamespace` +- `#511`_ Improved documentation on the life-cycle of a netref/proxy-object + +.. _#515: https://github.com/tomerfiliba-org/rpyc/pull/515 +.. _#507: https://github.com/tomerfiliba-org/rpyc/pull/507 +.. _#516: https://github.com/tomerfiliba-org/rpyc/issues/516 +.. _#515: https://github.com/tomerfiliba-org/rpyc/pull/515 +.. _#511: https://github.com/tomerfiliba-org/rpyc/issues/511 + 5.2.3 ===== Date: 2022-08-03 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/demos/async_client/server.py new/rpyc-5.3.0/demos/async_client/server.py --- old/rpyc-5.2.3/demos/async_client/server.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/demos/async_client/server.py 2022-11-26 07:09:01.000000000 +0100 @@ -20,8 +20,8 @@ def exposed_function(self, client_event, block_server_thread=False): if block_server_thread: # For some reason - _wait = lambda : getattr(client_event, 'wait')() # delays attr proxy behavior - _set = lambda : getattr(client_event, 'set')() # delays attr proxy behavior + def _wait(): return getattr(client_event, 'wait')() # delays attr proxy behavior + def _set(): return getattr(client_event, 'set')() # delays attr proxy behavior else: _wait = rpyc.async_(client_event.wait) # amortize proxy behavior _set = rpyc.async_(client_event.set) # amortize proxy behavior @@ -34,5 +34,6 @@ _set() logger.debug('Client event set, it may resume...') + if __name__ == "__main__": rpyc.ThreadedServer(service=Service, hostname="localhost", port=18812).start() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/demos/boilerplate/rpyc_server.py new/rpyc-5.3.0/demos/boilerplate/rpyc_server.py --- old/rpyc-5.2.3/demos/boilerplate/rpyc_server.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/demos/boilerplate/rpyc_server.py 2022-11-26 07:09:01.000000000 +0100 @@ -2,4 +2,4 @@ from rpyc_service import MyServiceFactory if __name__ == "__main__": - ThreadedServer(MyServiceFactory, port = 18000).start() \ No newline at end of file + ThreadedServer(MyServiceFactory, port=18000).start() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/demos/boilerplate/rpyc_service.py new/rpyc-5.3.0/demos/boilerplate/rpyc_service.py --- old/rpyc-5.2.3/demos/boilerplate/rpyc_service.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/demos/boilerplate/rpyc_service.py 2022-11-26 07:09:01.000000000 +0100 @@ -8,14 +8,14 @@ class exposed_MyService(object): # exposing names is not limited to methods :) - def __init__(self, filename, callback, interval = 1): + def __init__(self, filename, callback, interval=1): print("news client with " + filename + " " + str(callback)) self.filename = filename self.interval = interval self.last_stat = None self.callback = rpyc.async_(callback) # create an async callback self.active = True - self.thread = Thread(target = self.work) + self.thread = Thread(target=self.work) self.thread.start() def exposed_stop(self): # this method has to be exposed too diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/docs/conf.py new/rpyc-5.3.0/docs/conf.py --- old/rpyc-5.2.3/docs/conf.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/docs/conf.py 2022-11-26 07:09:01.000000000 +0100 @@ -11,7 +11,10 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os, time +from rpyc.version import __version__, release_date +import sys +import os +import time sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # If extensions (or modules to document with autodoc) are in another directory, @@ -49,7 +52,6 @@ # built documents. # # The short X.Y version. -from rpyc.version import __version__, release_date version = __version__ # The full version, including alpha/beta/rc tags. release = __version__ + "/" + release_date @@ -188,8 +190,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'RPyC.tex', u'RPyC Documentation', - u'Tomer Filiba', 'manual'), + ('index', 'RPyC.tex', u'RPyC Documentation', + u'Tomer Filiba', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/docs/docs/rpyc-release-process.rst new/rpyc-5.3.0/docs/docs/rpyc-release-process.rst --- old/rpyc-5.2.3/docs/docs/rpyc-release-process.rst 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/docs/docs/rpyc-release-process.rst 2022-11-26 07:09:01.000000000 +0100 @@ -28,7 +28,8 @@ --------------------------------- To create an initial entry draft, run some shell commands. -.. code-block:: python +.. code-block:: bash + owner="tomerfiliba-org" repo="rpyc" #url="https://github.com/${owner}/${repo}" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/pyproject.toml new/rpyc-5.3.0/pyproject.toml --- old/rpyc-5.2.3/pyproject.toml 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/pyproject.toml 2022-11-26 07:09:01.000000000 +0100 @@ -25,6 +25,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Internet", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Object Brokering", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/rpyc/__init__.py new/rpyc-5.3.0/rpyc/__init__.py --- old/rpyc-5.2.3/rpyc/__init__.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/rpyc/__init__.py 2022-11-26 07:09:01.000000000 +0100 @@ -1,43 +1,27 @@ -""" -:: - - ##### ##### #### - ## ## ## ## ## #### - ## ## ## ## ## # - ##### ##### ## ## ## ## - ## ## ## ## ## ## # - ## ## ## ### ## ### - ## ## ## ## ##### - -------------------- ## ------------------------------------------ - ## - -Remote Python Call (RPyC) +"""Remote Python Call (RPyC) is a transparent and symmetric distributed computing library Licensed under the MIT license (see `LICENSE` file) -A transparent, symmetric and light-weight RPC and distributed computing -library for python. - Usage:: >>> import rpyc >>> c = rpyc.connect_by_service("SERVICENAME") - >>> print c.root.some_function(1, 2, 3) + >>> print(c.root.some_function(1, 2, 3)) Classic-style usage:: >>> import rpyc >>> # `hostname` is assumed to be running a slave-service server >>> c = rpyc.classic.connect("hostname") - >>> print c.execute("x = 5") + >>> print(c.execute("x = 5")) None - >>> print c.eval("x + 2") + >>> print(c.eval("x + 2")) 7 - >>> print c.modules.os.listdir(".") #doctest: +ELLIPSIS + >>> print(c.modules.os.listdir(".")) # doctest: +ELLIPSIS [...] - >>> print c.modules["xml.dom.minidom"].parseString("<a/>") #doctest: +ELLIPSIS + >>> print(c.modules["xml.dom.minidom"].parseString("<a/>")) # doctest: +ELLIPSIS <xml.dom.minidom.Document instance at ...> - >>> f = c.builtin.open("foobar.txt", "rb") #doctest: +SKIP - >>> print f.read(100) #doctest: +SKIP + >>> f = c.builtin.open("foobar.txt", "rb") # doctest: +SKIP + >>> print(f.read(100)) # doctest: +SKIP ... """ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/rpyc/core/brine.py new/rpyc-5.3.0/rpyc/core/brine.py --- old/rpyc-5.2.3/rpyc/core/brine.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/rpyc/core/brine.py 2022-11-26 07:09:01.000000000 +0100 @@ -51,10 +51,14 @@ TAG_COMPLEX = b"\x1b" IMM_INTS = dict((i, bytes([i + 0x50])) for i in range(-0x30, 0xa0)) -I1 = Struct("!B") -I4 = Struct("!L") -F8 = Struct("!d") -C16 = Struct("!dd") +# Below "!" is used to set byte order as network (= big-endian). See https://docs.python.org/3/library/struct.html +F8 = Struct("!d") # Python type float w/ size [8] (ctype double) +C16 = Struct("!dd") # Successive floats (complex numbers) +I1 = Struct("!B") # Python type int w/ size [1] (ctype unsigned char) +I4 = Struct("!L") # Python type int w/ size [4] (ctype unsigned long) +# I4I4 is successive ints w/ size 4 and was introduced to pack local thread id and remote thread id +# Since PyThread_get_thread_ident returns a type of unsigned long, !LL can store both thread IDs. +I4I4 = Struct("!LL") _dump_registry = {} _load_registry = {} @@ -67,6 +71,7 @@ return func return deco + # =============================================================================== # dumping # =============================================================================== @@ -181,6 +186,7 @@ def _dump(obj, stream): _dump_registry.get(type(obj), _undumpable)(obj, stream) + # =============================================================================== # loading # =============================================================================== diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/rpyc/core/protocol.py new/rpyc-5.3.0/rpyc/core/protocol.py --- old/rpyc-5.2.3/rpyc/core/protocol.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/rpyc/core/protocol.py 2022-11-26 07:09:01.000000000 +0100 @@ -6,6 +6,11 @@ import time # noqa: F401 import gc # noqa: F401 +import collections +import concurrent.futures as c_futures +import os +import threading + from threading import Lock, Condition, RLock from rpyc.lib import spawn, Timeout, get_methods, get_id_pack, hasattr_static from rpyc.lib.compat import pickle, next, maxint, select_error, acquire_lock # noqa: F401 @@ -62,6 +67,7 @@ sync_request_timeout=30, before_closed=None, close_catchall=False, + bind_threads=os.environ.get('RPYC_BIND_THREADS') == 'true', ) """ The default configuration dictionary of the protocol. You can override these parameters @@ -119,6 +125,9 @@ do no have this configuration option set. ``sync_request_timeout`` ``30`` Default timeout for waiting results +``bind_threads`` ``False`` Whether to restrict request/reply by thread (experimental). + The default value is False. Setting the environment variable + `RPYC_BIND_THREADS` to `"true"` will enable this feature. ======================================= ================ ===================================================== """ @@ -129,6 +138,11 @@ class Connection(object): """The RPyC *connection* (AKA *protocol*). + Objects referenced over the connection are either local or remote. This class retains a strong reference to + local objects that is deleted when the reference count is zero. Remote/proxied objects have a life-cycle + controlled by a different address space. Since garbage collection is handled on the remote end, a weak reference + is used for netrefs. + :param root: the :class:`~rpyc.core.service.Service` object to expose :param channel: the :class:`~rpyc.core.channel.Channel` over which messages are passed :param config: the connection's configuration dict (overriding parameters @@ -157,6 +171,13 @@ self._send_queue = [] self._local_root = root self._closed = False + self._bind_threads = self._config['bind_threads'] + if self._bind_threads: + self._lock = threading.Lock() + self._threads = {} + self._receiving = False + self._thread_pool = [] + self._thread_pool_executor = c_futures.ThreadPoolExecutor() def __del__(self): self.close() @@ -187,6 +208,8 @@ # self._seqcounter = None # self._config.clear() del self._HANDLERS + if self._bind_threads: + self._thread_pool_executor.shutdown(wait=False) # TODO where? def close(self): # IO """closes the connection, releasing all held resources""" @@ -235,6 +258,15 @@ def _send(self, msg, seq, args): # IO data = brine.dump((msg, seq, args)) + if self._bind_threads: + this_thread = self._get_thread() + data = brine.I4I4.pack(this_thread.id, this_thread._remote_thread_id) + data + if msg == consts.MSG_REQUEST: + this_thread._occupation_count += 1 + else: + this_thread._occupation_count -= 1 + if this_thread._occupation_count == 0: + this_thread._remote_thread_id = 0 # GC might run while sending data # if so, a BaseNetref.__del__ might be called # BaseNetref.__del__ must call asyncreq, @@ -359,15 +391,23 @@ def _dispatch(self, data): # serving---dispatch? msg, seq, args = brine.load(data) if msg == consts.MSG_REQUEST: + if self._bind_threads: + self._get_thread()._occupation_count += 1 self._dispatch_request(seq, args) - elif msg == consts.MSG_REPLY: - obj = self._unbox(args) - self._seq_request_callback(msg, seq, False, obj) - elif msg == consts.MSG_EXCEPTION: - obj = self._unbox_exc(args) - self._seq_request_callback(msg, seq, True, obj) else: - raise ValueError(f"invalid message type: {msg!r}") + if self._bind_threads: + this_thread = self._get_thread() + this_thread._occupation_count -= 1 + if this_thread._occupation_count == 0: + this_thread._remote_thread_id = 0 + if msg == consts.MSG_REPLY: + obj = self._unbox(args) + self._seq_request_callback(msg, seq, False, obj) + elif msg == consts.MSG_EXCEPTION: + obj = self._unbox_exc(args) + self._seq_request_callback(msg, seq, True, obj) + else: + raise ValueError(f"invalid message type: {msg!r}") def serve(self, timeout=1, wait_for_lock=True): # serving """Serves a single request or reply that arrives within the given @@ -375,10 +415,11 @@ might trigger multiple (nested) requests, thus this function may be reentrant. - :returns: ``True`` if a request or reply were received, ``False`` - otherwise. + :returns: ``True`` if a request or reply were received, ``False`` otherwise. """ timeout = Timeout(timeout) + if self._bind_threads: + return self._serve_bound(timeout, wait_for_lock) with self._recv_event: # Exit early if we cannot acquire the recvlock if not self._recvlock.acquire(False): @@ -410,6 +451,193 @@ self._recvlock.release() return False + def _serve_bound(self, timeout, wait_for_lock): + """Serves messages like `serve` with the added benefit of making request/reply thread bound. + - Experimental functionality `RPYC_BIND_THREADS` + + The first 8 bytes indicate the sending thread ID and intended recipient ID. When the recipient + thread ID is not the thread that received the data, the remote thread ID and message are appended + to the intended threads `_deque` and `_event` is set. + + :returns: ``True`` if a request or reply were received, ``False`` otherwise. + """ + this_thread = self._get_thread() + wait = False + + with self._lock: + message_available = this_thread._event.is_set() and len(this_thread._deque) != 0 + + if message_available: + remote_thread_id, message = this_thread._deque.popleft() + if len(this_thread._deque) == 0: + this_thread._event.clear() + + else: + if self._receiving: # enter pool + self._thread_pool.append(this_thread) + wait = True + + else: + self._receiving = True + + if message_available: # just process + this_thread._remote_thread_id = remote_thread_id + self._dispatch(message) + return True + + if wait: + while True: + if wait_for_lock: + this_thread._event.wait(timeout.timeleft()) + + with self._lock: + if this_thread._event.is_set(): + message_available = len(this_thread._deque) != 0 + + if message_available: + remote_thread_id, message = this_thread._deque.popleft() + if len(this_thread._deque) == 0: + this_thread._event.clear() + + else: + this_thread._event.clear() + + if self._receiving: # another thread was faster + continue + + self._receiving = True + + self._thread_pool.remove(this_thread) # leave pool + break + + else: # timeout + return False + + if message_available: + this_thread._remote_thread_id = remote_thread_id + self._dispatch(message) + return True + + while True: + # from upstream + try: + message = self._channel.poll(timeout) and self._channel.recv() + + except Exception as exception: + if isinstance(exception, EOFError): + self.close() # sends close async request + + with self._lock: + self._receiving = False + + for thread in self._thread_pool: + thread._event.set() + break + + raise + + if not message: # timeout; from upstream + with self._lock: + for thread in self._thread_pool: + if not thread._event.is_set(): + self._receiving = False + thread._event.set() + break + + else: # stop receiving + self._receiving = False + + return False + + remote_thread_id, local_thread_id = brine.I4I4.unpack(message[:16]) + message = message[16:] + + this = False + + if local_thread_id == 0: # root request + if this_thread._occupation_count == 0: # this + this = True + + else: # other + new = False + + with self._lock: + for thread in self._thread_pool: + if thread._occupation_count == 0 and not thread._event.is_set(): + thread._deque.append((remote_thread_id, message)) + thread._event.set() + break + + else: + new = True + + if new: + self._thread_pool_executor.submit(self._serve_temporary, remote_thread_id, message) + + elif local_thread_id == this_thread.id: + this = True + + else: # sub request + thread = self._get_thread(id=local_thread_id) + with self._lock: + thread._deque.append((remote_thread_id, message)) + thread._event.set() + + if this: + with self._lock: + for thread in self._thread_pool: + if not thread._event.is_set(): + self._receiving = False + thread._event.set() + break + + else: # stop receiving + self._receiving = False + + this_thread._remote_thread_id = remote_thread_id + self._dispatch(message) + return True + + def _serve_temporary(self, remote_thread_id, message): + """Callable that is used to schedule serve as a new thread + - Experimental functionality `RPYC_BIND_THREADS` + + :returns: None + """ + thread = self._get_thread() + thread._deque.append((remote_thread_id, message)) + thread._event.set() + + # from upstream + try: + while not self.closed: + self.serve(None) + + if thread._occupation_count == 0: + break + + except (socket.error, select_error, IOError): + if not self.closed: + raise + except EOFError: + pass + + def _get_thread(self, id=None): + """Get internal thread information for current thread for ID, when None use current thread. + - Experimental functionality `RPYC_BIND_THREADS` + + :returns: _Thread + """ + if id is None: + id = threading.get_ident() + + thread = self._threads.get(id) + if thread is None: + thread = _Thread(id) + self._threads[id] = thread + + return thread + def poll(self, timeout=0): # serving """Serves a single transaction, should one arrives in the given interval. Note that handling a request/reply may trigger nested @@ -686,3 +914,17 @@ stop = maxint getslice = self._handle_getattr(obj, fallback) return getslice(start, stop, *args) + + +class _Thread: + """Internal thread information for the RPYC protocol used for thread binding.""" + + def __init__(self, id): + super().__init__() + + self.id = id + + self._remote_thread_id = 0 + self._occupation_count = 0 + self._event = threading.Event() + self._deque = collections.deque() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/rpyc/core/service.py new/rpyc-5.3.0/rpyc/core/service.py --- old/rpyc-5.2.3/rpyc/core/service.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/rpyc/core/service.py 2022-11-26 07:09:01.000000000 +0100 @@ -6,6 +6,7 @@ exposed *service A*, while the other may expose *service B*. As long as the two can interoperate, you're good to go. """ +import importlib.util from functools import partial from rpyc.lib import hybridmethod @@ -119,24 +120,20 @@ def __init__(self, getmodule): self.__getmodule = getmodule - self.__cache = {} + self.__cache = {m: self.__getmodule(m) for m in ('builtins', 'importlib.util')} def __contains__(self, name): - try: - self[name] - except ImportError: - return False - else: - return True + """Returns True if a module CAN be imported (the loader may still fail to execute module)""" + return self.__cache['importlib.util']._find_spec_from_path(name) is not None def __getitem__(self, name): - if type(name) is tuple: - name = ".".join(name) + """Acts as a 'read-through-cache' for results of getmodule""" if name not in self.__cache: self.__cache[name] = self.__getmodule(name) return self.__cache[name] def __getattr__(self, name): + """Provides dot notation access to modules""" try: return self[name] except ImportError: @@ -160,7 +157,9 @@ def getmodule(self, name): """imports an arbitrary module""" - return __import__(name, None, None, "*") + if type(name) is tuple: + name = ".".join(name) + return importlib.import_module(name) def getconn(self): """returns the local connection instance to the other side""" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/rpyc/lib/__init__.py new/rpyc-5.3.0/rpyc/lib/__init__.py --- old/rpyc-5.2.3/rpyc/lib/__init__.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/rpyc/lib/__init__.py 2022-11-26 07:09:01.000000000 +0100 @@ -11,6 +11,9 @@ from rpyc.lib.compat import maxint # noqa: F401 +SPAWN_THREAD_PREFIX = 'RpycSpawnThread' + + class MissingModule(object): __slots__ = ["__name"] @@ -85,7 +88,8 @@ def spawn(*args, **kwargs): """Start and return daemon thread. ``spawn(func, *args, **kwargs)``.""" func, args = args[0], args[1:] - thread = threading.Thread(target=func, args=args, kwargs=kwargs) + str_id_pack = '-'.join([f'{i}' for i in get_id_pack(func)]) + thread = threading.Thread(name=f'{SPAWN_THREAD_PREFIX}-{str_id_pack}', target=func, args=args, kwargs=kwargs) thread.daemon = True thread.start() return thread diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/rpyc/lib/compat.py new/rpyc-5.3.0/rpyc/lib/compat.py --- old/rpyc-5.2.3/rpyc/lib/compat.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/rpyc/lib/compat.py 2022-11-26 07:09:01.000000000 +0100 @@ -6,6 +6,8 @@ import time is_py_3k = (sys.version_info[0] >= 3) +is_py_gte311 = is_py_3k and (sys.version_info[1] >= 11) +is_py_gte310 = is_py_3k and (sys.version_info[1] >= 10) is_py_gte38 = is_py_3k and (sys.version_info[1] >= 8) is_py_gte37 = is_py_3k and (sys.version_info[1] >= 7) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/rpyc/utils/server.py new/rpyc-5.3.0/rpyc/utils/server.py --- old/rpyc-5.2.3/rpyc/utils/server.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/rpyc/utils/server.py 2022-11-26 07:09:01.000000000 +0100 @@ -80,7 +80,8 @@ else: family = socket.AF_INET self.listener = socket.socket(family, socket.SOCK_STREAM) - address = socket.getaddrinfo(hostname, port, family=family, type=socket.SOCK_STREAM, proto=socket.IPPROTO_TCP, flags=socket.AI_PASSIVE)[0][-1] + address = socket.getaddrinfo(hostname, port, family=family, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP, flags=socket.AI_PASSIVE)[0][-1] if reuse_addr and sys.platform != "win32": # warning: reuseaddr is not what you'd expect on windows! @@ -108,7 +109,6 @@ also unregisters from the registry server""" if self._closed: return - self._closed = True self.active = False if self.auto_register: try: @@ -128,6 +128,7 @@ pass c.close() self.clients.clear() + self._closed = True def fileno(self): """returns the listener socket's file descriptor""" @@ -341,7 +342,7 @@ self.workers = [] for i in range(self.nbthreads): t = spawn(self._serve_clients) - t.setName(f"Worker{i}") + t.name = f"Worker{i}" self.workers.append(t) # setup a thread for polling inactive connections self.polling_thread = spawn(self._poll_inactive_clients) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/rpyc/utils/teleportation.py new/rpyc-5.3.0/rpyc/utils/teleportation.py --- old/rpyc-5.2.3/rpyc/utils/teleportation.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/rpyc/utils/teleportation.py 2022-11-26 07:09:01.000000000 +0100 @@ -1,6 +1,6 @@ import opcode -from rpyc.lib.compat import is_py_gte38 +from rpyc.lib.compat import is_py_gte38, is_py_gte310, is_py_gte311 from types import CodeType, FunctionType from rpyc.core import brine, netref from dis import _unpack_opargs @@ -48,12 +48,19 @@ else: raise TypeError(f"Cannot export a function with non-brinable constants: {const!r}") - if is_py_gte38: + if is_py_gte311: + # Constructor was changed in 3.11 by adding exceptionable and qualname + exported = (cobj.co_argcount, cobj.co_posonlyargcount, cobj.co_kwonlyargcount, cobj.co_nlocals, + cobj.co_stacksize, cobj.co_flags, cobj.co_code, tuple(consts2), cobj.co_names, cobj.co_varnames, + cobj.co_filename, cobj.co_name, cobj.co_qualname, cobj.co_firstlineno, cobj.co_linetable, + cobj.co_exceptiontable, cobj.co_freevars, cobj.co_cellvars) + elif is_py_gte38: # Constructor was changed in 3.8 to support "advanced" programming styles exported = (cobj.co_argcount, cobj.co_posonlyargcount, cobj.co_kwonlyargcount, cobj.co_nlocals, cobj.co_stacksize, cobj.co_flags, cobj.co_code, tuple(consts2), cobj.co_names, cobj.co_varnames, - cobj.co_filename, cobj.co_name, cobj.co_firstlineno, cobj.co_lnotab, cobj.co_freevars, - cobj.co_cellvars) + cobj.co_filename, cobj.co_name, cobj.co_firstlineno, + cobj.co_linetable if is_py_gte310 else cobj.co_lnotab, # 3.10 switched from lnotab to linetable + cobj.co_freevars, cobj.co_cellvars) else: exported = (cobj.co_argcount, cobj.co_kwonlyargcount, cobj.co_nlocals, cobj.co_stacksize, cobj.co_flags, cobj.co_code, tuple(consts2), cobj.co_names, cobj.co_varnames, cobj.co_filename, @@ -81,14 +88,25 @@ def _import_codetup(codetup): - # Handle tuples sent from 3.8 as well as 3 < version < 3.8. - if len(codetup) == 16: + posonlyargcount = 0 + exceptiontable = b"" + qualname = "" + # Handle tuples sent from >=3.11, >=3.8 and <=3.8 + if len(codetup) == 18: + (argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, code, consts, names, varnames, + filename, name, qualname, firstlineno, lnotab, exceptiontable, freevars, cellvars) = codetup + if not is_py_gte310: + # Since version, codetup length, and lnotab length, lnotab is set as though there is no linenumber + lnotab = b'' # lnotab_notes.txt describes lnotab as imperfect anyway + elif len(codetup) == 16: (argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, lnotab, freevars, cellvars) = codetup + if not is_py_gte310 and len(lnotab) > 2: + # Since version, codetup length, and lnotab length, lnotab is set as though there is no linenumber + lnotab = b'' # lnotab_notes.txt describes lnotab as imperfect anyway else: (argcount, kwonlyargcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, lnotab, freevars, cellvars) = codetup - posonlyargcount = 0 consts2 = [] for const in consts: @@ -97,7 +115,10 @@ else: consts2.append(const) consts = tuple(consts2) - if is_py_gte38: + if is_py_gte311: + codetup = (argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, code, consts, names, varnames, + filename, name, qualname, firstlineno, lnotab, exceptiontable, freevars, cellvars) + elif is_py_gte38: codetup = (argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, lnotab, freevars, cellvars) else: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/rpyc/version.py new/rpyc-5.3.0/rpyc/version.py --- old/rpyc-5.2.3/rpyc/version.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/rpyc/version.py 2022-11-26 07:09:01.000000000 +0100 @@ -1,3 +1,3 @@ -__version__ = '5.2.3' +__version__ = '5.3.0' version = tuple(__version__.split('.')) -release_date = "2022-08-03" +release_date = "2022-11-25" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/tests/test_attr_access.py new/rpyc-5.3.0/tests/test_attr_access.py --- old/rpyc-5.2.3/tests/test_attr_access.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/tests/test_attr_access.py 2022-11-26 07:09:01.000000000 +0100 @@ -218,6 +218,7 @@ class TestDescriptorErrors(unittest.TestCase): """Validate stack traces are consistent independent of how exposed attribute is accessed #478 #479""" + def setUp(self): self.cfg = copy.copy(rpyc.core.protocol.DEFAULT_CONFIG) self.server = ThreadedServer(MyDecoratedService(), port=0) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/tests/test_classic.py new/rpyc-5.3.0/tests/test_classic.py --- old/rpyc-5.2.3/tests/test_classic.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/tests/test_classic.py 2022-11-26 07:09:01.000000000 +0100 @@ -51,6 +51,11 @@ self.assertTrue(conn.builtin.open is open) self.assertEqual(conn.eval("2+3"), 5) + def test_modules(self): + self.assertIn('test_magic', self.conn.modules) + self.assertNotIn('test_badmagic', self.conn.modules) + self.assertIsNone(self.conn.builtins.locals()['self']._last_traceback) + if __name__ == "__main__": unittest.main() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/tests/test_custom_service.py new/rpyc-5.3.0/tests/test_custom_service.py --- old/rpyc-5.2.3/tests/test_custom_service.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/tests/test_custom_service.py 2022-11-26 07:09:01.000000000 +0100 @@ -93,6 +93,8 @@ def tearDown(self): if not self.conn.closed: self.conn.close() + if not self.prefixed_conn.closed: + self.prefixed_conn.close() time.sleep(0.5) # this will wait a little, making sure # on_disconnect_called is already True self.assertTrue(self.service.on_disconnect_called) @@ -122,7 +124,7 @@ self.prefixed_conn.root.prefix_get_decorated_prefix self.assertFalse(hasattr(self.conn.root, 'get_decorated_prefix')) smc = self.conn.root.MyClass('a', 'b') - self.assertEquals(smc.foo(), 'ab') + self.assertEqual(smc.foo(), 'ab') def test_safeattrs(self): x = self.conn.root.getlist() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/tests/test_dataclass.py new/rpyc-5.3.0/tests/test_dataclass.py --- old/rpyc-5.2.3/tests/test_dataclass.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/tests/test_dataclass.py 2022-11-26 07:09:01.000000000 +0100 @@ -4,6 +4,7 @@ from rpyc.lib.compat import is_py_gte37 if is_py_gte37: from dataclasses import dataclass + @dataclass class StdTypes(object): exposed_intObj: int = 31 @@ -20,6 +21,7 @@ def exposed_create_dataclass(self): return StdTypes() + @unittest.skipUnless(is_py_gte37, "Skipping since dataclasses is only in 3.7 and above") class TestRemoteDataclass(unittest.TestCase): def setUp(self): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/tests/test_deploy.py new/rpyc-5.3.0/tests/test_deploy.py --- old/rpyc-5.2.3/tests/test_deploy.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/tests/test_deploy.py 2022-11-26 07:09:01.000000000 +0100 @@ -23,6 +23,7 @@ print(conn.modules.sys) func = conn.modules.os.getcwd print(func()) + conn.close() try: func() @@ -46,14 +47,15 @@ rem = SshMachine("localhost") SshMachine.python = rem[sys.executable] dep = DeployedServer(rem) - dep.classic_connect() + conn = dep.classic_connect() dep.close(timeout=expected_timeout) rem.close() + conn.close() finally: subprocess.Popen.communicate = original_communicate # The last three calls to communicate() happen during close(), so check they # applied the timeout. - assert observed_timeouts[-3:] == [expected_timeout] * 3 + self.assertEqual(observed_timeouts[-3:], [expected_timeout] * 3) def test_close_timeout_default_none(self): observed_timeouts = [] @@ -68,13 +70,14 @@ rem = SshMachine("localhost") SshMachine.python = rem[sys.executable] dep = DeployedServer(rem) - dep.classic_connect() + conn = dep.classic_connect() dep.close() rem.close() + conn.close() finally: subprocess.Popen.communicate = original_communicate # No timeout specified, so Popen.communicate should have been called with timeout None. - assert observed_timeouts == [None] * len(observed_timeouts) + self.assertEqual(observed_timeouts, [None] * len(observed_timeouts)) @unittest.skipIf(_paramiko_import_failed, "Paramiko is not available") def test_deploy_paramiko(self): @@ -84,6 +87,7 @@ print(conn.modules.sys) func = conn.modules.os.getcwd print(func()) + conn.close() try: func() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/tests/test_get_id_pack.py new/rpyc-5.3.0/tests/test_get_id_pack.py --- old/rpyc-5.2.3/tests/test_get_id_pack.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/tests/test_get_id_pack.py 2022-11-26 07:09:01.000000000 +0100 @@ -23,7 +23,7 @@ cls.chained_conn.close() cls.conn.close() while cls.server2.clients or cls.server.clients: - pass #sti + pass # sti cls.server2.close() cls.server.close() cls.thd.join() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/tests/test_oneshot_server.py new/rpyc-5.3.0/tests/test_oneshot_server.py --- old/rpyc-5.2.3/tests/test_oneshot_server.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/tests/test_oneshot_server.py 2022-11-26 07:09:01.000000000 +0100 @@ -27,8 +27,8 @@ while not self.server._closed: pass self.assertTrue(self.server._closed) - with self.assertRaises(Exception): - conn = rpyc.connect("localhost", port=18878) + self.assertTrue(self.server.listener._closed) + self.assertEqual(len(self.server.clients), 0) if __name__ == "__main__": diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/tests/test_registry.py new/rpyc-5.3.0/tests/test_registry.py --- old/rpyc-5.2.3/tests/test_registry.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/tests/test_registry.py 2022-11-26 07:09:01.000000000 +0100 @@ -76,6 +76,7 @@ expected = ("FOO",) self.assertEqual(set(res), set(expected)) + class TestTcpRegistry(BaseRegistryTest, unittest.TestCase): def _get_server(self): return TCPRegistryServer(pruning_timeout=PRUNING_TIMEOUT, allow_listing=True) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/tests/test_ssh.py new/rpyc-5.3.0/tests/test_ssh.py --- old/rpyc-5.2.3/tests/test_ssh.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/tests/test_ssh.py 2022-11-26 07:09:01.000000000 +0100 @@ -29,9 +29,9 @@ # User <username> # IdentityFile <id_rsa> cls.server = ThreadedServer(SlaveService, hostname="localhost", - ipv6=False, port=18888, auto_register=False) + ipv6=False, port=18888, auto_register=False) cls.server._start_in_thread() - cls.remote_machine = SshMachine("localhost") + cls.remote_machine = SshMachine("localhost") cls.conn = rpyc.classic.ssh_connect(cls.remote_machine, 18888) cls.conn2 = rpyc.ssh_connect(cls.remote_machine, 18888, service=MasterService) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/rpyc-5.2.3/tests/test_teleportation.py new/rpyc-5.3.0/tests/test_teleportation.py --- old/rpyc-5.2.3/tests/test_teleportation.py 2022-08-04 05:46:31.000000000 +0200 +++ new/rpyc-5.3.0/tests/test_teleportation.py 2022-11-26 07:09:01.000000000 +0100 @@ -1,4 +1,7 @@ from __future__ import with_statement +from rpyc.utils.classic import teleport_function +from rpyc.lib.compat import is_py_3k, is_py_gte38, is_py_gte311 +from rpyc.utils.teleportation import export_function, import_function import subprocess import sys import os @@ -8,10 +11,6 @@ import tracemalloc tracemalloc.start() -from rpyc.utils.teleportation import export_function, import_function -from rpyc.lib.compat import is_py_3k, is_py_gte38 -from rpyc.utils.classic import teleport_function - def b(st): if sys.version_info[0] >= 3: @@ -112,9 +111,16 @@ cobj.co_varnames, cobj.co_filename, cobj.co_name, cobj.co_firstlineno, cobj.co_lnotab, cobj.co_freevars, cobj.co_cellvars) - if is_py_3k: - pow37 = lambda x, y : x ** y # noqa - pow38 = lambda x, y : x ** y # noqa + def get311_schema(cobj): + return (cobj.co_argcount, 2, cobj.co_kwonlyargcount, cobj.co_nlocals, + cobj.co_stacksize, cobj.co_flags, cobj.co_code, cobj.co_consts, cobj.co_names, + cobj.co_varnames, cobj.co_filename, cobj.co_name, cobj.co_qualname, + cobj.co_firstlineno, cobj.co_lnotab, cobj.co_exceptiontable, + cobj.co_freevars, cobj.co_cellvars) + + if is_py_gte38: + def pow37(x, y): return x ** y # noqa + def pow38(x, y): return x ** y # noqa export37 = get37_schema(pow37.__code__) export38 = get38_schema(pow38.__code__) schema37 = (pow37.__name__, pow37.__module__, pow37.__defaults__, pow37.__kwdefaults__, export37) @@ -124,13 +130,23 @@ self.assertEqual(pow37_netref(2, 3), pow37(2, 3)) self.assertEqual(pow38_netref(2, 3), pow38(2, 3)) self.assertEqual(pow37_netref(x=2, y=3), pow37(x=2, y=3)) - if not is_py_gte38: - return # skip remained of tests for 3.7 - pow38.__code__ = types.CodeType(*export38) # pow38 = lambda x, y, /: x ** y - with self.assertRaises(TypeError): # show local behavior - pow38(x=2, y=3) - with self.assertRaises(TypeError): - pow38_netref(x=2, y=3) + if is_py_gte38 and not is_py_gte311: + pow38.__code__ = types.CodeType(*export38) # pow38 = lambda x, y, /: x ** y + with self.assertRaises(TypeError): # show local behavior + pow38(x=2, y=3) + with self.assertRaises(TypeError): + pow38_netref(x=2, y=3) + if is_py_gte311: + def pow311(x, y): return x ** y # noqa + export311 = get311_schema(pow311.__code__) + schema311 = (pow311.__name__, pow311.__module__, pow311.__defaults__, pow311.__kwdefaults__, export311) + pow311_netref = self.conn.modules["rpyc.utils.teleportation"].import_function(schema311) + self.assertTrue(is_py_gte311) + pow311.__code__ = types.CodeType(*export311) # pow311 = lambda x, y, /: x ** y + with self.assertRaises(TypeError): # show local behavior + pow311(x=2, y=3) + with self.assertRaises(TypeError): + pow311_netref(x=2, y=3) if __name__ == "__main__":