Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package python-portpicker for openSUSE:Factory checked in at 2022-01-16 23:18:19 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-portpicker (Old) and /work/SRC/openSUSE:Factory/.python-portpicker.new.1892 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-portpicker" Sun Jan 16 23:18:19 2022 rev:4 rq:946778 version:1.5.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-portpicker/python-portpicker.changes 2020-05-19 14:49:21.976187444 +0200 +++ /work/SRC/openSUSE:Factory/.python-portpicker.new.1892/python-portpicker.changes 2022-01-16 23:19:17.702378367 +0100 @@ -1,0 +2,6 @@ +Sun Jan 16 12:43:43 UTC 2022 - Dirk M??ller <dmuel...@suse.com> + +- update to to 1.5.0: + * python 3.10 support + +------------------------------------------------------------------- Old: ---- portpicker-1.3.1.tar.gz New: ---- portpicker-1.5.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-portpicker.spec ++++++ --- /var/tmp/diff_new_pack.Ufccwa/_old 2022-01-16 23:19:19.022379016 +0100 +++ /var/tmp/diff_new_pack.Ufccwa/_new 2022-01-16 23:19:19.034379022 +0100 @@ -1,7 +1,7 @@ # # spec file for package python-portpicker # -# Copyright (c) 2020 SUSE LLC +# Copyright (c) 2022 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -18,7 +18,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-portpicker -Version: 1.3.1 +Version: 1.5.0 Release: 0 Summary: A library to choose unique available network ports License: Apache-2.0 @@ -27,8 +27,9 @@ Source0: https://files.pythonhosted.org/packages/source/p/portpicker/portpicker-%{version}.tar.gz BuildRequires: %{python_module setuptools} BuildRequires: fdupes +BuildRequires: python-rpm-macros Requires(post): update-alternatives -Requires(postun): update-alternatives +Requires(postun):update-alternatives BuildArch: noarch # SECTION test requirements BuildRequires: %{python_module mock} @@ -42,6 +43,7 @@ %prep %setup -q -n portpicker-%{version} +test -f setup.py || echo "import setuptools; setuptools.setup()" > setup.py %build %python_build ++++++ portpicker-1.3.1.tar.gz -> portpicker-1.5.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/ChangeLog.md new/portpicker-1.5.0/ChangeLog.md --- old/portpicker-1.3.1/ChangeLog.md 2019-03-05 17:20:26.000000000 +0100 +++ new/portpicker-1.5.0/ChangeLog.md 2021-11-09 03:17:28.000000000 +0100 @@ -1,37 +1,59 @@ +## 1.5.0 + +* Add portserver support to Windows using named pipes. To create or connect to + a server, prefix the name of the server with `@` (e.g. + `@unittest-portserver`). + +## 1.4.0 + +* Use `async def` instead of `@asyncio.coroutine` in order to support 3.10. +* The portserver now checks for and rejects pid values that are out of range. +* Declare a minimum Python version of 3.6 in the package config. +* Rework `portserver_test.py` to launch an actual portserver process instead + of mocks. + +## 1.3.9 + +* No portpicker or portserver code changes +* Fixed the portserver test on recent Python 3.x versions. +* Switched to setup.cfg based packaging. +* We no longer declare ourselves Python 2.7 or 3.3-3.5 compatible. + ## 1.3.1 - * Fix a race condition in `pick_unused_port()` involving the free ports set. +* Fix a race condition in `pick_unused_port()` involving the free ports set. ## 1.3.0 -* Adds an optional `portserver_address` parameter to `pick_unused_port()` so - that callers can specify their own regardless of `os.environ`. -* `pick_unused_port()` now raises `NoFreePortFoundError` when no available port - could be found rather than spinning in a loop trying forever. -* Fall back to `socket.AF_INET` when `socket.AF_UNIX` support is not available - to communicate with a portserver. +* Adds an optional `portserver_address` parameter to `pick_unused_port()` so + that callers can specify their own regardless of `os.environ`. +* `pick_unused_port()` now raises `NoFreePortFoundError` when no available + port could be found rather than spinning in a loop trying forever. +* Fall back to `socket.AF_INET` when `socket.AF_UNIX` support is not available + to communicate with a portserver. ## 1.2.0 -* Introduced `add_reserved_port()` and `return_port()` APIs to allow ports to - be recycled and allow users to bring ports of their own. +* Introduced `add_reserved_port()` and `return_port()` APIs to allow ports to + be recycled and allow users to bring ports of their own. ## 1.1.1 -* Changed default port range to 15000-24999 to avoid ephemeral ports. -* Portserver bugfix. +* Changed default port range to 15000-24999 to avoid ephemeral ports. +* Portserver bugfix. ## 1.1.0 -* Renamed portpicker APIs to use PEP8 style function names in code and docs. -* Legacy CapWords API name compatibility is maintained (and explicitly tested). +* Renamed portpicker APIs to use PEP8 style function names in code and docs. +* Legacy CapWords API name compatibility is maintained (and explicitly + tested). ## 1.0.1 -* Code reindented to use 4 space indents and run through - [YAPF](https://github.com/google/yapf) for consistent style. -* Not packaged for release. +* Code reindented to use 4 space indents and run through + [YAPF](https://github.com/google/yapf) for consistent style. +* Not packaged for release. ## 1.0.0 -* Original open source release. +* Original open source release. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/MANIFEST.in new/portpicker-1.5.0/MANIFEST.in --- old/portpicker-1.3.1/MANIFEST.in 2017-10-09 19:32:19.000000000 +0200 +++ new/portpicker-1.5.0/MANIFEST.in 2021-05-25 00:49:41.000000000 +0200 @@ -6,3 +6,4 @@ include ChangeLog.md include setup.py include test.sh +exclude package.sh diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/PKG-INFO new/portpicker-1.5.0/PKG-INFO --- old/portpicker-1.3.1/PKG-INFO 2019-03-05 17:54:53.000000000 +0100 +++ new/portpicker-1.5.0/PKG-INFO 2021-11-09 03:19:26.347873200 +0100 @@ -1,27 +1,34 @@ -Metadata-Version: 1.1 +Metadata-Version: 2.1 Name: portpicker -Version: 1.3.1 +Version: 1.5.0 Summary: A library to choose unique available network ports. Home-page: https://github.com/google/python_portpicker -Author: Google -Author-email: g...@krypto.org +Maintainer: Google LLC +Maintainer-email: g...@krypto.org License: Apache 2.0 -Description-Content-Type: UNKNOWN -Description: Portpicker provides an API to find and return an available network - port for an application to bind to. Ideally suited for use from - unittests or for test harnesses that launch local servers. Platform: POSIX +Platform: Windows Classifier: Development Status :: 5 - Production/Stable Classifier: License :: OSI Approved :: Apache Software License Classifier: Intended Audience :: Developers Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: Jython Classifier: Programming Language :: Python :: Implementation :: PyPy +Requires-Python: >=3.6 +License-File: LICENSE + +Portpicker provides an API to find and return an available +network port for an application to bind to. Ideally suited for use from +unittests or for test harnesses that launch local servers. + +It also contains an optional portserver that can be used to coordinate +allocation of network ports on a single build/test farm host across all +processes willing to use a port server aware port picker library such as +this one. + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/README.md new/portpicker-1.5.0/README.md --- old/portpicker-1.3.1/README.md 2019-01-15 22:11:37.000000000 +0100 +++ new/portpicker-1.5.0/README.md 2021-05-25 00:49:41.000000000 +0200 @@ -1,18 +1,22 @@ # Python portpicker module -This module is useful for finding unused network ports on a host. -It supports both Python 2 and Python 3. +[](https://badge.fury.io/py/portpicker) + +[](https://travis-ci.org/google/python_portpicker) -This module provides a pure Python `pick_unused_port()` function. -It can also be called via the command line for use in shell scripts. +This module is useful for finding unused network ports on a host. If you need +legacy Python 2 support, use the 1.3.x releases. + +This module provides a pure Python `pick_unused_port()` function. It can also be +called via the command line for use in shell scripts. If your code can accept a bound TCP socket rather than a port number consider using `socket.bind(('localhost', 0))` to bind atomically to an available port rather than using this library at all. There is a race condition between picking a port and your application code -binding to it. The use of a port server by all of your test code to avoid -that problem is recommended on loaded test hosts running many tests at a time. +binding to it. The use of a port server by all of your test code to avoid that +problem is recommended on loaded test hosts running many tests at a time. Unless you are using a port server, subsequent calls to `pick_unused_port()` to obtain an additional port are not guaranteed to return a unique port. @@ -20,22 +24,22 @@ ### What is the optional port server? A port server is intended to be run as a daemon, for use by all processes -running on the host. It coordinates uses of network ports by anything using -a portpicker library. If you are using hosts as part of a test automation -cluster, each one should run a port server as a daemon. You should set the +running on the host. It coordinates uses of network ports by anything using a +portpicker library. If you are using hosts as part of a test automation cluster, +each one should run a port server as a daemon. You should set the `PORTSERVER_ADDRESS=@unittest-portserver` environment variable on all of your test runners so that portpicker makes use of it. -A sample port server is included. This portserver implementation works but has -not spent time in production. If you use it with good results please report -back so that this statement can be updated to reflect that. :) - -A port server listens on a unix socket, reads a pid from a new connection, -tests the ports it is managing and replies with a port assignment port for that -pid. A port is only reclaimed for potential reassignment to another process -after the process it was originally assigned to has died. Processes that need -multiple ports can simply issue multiple requests and are guaranteed they will -each be unique. +A sample port server is included. This portserver implementation works but has +not spent time in production. If you use it with good results please report back +so that this statement can be updated to reflect that. :) + +A port server listens on a unix socket, reads a pid from a new connection, tests +the ports it is managing and replies with a port assignment port for that pid. A +port is only reclaimed for potential reassignment to another process after the +process it was originally assigned to has died. Processes that need multiple +ports can simply issue multiple requests and are guaranteed they will each be +unique. ## Typical usage: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/pyproject.toml new/portpicker-1.5.0/pyproject.toml --- old/portpicker-1.3.1/pyproject.toml 1970-01-01 01:00:00.000000000 +0100 +++ new/portpicker-1.5.0/pyproject.toml 2021-05-25 00:49:41.000000000 +0200 @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools >= 40.9.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.tox] +legacy_tox_ini = """ +[tox] +envlist = py{36,37,38,39} +isolated_build = true +skip_missing_interpreters = true +# minimum tox version +minversion = 3.3.0 +[testenv] +deps = + check-manifest >= 0.42 + pytest +commands = + check-manifest --ignore 'src/tests/**' + python -c 'from setuptools import setup; setup()' check -m -s + py.test -s {posargs} +""" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/setup.cfg new/portpicker-1.5.0/setup.cfg --- old/portpicker-1.3.1/setup.cfg 2019-03-05 17:54:53.000000000 +0100 +++ new/portpicker-1.5.0/setup.cfg 2021-11-09 03:19:26.347873200 +0100 @@ -1,3 +1,44 @@ +[metadata] +name = portpicker +version = 1.5.0 +maintainer = Google LLC +maintainer_email = g...@krypto.org +license = Apache 2.0 +license_files = LICENSE +description = A library to choose unique available network ports. +url = https://github.com/google/python_portpicker +long_description = Portpicker provides an API to find and return an available + network port for an application to bind to. Ideally suited for use from + unittests or for test harnesses that launch local servers. + + It also contains an optional portserver that can be used to coordinate + allocation of network ports on a single build/test farm host across all + processes willing to use a port server aware port picker library such as + this one. +classifiers = + Development Status :: 5 - Production/Stable + License :: OSI Approved :: Apache Software License + Intended Audience :: Developers + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy +platforms = POSIX, Windows +requires = + +[options] +install_requires = psutil +python_requires = >= 3.6 +package_dir = + =src +py_modules = portpicker +scripts = src/portserver.py + [egg_info] tag_build = tag_date = 0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/setup.py new/portpicker-1.5.0/setup.py --- old/portpicker-1.3.1/setup.py 2019-03-05 17:23:51.000000000 +0100 +++ new/portpicker-1.5.0/setup.py 1970-01-01 01:00:00.000000000 +0100 @@ -1,70 +0,0 @@ -# Copyright 2015 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Simple distutils setup for the pure Python portpicker.""" - -import sys -import textwrap - - -import setuptools - - -def main(): - requires = [] - scripts = [] - py_version = sys.version_info[:2] - if py_version < (3, 3): - requires.append('mock(>=1.0)') - if py_version == (3, 3): - requires.append('asyncio(>=3.4)') - if py_version >= (3, 3): - # The example portserver implementation requires Python 3 and asyncio. - scripts.append('src/portserver.py') - - setuptools.setup( - name='portpicker', - version='1.3.1', - description='A library to choose unique available network ports.', - long_description=textwrap.dedent("""\ - Portpicker provides an API to find and return an available network - port for an application to bind to. Ideally suited for use from - unittests or for test harnesses that launch local servers."""), - license='Apache 2.0', - maintainer='Google', - maintainer_email='g...@krypto.org', - url='https://github.com/google/python_portpicker', - package_dir={'': 'src'}, - py_modules=['portpicker'], - platforms=['POSIX'], - requires=requires, - scripts=scripts, - classifiers= - ['Development Status :: 5 - Production/Stable', - 'License :: OSI Approved :: Apache Software License', - 'Intended Audience :: Developers', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: Jython', - 'Programming Language :: Python :: Implementation :: PyPy']) - - -if __name__ == '__main__': - main() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/src/portpicker.egg-info/PKG-INFO new/portpicker-1.5.0/src/portpicker.egg-info/PKG-INFO --- old/portpicker-1.3.1/src/portpicker.egg-info/PKG-INFO 2019-03-05 17:54:53.000000000 +0100 +++ new/portpicker-1.5.0/src/portpicker.egg-info/PKG-INFO 2021-11-09 03:19:26.000000000 +0100 @@ -1,27 +1,34 @@ -Metadata-Version: 1.1 +Metadata-Version: 2.1 Name: portpicker -Version: 1.3.1 +Version: 1.5.0 Summary: A library to choose unique available network ports. Home-page: https://github.com/google/python_portpicker -Author: Google -Author-email: g...@krypto.org +Maintainer: Google LLC +Maintainer-email: g...@krypto.org License: Apache 2.0 -Description-Content-Type: UNKNOWN -Description: Portpicker provides an API to find and return an available network - port for an application to bind to. Ideally suited for use from - unittests or for test harnesses that launch local servers. Platform: POSIX +Platform: Windows Classifier: Development Status :: 5 - Production/Stable Classifier: License :: OSI Approved :: Apache Software License Classifier: Intended Audience :: Developers Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: Jython Classifier: Programming Language :: Python :: Implementation :: PyPy +Requires-Python: >=3.6 +License-File: LICENSE + +Portpicker provides an API to find and return an available +network port for an application to bind to. Ideally suited for use from +unittests or for test harnesses that launch local servers. + +It also contains an optional portserver that can be used to coordinate +allocation of network ports on a single build/test farm host across all +processes willing to use a port server aware port picker library such as +this one. + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/src/portpicker.egg-info/SOURCES.txt new/portpicker-1.5.0/src/portpicker.egg-info/SOURCES.txt --- old/portpicker-1.3.1/src/portpicker.egg-info/SOURCES.txt 2019-03-05 17:54:53.000000000 +0100 +++ new/portpicker-1.5.0/src/portpicker.egg-info/SOURCES.txt 2021-11-09 03:19:26.000000000 +0100 @@ -3,13 +3,15 @@ LICENSE MANIFEST.in README.md -setup.py +pyproject.toml +setup.cfg test.sh src/portpicker.py src/portserver.py src/portpicker.egg-info/PKG-INFO src/portpicker.egg-info/SOURCES.txt src/portpicker.egg-info/dependency_links.txt +src/portpicker.egg-info/requires.txt src/portpicker.egg-info/top_level.txt src/tests/portpicker_test.py src/tests/portserver_test.py \ No newline at end of file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/src/portpicker.egg-info/requires.txt new/portpicker-1.5.0/src/portpicker.egg-info/requires.txt --- old/portpicker-1.3.1/src/portpicker.egg-info/requires.txt 1970-01-01 01:00:00.000000000 +0100 +++ new/portpicker-1.5.0/src/portpicker.egg-info/requires.txt 2021-11-09 03:19:26.000000000 +0100 @@ -0,0 +1 @@ +psutil diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/src/portpicker.py new/portpicker-1.5.0/src/portpicker.py --- old/portpicker-1.3.1/src/portpicker.py 2019-03-05 17:20:26.000000000 +0100 +++ new/portpicker-1.5.0/src/portpicker.py 2021-07-12 03:38:42.000000000 +0200 @@ -43,6 +43,11 @@ import socket import sys +if sys.platform == 'win32': + import _winapi +else: + _winapi = None + # The legacy Bind, IsPortFree, etc. names are not exported. __all__ = ('bind', 'is_port_free', 'pick_unused_port', 'return_port', 'add_reserved_port', 'get_port_from_port_server') @@ -63,7 +68,6 @@ class NoFreePortFoundError(Exception): """Exception indicating that no free port could be found.""" - pass def add_reserved_port(port): @@ -217,6 +221,61 @@ raise NoFreePortFoundError() +def _get_linux_port_from_port_server(portserver_address, pid): + # An AF_UNIX address may start with a zero byte, in which case it is in the + # "abstract namespace", and doesn't have any filesystem representation. + # See 'man 7 unix' for details. + # The convention is to write '@' in the address to represent this zero byte. + if portserver_address[0] == '@': + portserver_address = '\0' + portserver_address[1:] + + try: + # Create socket. + if hasattr(socket, 'AF_UNIX'): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # pylint: disable=no-member + else: + # fallback to AF_INET if this is not unix + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + # Connect to portserver. + sock.connect(portserver_address) + + # Write request. + sock.sendall(('%d\n' % pid).encode('ascii')) + + # Read response. + # 1K should be ample buffer space. + return sock.recv(1024) + finally: + sock.close() + except socket.error as error: + print('Socket error when connecting to portserver:', error, + file=sys.stderr) + return None + + +def _get_windows_port_from_port_server(portserver_address, pid): + if portserver_address[0] == '@': + portserver_address = '\\\\.\\pipe\\' + portserver_address[1:] + + try: + handle = _winapi.CreateFile( + portserver_address, + _winapi.GENERIC_READ | _winapi.GENERIC_WRITE, + 0, + 0, + _winapi.OPEN_EXISTING, + 0, + 0) + + _winapi.WriteFile(handle, ('%d\n' % pid).encode('ascii')) + data, _ = _winapi.ReadFile(handle, 6, 0) + return data + except FileNotFoundError as error: + print('File error when connecting to portserver:', error, + file=sys.stderr) + return None + def get_port_from_port_server(portserver_address, pid=None): """Request a free a port from a system-wide portserver. @@ -240,38 +299,16 @@ """ if not portserver_address: return None - # An AF_UNIX address may start with a zero byte, in which case it is in the - # "abstract namespace", and doesn't have any filesystem representation. - # See 'man 7 unix' for details. - # The convention is to write '@' in the address to represent this zero byte. - if portserver_address[0] == '@': - portserver_address = '\0' + portserver_address[1:] if pid is None: pid = os.getpid() - try: - # Create socket. - if hasattr(socket, 'AF_UNIX'): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - else: - # fallback to AF_INET if this is not unix - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - # Connect to portserver. - sock.connect(portserver_address) - - # Write request. - sock.sendall(('%d\n' % pid).encode('ascii')) + if _winapi: + buf = _get_windows_port_from_port_server(portserver_address, pid) + else: + buf = _get_linux_port_from_port_server(portserver_address, pid) - # Read response. - # 1K should be ample buffer space. - buf = sock.recv(1024) - finally: - sock.close() - except socket.error as e: - print('Socket error when connecting to portserver:', e, - file=sys.stderr) + if buf is None: return None try: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/src/portserver.py new/portpicker-1.5.0/src/portserver.py --- old/portpicker-1.3.1/src/portserver.py 2017-10-09 19:32:19.000000000 +0200 +++ new/portpicker-1.5.0/src/portserver.py 2021-07-12 03:38:42.000000000 +0200 @@ -31,10 +31,12 @@ import asyncio import collections import logging -import os import signal import socket import sys +import psutil +import subprocess +from datetime import datetime, timezone, timedelta log = None # Initialized to a logging.Logger by _configure_logging(). @@ -44,18 +46,16 @@ def _get_process_command_line(pid): try: - with open('/proc/{}/cmdline'.format(pid), 'rt') as cmdline_f: - return cmdline_f.read() - except IOError: + return psutil.Process(pid).cmdline() + except psutil.NoSuchProcess: return '' def _get_process_start_time(pid): try: - with open('/proc/{}/stat'.format(pid), 'rt') as pid_stat_f: - return int(pid_stat_f.readline().split()[21]) - except IOError: - return 0 + return psutil.Process(pid).create_time() + except psutil.NoSuchProcess: + return 0.0 # TODO: Consider importing portpicker.bind() instead of duplicating the code. @@ -115,14 +115,27 @@ # had been reparented to init. log.info('Not allocating a port to init.') return False - try: - os.kill(pid, 0) - except ProcessLookupError: + + if not psutil.pid_exists(pid): log.info('Not allocating a port to a non-existent process') return False return True +async def _start_windows_server(client_connected_cb, path): + """Start the server on Windows using named pipes.""" + def protocol_factory(): + stream_reader = asyncio.StreamReader() + stream_reader_protocol = asyncio.StreamReaderProtocol( + stream_reader, client_connected_cb) + return stream_reader_protocol + + loop = asyncio.get_event_loop() + server, *_ = await loop.start_serving_pipe(protocol_factory, address=path) + + return server + + class _PortInfo(object): """Container class for information about a given port assignment. @@ -137,7 +150,7 @@ def __init__(self, port): self.port = port self.pid = 0 - self.start_time = 0 + self.start_time = 0.0 class _PortPool(object): @@ -178,7 +191,7 @@ candidate = self._port_queue.pop() self._port_queue.appendleft(candidate) check_count += 1 - if (candidate.start_time == 0 or + if (candidate.start_time == 0.0 or candidate.start_time != _get_process_start_time(candidate.pid)): if _is_port_free(candidate.port): candidate.pid = pid @@ -227,9 +240,8 @@ for port in ports_to_serve: self._port_pool.add_port_to_free_pool(port) - @asyncio.coroutine - def handle_port_request(self, reader, writer): - client_data = yield from reader.read(100) + async def handle_port_request(self, reader, writer): + client_data = await reader.read(100) self._handle_port_request(client_data, writer) writer.close() @@ -241,6 +253,8 @@ writer: The asyncio Writer for the response to be written to. """ try: + if len(client_data) > 20: + raise ValueError('More than 20 characters in "pid".') pid = int(client_data) except ValueError as error: self._client_request_errors += 1 @@ -286,10 +300,13 @@ default='15000-24999', help='Comma separated N-P Range(s) of ports to manage (inclusive).') parser.add_argument( - '--portserver_unix_socket_address', + '--portserver_address', + '--portserver_unix_socket_address', # Alias to be backward compatible type=str, default='@unittest-portserver', - help='Address of AF_UNIX socket on which to listen (first @ is a NUL).') + help='Address of AF_UNIX socket on which to listen on Unix (first @ is ' + 'a NUL) or the name of the pipe on Windows (first @ is the ' + r'\\.\pipe\ prefix).') parser.add_argument('--verbose', action='store_true', default=False, @@ -347,13 +364,33 @@ request_handler = _PortServerRequestHandler(ports_to_serve) + if sys.platform == 'win32': + asyncio.set_event_loop(asyncio.ProactorEventLoop()) + event_loop = asyncio.get_event_loop() - event_loop.add_signal_handler(signal.SIGUSR1, request_handler.dump_stats) - coro = asyncio.start_unix_server( - request_handler.handle_port_request, - path=config.portserver_unix_socket_address.replace('@', '\0', 1), - loop=event_loop) - server_address = config.portserver_unix_socket_address + + if sys.platform == 'win32': + # On Windows, we need to periodically pause the loop to allow the user + # to send a break signal (e.g. ctrl+c) + def listen_for_signal(): + event_loop.call_later(0.5, listen_for_signal) + + event_loop.call_later(0.5, listen_for_signal) + + coro = _start_windows_server( + request_handler.handle_port_request, + path=config.portserver_address.replace('@', '\\\\.\\pipe\\', 1)) + else: + event_loop.add_signal_handler( + signal.SIGUSR1, request_handler.dump_stats) # pylint: disable=no-member + + old_py_loop = {'loop': event_loop} if sys.version_info < (3, 10) else {} + coro = asyncio.start_unix_server( + request_handler.handle_port_request, + path=config.portserver_address.replace('@', '\0', 1), + **old_py_loop) + + server_address = config.portserver_address server = event_loop.run_until_complete(coro) log.info('Serving on %s', server_address) @@ -363,8 +400,12 @@ log.info('Stopping due to ^C.') server.close() - event_loop.run_until_complete(server.wait_closed()) - event_loop.remove_signal_handler(signal.SIGUSR1) + + if sys.platform != 'win32': + # PipeServer doesn't have a wait_closed() function + event_loop.run_until_complete(server.wait_closed()) + event_loop.remove_signal_handler(signal.SIGUSR1) # pylint: disable=no-member + event_loop.close() request_handler.dump_stats() log.info('Goodbye.') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/src/tests/portpicker_test.py new/portpicker-1.5.0/src/tests/portpicker_test.py --- old/portpicker-1.3.1/src/tests/portpicker_test.py 2019-01-18 02:31:24.000000000 +0100 +++ new/portpicker-1.5.0/src/tests/portpicker_test.py 2021-07-12 03:38:42.000000000 +0200 @@ -17,11 +17,18 @@ """Unittests for the portpicker module.""" from __future__ import print_function +import errno import os import random import socket import sys import unittest +from contextlib import ExitStack + +if sys.platform == 'win32': + import _winapi +else: + _winapi = None try: # pylint: disable=no-name-in-module @@ -99,27 +106,82 @@ self.assertTrue(self.IsUnusedUDPPort(port)) def testSendsPidToPortServer(self): - server = mock.Mock() - server.recv.return_value = b'42768\n' - with mock.patch.object(socket, 'socket', return_value=server): - port = portpicker.get_port_from_port_server('portserver', pid=1234) - server.sendall.assert_called_once_with(b'1234\n') + with ExitStack() as stack: + if _winapi: + create_file_mock = mock.Mock() + create_file_mock.return_value = 0 + read_file_mock = mock.Mock() + write_file_mock = mock.Mock() + read_file_mock.return_value = (b'42768\n', 0) + stack.enter_context( + mock.patch('_winapi.CreateFile', new=create_file_mock)) + stack.enter_context( + mock.patch('_winapi.WriteFile', new=write_file_mock)) + stack.enter_context( + mock.patch('_winapi.ReadFile', new=read_file_mock)) + port = portpicker.get_port_from_port_server( + 'portserver', pid=1234) + write_file_mock.assert_called_once_with(0, b'1234\n') + else: + server = mock.Mock() + server.recv.return_value = b'42768\n' + stack.enter_context( + mock.patch.object(socket, 'socket', return_value=server)) + port = portpicker.get_port_from_port_server( + 'portserver', pid=1234) + server.sendall.assert_called_once_with(b'1234\n') + self.assertEqual(port, 42768) def testPidDefaultsToOwnPid(self): - server = mock.Mock() - server.recv.return_value = b'52768\n' - with mock.patch.object(socket, 'socket', return_value=server): - with mock.patch.object(os, 'getpid', return_value=9876): + with ExitStack() as stack: + stack.enter_context( + mock.patch.object(os, 'getpid', return_value=9876)) + + if _winapi: + create_file_mock = mock.Mock() + create_file_mock.return_value = 0 + read_file_mock = mock.Mock() + write_file_mock = mock.Mock() + read_file_mock.return_value = (b'52768\n', 0) + stack.enter_context( + mock.patch('_winapi.CreateFile', new=create_file_mock)) + stack.enter_context( + mock.patch('_winapi.WriteFile', new=write_file_mock)) + stack.enter_context( + mock.patch('_winapi.ReadFile', new=read_file_mock)) + port = portpicker.get_port_from_port_server('portserver') + write_file_mock.assert_called_once_with(0, b'9876\n') + else: + server = mock.Mock() + server.recv.return_value = b'52768\n' + stack.enter_context( + mock.patch.object(socket, 'socket', return_value=server)) port = portpicker.get_port_from_port_server('portserver') server.sendall.assert_called_once_with(b'9876\n') + self.assertEqual(port, 52768) @mock.patch.dict(os.environ,{'PORTSERVER_ADDRESS': 'portserver'}) def testReusesPortServerPorts(self): - server = mock.Mock() - server.recv.side_effect = [b'12345\n', b'23456\n', b'34567\n'] - with mock.patch.object(socket, 'socket', return_value=server): + with ExitStack() as stack: + if _winapi: + read_file_mock = mock.Mock() + read_file_mock.side_effect = [ + (b'12345\n', 0), + (b'23456\n', 0), + (b'34567\n', 0), + ] + stack.enter_context(mock.patch('_winapi.CreateFile')) + stack.enter_context(mock.patch('_winapi.WriteFile')) + stack.enter_context( + mock.patch('_winapi.ReadFile', new=read_file_mock)) + else: + server = mock.Mock() + server.recv.side_effect = [b'12345\n', b'23456\n', b'34567\n'] + stack.enter_context( + mock.patch.object(socket, 'socket', return_value=server)) + self.assertEqual(portpicker.pick_unused_port(), 12345) self.assertEqual(portpicker.pick_unused_port(), 23456) portpicker.return_port(12345) @@ -129,7 +191,12 @@ def testDoesntReuseRandomPorts(self): ports = set() for _ in range(10): - port = portpicker.pick_unused_port() + try: + port = portpicker.pick_unused_port() + except portpicker.NoFreePortFoundError: + # This sometimes happens when not using portserver. Just + # skip to the next attempt. + continue ports.add(port) portpicker.return_port(port) self.assertGreater(len(ports), 5) # Allow some random reuse. @@ -164,10 +231,20 @@ # will heavily exercise the "pick a port randomly" part of the # port picking code, but may never hit the "OS assigns a port" # code. + ports = 0 for _ in range(100): - port = portpicker._pick_unused_port_without_server() + try: + port = portpicker._pick_unused_port_without_server() + except portpicker.NoFreePortFoundError: + # Without the portserver, pick_unused_port can sometimes fail + # to find a free port. Check that it passes most of the time. + continue self.assertTrue(self.IsUnusedTCPPort(port)) self.assertTrue(self.IsUnusedUDPPort(port)) + ports += 1 + # Getting a port shouldn't have failed very often, even on machines + # with a heavy socket load. + self.assertGreater(ports, 95) def testOSAssignedPorts(self): self.last_assigned_port = None @@ -184,36 +261,47 @@ return None with mock.patch.object(portpicker, 'bind', error_for_explicit_ports): + # Without server, this can be little flaky, so check that it + # passes most of the time. + ports = 0 for _ in range(100): - port = portpicker._pick_unused_port_without_server() + try: + port = portpicker._pick_unused_port_without_server() + except portpicker.NoFreePortFoundError: + continue self.assertTrue(self.IsUnusedTCPPort(port)) self.assertTrue(self.IsUnusedUDPPort(port)) + ports += 1 + self.assertGreater(ports, 95) - def testPickPortsWithError(self): - r = random.Random() - - def bind_with_error(port, socket_type, socket_proto): - # 95% failure rate means both port picking methods will be - # exercised. - if int(r.uniform(0, 20)) == 0: - return self._bind(port, socket_type, socket_proto) + def pickUnusedPortWithoutServer(self): + # Try a few times to pick a port, to avoid flakiness and to make sure + # the code path we want was exercised. + for _ in range(5): + try: + port = portpicker._pick_unused_port_without_server() + except portpicker.NoFreePortFoundError: + continue else: - return None + self.assertTrue(self.IsUnusedTCPPort(port)) + self.assertTrue(self.IsUnusedUDPPort(port)) + return + self.fail("Failed to find a free port") - with mock.patch.object(portpicker, 'bind', bind_with_error): - got_at_least_one_port = False - for _ in range(100): - try: - port = portpicker._pick_unused_port_without_server() - except portpicker.NoFreePortFoundError: - continue - else: - got_at_least_one_port = True - self.assertTrue(self.IsUnusedTCPPort(port)) - self.assertTrue(self.IsUnusedUDPPort(port)) - self.assertTrue(got_at_least_one_port) + def testPickPortsWithoutServer(self): + # Test the first part of _pick_unused_port_without_server, which + # tries a few random ports and checks is_port_free. + self.pickUnusedPortWithoutServer() + + # Now test the second part, the fallback from above, which asks the + # OS for a port. + def mock_port_free(port): + return False - def testIsPortFree(self): + with mock.patch.object(portpicker, 'is_port_free', mock_port_free): + self.pickUnusedPortWithoutServer() + + def checkIsPortFree(self): """This might be flaky unless this test is run with a portserver.""" # The port should be free initially. port = portpicker.pick_unused_port() @@ -221,12 +309,18 @@ cases = [ (socket.AF_INET, socket.SOCK_STREAM, None), - (socket.AF_INET6, socket.SOCK_STREAM, 0), (socket.AF_INET6, socket.SOCK_STREAM, 1), (socket.AF_INET, socket.SOCK_DGRAM, None), - (socket.AF_INET6, socket.SOCK_DGRAM, 0), (socket.AF_INET6, socket.SOCK_DGRAM, 1), ] + + # Using v6only=0 on Windows doesn't result in collisions + if not _winapi: + cases.extend([ + (socket.AF_INET6, socket.SOCK_STREAM, 0), + (socket.AF_INET6, socket.SOCK_DGRAM, 0), + ]) + for (sock_family, sock_type, v6only) in cases: # Occupy the port on a subset of possible protocols. try: @@ -248,7 +342,16 @@ print('Kernel does not support IPV6_V6ONLY=%d' % v6only, file=sys.stderr) # Don't care; just proceed with the default. - sock.bind(('', port)) + + # Socket may have been taken in the mean time, so catch the + # socket.error with errno set to EADDRINUSE and skip this + # attempt. + try: + sock.bind(('', port)) + except socket.error as e: + if e.errno == errno.EADDRINUSE: + raise portpicker.NoFreePortFoundError + raise # The port should be busy. self.assertFalse(portpicker.is_port_free(port)) @@ -257,6 +360,17 @@ # Now it's free again. self.assertTrue(portpicker.is_port_free(port)) + def testIsPortFree(self): + # This can be quite flaky on a busy host, so try a few times. + for _ in range(10): + try: + self.checkIsPortFree() + except portpicker.NoFreePortFoundError: + pass + else: + return + self.fail("checkPortIsFree failed every time.") + def testIsPortFreeException(self): port = portpicker.pick_unused_port() with mock.patch.object(socket, 'socket') as mock_sock: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/src/tests/portserver_test.py new/portpicker-1.5.0/src/tests/portserver_test.py --- old/portpicker-1.3.1/src/tests/portserver_test.py 2017-10-09 19:32:19.000000000 +0200 +++ new/portpicker-1.5.0/src/tests/portserver_test.py 2021-07-12 03:38:42.000000000 +0200 @@ -16,21 +16,32 @@ # """Tests for the example portserver.""" -from __future__ import print_function import asyncio import os +import signal import socket +import subprocess import sys +import time import unittest from unittest import mock +from multiprocessing import Process import portpicker + +# On Windows, portserver.py is located in the "Scripts" folder, which isn't +# added to the import path by default +if sys.platform == 'win32': + sys.path.append(os.path.join(os.path.split(sys.executable)[0])) + import portserver def setUpModule(): portserver._configure_logging(verbose=True) +def exit_immediately(): + os._exit(0) class PortserverFunctionsTest(unittest.TestCase): @@ -51,12 +62,18 @@ cases = [ (socket.AF_INET, socket.SOCK_STREAM, None), - (socket.AF_INET6, socket.SOCK_STREAM, 0), (socket.AF_INET6, socket.SOCK_STREAM, 1), (socket.AF_INET, socket.SOCK_DGRAM, None), - (socket.AF_INET6, socket.SOCK_DGRAM, 0), (socket.AF_INET6, socket.SOCK_DGRAM, 1), ] + + # Using v6only=0 on Windows doesn't result in collisions + if sys.platform != 'win32': + cases.extend([ + (socket.AF_INET6, socket.SOCK_STREAM, 0), + (socket.AF_INET6, socket.SOCK_DGRAM, 0), + ]) + for (sock_family, sock_type, v6only) in cases: # Occupy the port on a subset of possible protocols. try: @@ -66,6 +83,10 @@ file=sys.stderr) # Skip this case, since we cannot occupy a port. continue + + if not hasattr(socket, 'IPPROTO_IPV6'): + v6only = None + if v6only is not None: try: sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, @@ -92,11 +113,12 @@ self.assertFalse(portserver._should_allocate_port(0)) self.assertFalse(portserver._should_allocate_port(1)) self.assertTrue(portserver._should_allocate_port, os.getpid()) - child_pid = os.fork() - if child_pid == 0: - os._exit(0) - else: - os.waitpid(child_pid, 0) + + p = Process(target=exit_immediately) + p.start() + child_pid = p.pid + p.join() + # This test assumes that after waitpid returns the kernel has finished # cleaning the process. We also assume that the kernel will not reuse # the former child's pid before our next call checks for its existence. @@ -129,32 +151,108 @@ portserver._configure_logging(False) portserver._configure_logging(True) + + _test_socket_addr = f'@TST-{os.getpid()}' + @mock.patch.object( sys, 'argv', ['PortserverFunctionsTest.test_main', - '--portserver_unix_socket_address=@TST-%d' % os.getpid()] + f'--portserver_unix_socket_address={_test_socket_addr}'] ) @mock.patch.object(portserver, '_parse_port_ranges') - @mock.patch('asyncio.get_event_loop') - @mock.patch('asyncio.start_unix_server') - def test_main(self, *unused_mocks): + def test_main_no_ports(self, *unused_mocks): portserver._parse_port_ranges.return_value = set() with self.assertRaises(SystemExit): portserver.main() - # Give it at least one port and try again. - portserver._parse_port_ranges.return_value = {self.port} - - mock_event_loop = mock.Mock(spec=asyncio.base_events.BaseEventLoop) - asyncio.get_event_loop.return_value = mock_event_loop - asyncio.start_unix_server.return_value = mock.Mock() - mock_event_loop.run_forever.side_effect = KeyboardInterrupt - - portserver.main() - - mock_event_loop.run_until_complete.assert_any_call( - asyncio.start_unix_server.return_value) - mock_event_loop.close.assert_called_once_with() - # NOTE: This could be improved. Tests of main() are often gross. + @unittest.skipUnless(sys.executable, 'Requires a stand alone interpreter') + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'AF_UNIX required') + def test_portserver_binary(self): + """Launch python portserver.py and test it.""" + # Blindly assuming tree layout is src/tests/portserver_test.py + # with src/portserver.py. + portserver_py = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + 'portserver.py') + anon_addr = self._test_socket_addr.replace('@', '\0') + + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + with self.assertRaises( + ConnectionRefusedError, + msg=f'{self._test_socket_addr} should not listen yet.'): + conn.connect(anon_addr) + conn.close() + + server = subprocess.Popen( + [sys.executable, portserver_py, + f'--portserver_unix_socket_address={self._test_socket_addr}'], + stderr=subprocess.PIPE, + ) + try: + # Wait a few seconds for the server to start listening. + start_time = time.monotonic() + while True: + time.sleep(0.05) + try: + conn.connect(anon_addr) + conn.close() + except ConnectionRefusedError: + delta = time.monotonic() - start_time + if delta < 4: + continue + else: + server.kill() + self.fail('Failed to connect to portserver ' + f'{self._test_socket_addr} within ' + f'{delta} seconds. STDERR:\n' + + server.stderr.read().decode('utf-8')) + else: + break + + ports = set() + port = portpicker.get_port_from_port_server( + portserver_address=self._test_socket_addr) + ports.add(port) + port = portpicker.get_port_from_port_server( + portserver_address=self._test_socket_addr) + ports.add(port) + + with subprocess.Popen('exit 0', shell=True) as quick_process: + quick_process.wait() + # This process doesn't exist so it should be a denied alloc. + # We use the pid from the above quick_process under the assumption + # that most OSes try to avoid rapid pid recycling. + denied_port = portpicker.get_port_from_port_server( + portserver_address=self._test_socket_addr, + pid=quick_process.pid) # A now unused pid. + self.assertIsNone(denied_port) + + self.assertEqual(len(ports), 2, msg=ports) + + # Check statistics from portserver + server.send_signal(signal.SIGUSR1) + # TODO implement an I/O timeout + for line in server.stderr: + if b'denied-allocations ' in line: + denied_allocations = int( + line.split(b'denied-allocations ', 2)[1]) + self.assertEqual(1, denied_allocations, msg=line) + elif b'total-allocations ' in line: + total_allocations = int( + line.split(b'total-allocations ', 2)[1]) + self.assertEqual(2, total_allocations, msg=line) + break + + rejected_port = portpicker.get_port_from_port_server( + portserver_address=self._test_socket_addr, + pid=99999999999999999999999999999999999) # Out of range. + self.assertIsNone(rejected_port) + + # Done. shutdown gracefully. + server.send_signal(signal.SIGINT) + server.communicate(timeout=2) + finally: + server.kill() + server.wait() class PortPoolTest(unittest.TestCase): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/portpicker-1.3.1/test.sh new/portpicker-1.5.0/test.sh --- old/portpicker-1.3.1/test.sh 2017-10-09 19:32:19.000000000 +0200 +++ new/portpicker-1.5.0/test.sh 2021-11-09 03:17:28.000000000 +0100 @@ -1,21 +1,12 @@ #!/bin/sh -ex -echo 'TESTING under Python 2' -mkdir -p build/test_envs/python2 -virtualenv --python=python2 build/test_envs/python2 -build/test_envs/python2/bin/pip install mock -# Without --upgrade pip won't copy local changes over to a new test install -# unless you've updated the package version number. -build/test_envs/python2/bin/pip install --upgrade . -build/test_envs/python2/bin/python2 src/tests/portpicker_test.py +unset PYTHONPATH +python3 -m venv build/venv +. build/venv/bin/activate -echo 'TESTING under Python 3' -mkdir -p build/test_envs/python3 -virtualenv --python=python3 build/test_envs/python3 -build/test_envs/python3/bin/pip install --upgrade . -build/test_envs/python3/bin/python3 src/tests/portpicker_test.py - -echo 'TESTING the portserver' -PYTHONPATH=src build/test_envs/python3/bin/python3 src/tests/portserver_test.py - -echo PASS +pip install --upgrade pip +pip install tox +# We should really do this differently, test from a `pip install .` so that +# testing relies on the setup.cfg install_requires instead of listing it here. +pip install psutil +tox -e "py3$(python -c 'import sys; print(sys.version_info.minor)')"