https://github.com/python/cpython/commit/13f61bf7f1fb9b9b109b089610e845d98e6dc937 commit: 13f61bf7f1fb9b9b109b089610e845d98e6dc937 branch: main author: Inada Naoki <[email protected]> committer: gpshead <[email protected]> date: 2024-09-02T20:32:38-07:00 summary:
gh-121313: multiprocessing: simplify by increasing the connection buffer size to 64KiB (GH-123559) Increases the multiprocessing connection buffer size from 8k to 64k for efficiency, without overallocating. Co-authored-by: Bénédikt Tran <[email protected]> Co-authored-by: Victor Stinner <[email protected]> files: A Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst D Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst M Lib/multiprocessing/connection.py diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index d84b52fe6d4f88..7e901cf2fb9852 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -13,7 +13,6 @@ import io import itertools import os -import stat import sys import socket import struct @@ -40,7 +39,9 @@ # # -BUFSIZE = 8192 +# 64 KiB is the default PIPE buffer size of most POSIX platforms. +BUFSIZE = 64 * 1024 + # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. @@ -361,11 +362,6 @@ def _get_more_data(self, ov, maxsize): f.write(ov.getbuffer()) return f -""" -The default size of a pipe on Linux systems is 16 times the base page size: -https://man7.org/linux/man-pages/man7/pipe.7.html -""" -PAGES_PER_PIPE = 16 class Connection(_ConnectionBase): """ @@ -378,14 +374,11 @@ def _close(self, _close=_multiprocessing.closesocket): _close(self._handle) _write = _multiprocessing.send _read = _multiprocessing.recv - _default_pipe_size = 0 else: def _close(self, _close=os.close): _close(self._handle) _write = os.write _read = os.read - _base_page_size = os.sysconf(os.sysconf_names['SC_PAGESIZE']) - _default_pipe_size = _base_page_size * PAGES_PER_PIPE def _send(self, buf, write=_write): remaining = len(buf) @@ -400,13 +393,8 @@ def _recv(self, size, read=_read): buf = io.BytesIO() handle = self._handle remaining = size - is_pipe = False - if size > self._default_pipe_size > 0: - mode = os.fstat(handle).st_mode - is_pipe = stat.S_ISFIFO(mode) - limit = self._default_pipe_size if is_pipe else remaining while remaining > 0: - to_read = min(limit, remaining) + to_read = min(BUFSIZE, remaining) chunk = read(handle, to_read) n = len(chunk) if n == 0: diff --git a/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst b/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst deleted file mode 100644 index 06abce9e67da44..00000000000000 --- a/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst +++ /dev/null @@ -1 +0,0 @@ -Limit reading size in multiprocessing connection._recv for pipes to default pipe size of 16 times base page size, in order to avoid memory overallocation and unnecessary memory management system calls. diff --git a/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst b/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst new file mode 100644 index 00000000000000..bb41063e684da4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst @@ -0,0 +1 @@ +Limit the reading size in the :class:`multiprocessing.connection.Connection` class to 64 KiB to prevent memory overallocation and unnecessary memory management system calls. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3/lists/python-checkins.python.org/ Member address: [email protected]
