commit: cdf328f5629c20ecf792a6200c8c2c24d932ee68 Author: Mike Gilbert <floppym <AT> gentoo <DOT> org> AuthorDate: Mon May 13 19:21:47 2024 +0000 Commit: Mike Gilbert <floppym <AT> gentoo <DOT> org> CommitDate: Tue May 21 17:27:30 2024 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=cdf328f5
Drop portage.util.shlex_split This has been a trivial wrapper since we dropped support for python2. Signed-off-by: Mike Gilbert <floppym <AT> gentoo.org> bin/dispatch-conf | 5 +++-- bin/ebuild | 5 ++--- bin/egencache | 5 ++--- bin/quickpkg | 9 +++++---- lib/_emerge/BinpkgExtractorAsync.py | 14 +++++--------- lib/_emerge/BinpkgFetcher.py | 4 ++-- lib/_emerge/actions.py | 3 ++- lib/_emerge/main.py | 3 ++- lib/portage/_emirrordist/FetchTask.py | 3 ++- lib/portage/_sets/dbapi.py | 6 +++--- lib/portage/_sets/libs.py | 4 ++-- lib/portage/dbapi/bintree.py | 5 ++--- lib/portage/dbapi/porttree.py | 5 +++-- lib/portage/dbapi/vartree.py | 17 ++++++----------- lib/portage/dispatch_conf.py | 5 +++-- lib/portage/emaint/modules/logs/logs.py | 5 +++-- lib/portage/emaint/modules/sync/sync.py | 3 ++- lib/portage/getbinpkg.py | 3 ++- lib/portage/gpg.py | 7 ++++--- lib/portage/gpkg.py | 11 ++++++----- lib/portage/package/ebuild/_config/LocationsManager.py | 4 ++-- .../package/ebuild/_config/env_var_validation.py | 4 ++-- .../package/ebuild/_parallel_manifest/ManifestTask.py | 5 +++-- lib/portage/package/ebuild/config.py | 6 +++--- lib/portage/package/ebuild/doebuild.py | 9 ++++----- lib/portage/package/ebuild/fetch.py | 6 +++--- lib/portage/repository/config.py | 4 ++-- lib/portage/sync/modules/git/git.py | 11 ++++++----- lib/portage/sync/modules/mercurial/mercurial.py | 15 ++++++++------- lib/portage/sync/modules/rsync/rsync.py | 9 +++------ lib/portage/tests/ebuild/test_fetch.py | 5 +++-- lib/portage/tests/emerge/conftest.py | 7 ++++--- lib/portage/tests/emerge/test_config_protect.py | 5 +++-- lib/portage/util/ExtractKernelVersion.py | 5 +++-- lib/portage/util/__init__.py | 11 +---------- lib/portage/util/_async/BuildLogger.py | 4 ++-- lib/portage/util/_dyn_libs/soname_deps.py | 6 +++--- 37 files changed, 116 insertions(+), 122 deletions(-) diff --git a/bin/dispatch-conf b/bin/dispatch-conf index e34e9587f7..93164d909e 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -14,6 +14,7 @@ import atexit import errno import re +import shlex import subprocess import sys import termios @@ -89,7 +90,7 @@ def cmd_var_is_valid(cmd): Return true if the first whitespace-separated token contained in cmd is an executable file, false otherwise. """ - cmd = portage.util.shlex_split(cmd) + cmd = shlex.split(cmd) if not cmd: return False @@ -130,7 +131,7 @@ class dispatch: if pager is None or not cmd_var_is_valid(pager): pager = "cat" - pager_basename = os.path.basename(portage.util.shlex_split(pager)[0]) + pager_basename = os.path.basename(shlex.split(pager)[0]) if pager_basename == "less": less_opts = self.options.get("less-opts") if less_opts is not None and less_opts.strip(): diff --git a/bin/ebuild b/bin/ebuild index 043e5bc476..113a6214d6 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -34,6 +34,7 @@ signal.signal(signal.SIGUSR1, debug_signal) import argparse from os import path as osp +import shlex import sys import textwrap @@ -107,9 +108,7 @@ def main(): parser.error("missing required args") if not opts.ignore_default_opts: - default_opts = portage.util.shlex_split( - portage.settings.get("EBUILD_DEFAULT_OPTS", "") - ) + default_opts = shlex.split(portage.settings.get("EBUILD_DEFAULT_OPTS", "")) opts, pargs = parser.parse_known_args(default_opts + sys.argv[1:]) debug = opts.debug diff --git a/bin/egencache b/bin/egencache index dbe8d27fee..36477e1abf 100755 --- a/bin/egencache +++ b/bin/egencache @@ -33,6 +33,7 @@ try: signal.signal(signal.SIGUSR1, debug_signal) import argparse + import shlex import stat import sys import functools @@ -1114,9 +1115,7 @@ try: default_opts = None if not options.ignore_default_opts: - default_opts = portage.util.shlex_split( - settings.get("EGENCACHE_DEFAULT_OPTS", "") - ) + default_opts = shlex.split(settings.get("EGENCACHE_DEFAULT_OPTS", "")) if default_opts: parser, options, args = parse_args(default_opts + args) diff --git a/bin/quickpkg b/bin/quickpkg index c688c5312e..520b9bc35e 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -5,6 +5,7 @@ import argparse import errno import math +import shlex import signal import subprocess import sys @@ -35,7 +36,7 @@ from portage.exception import ( PackageSetNotFound, PermissionDenied, ) -from portage.util import ensure_dirs, shlex_split, varexpand, _xattr +from portage.util import ensure_dirs, varexpand, _xattr from portage.util.cpuinfo import makeopts_to_job_count xattr = _xattr.xattr @@ -165,7 +166,7 @@ def quickpkg_atom(options, infos, arg, eout): ) try: - compression_binary = shlex_split( + compression_binary = shlex.split( varexpand(compression["compress"], mydict=settings) )[0] except IndexError as e: @@ -184,7 +185,7 @@ def quickpkg_atom(options, infos, arg, eout): "{JOBS}", str(makeopts_to_job_count(settings.get("MAKEOPTS", "1"))), ) - cmd = shlex_split(varexpand(cmd, mydict=settings)) + cmd = shlex.split(varexpand(cmd, mydict=settings)) # Filter empty elements that make Popen fail cmd = [x for x in cmd if x != ""] with open(binpkg_tmpfile, "wb") as fobj: @@ -425,7 +426,7 @@ if __name__ == "__main__": ) options, args = parser.parse_known_args(sys.argv[1:]) if not options.ignore_default_opts: - default_opts = shlex_split(portage.settings.get("QUICKPKG_DEFAULT_OPTS", "")) + default_opts = shlex.split(portage.settings.get("QUICKPKG_DEFAULT_OPTS", "")) options, args = parser.parse_known_args(default_opts + sys.argv[1:]) if not args: parser.error("no packages atoms given") diff --git a/lib/_emerge/BinpkgExtractorAsync.py b/lib/_emerge/BinpkgExtractorAsync.py index be7a653f91..d1b6374c08 100644 --- a/lib/_emerge/BinpkgExtractorAsync.py +++ b/lib/_emerge/BinpkgExtractorAsync.py @@ -11,12 +11,10 @@ from portage.util.compression_probe import ( ) from portage.util.cpuinfo import makeopts_to_job_count from portage.process import find_binary -from portage.util import ( - shlex_split, - varexpand, -) +from portage.util import varexpand from portage.exception import InvalidBinaryPackageFormat from portage.binpkg import get_binpkg_format +import shlex import signal import subprocess import tarfile @@ -46,9 +44,7 @@ class BinpkgExtractorAsync(SpawnProcess): output = process.communicate()[0] if b"--xattrs" in output: tar_options = ["--xattrs", "--xattrs-include='*'"] - for x in portage.util.shlex_split( - self.env.get("PORTAGE_XATTR_EXCLUDE", "") - ): + for x in shlex.split(self.env.get("PORTAGE_XATTR_EXCLUDE", "")): tar_options.append(portage._shell_quote(f"--xattrs-exclude={x}")) tar_options = " ".join(tar_options) @@ -82,7 +78,7 @@ class BinpkgExtractorAsync(SpawnProcess): return try: - decompression_binary = shlex_split(varexpand(decomp_cmd, mydict=self.env))[ + decompression_binary = shlex.split(varexpand(decomp_cmd, mydict=self.env))[ 0 ] except IndexError: @@ -93,7 +89,7 @@ class BinpkgExtractorAsync(SpawnProcess): if decomp.get("decompress_alt"): decomp_cmd = decomp.get("decompress_alt") try: - decompression_binary = shlex_split( + decompression_binary = shlex.split( varexpand(decomp_cmd, mydict=self.env) )[0] except IndexError: diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py index 587e4a57a3..19d08359f0 100644 --- a/lib/_emerge/BinpkgFetcher.py +++ b/lib/_emerge/BinpkgFetcher.py @@ -5,6 +5,7 @@ from _emerge.AsynchronousLock import AsynchronousLock from _emerge.CompositeTask import CompositeTask from _emerge.SpawnProcess import SpawnProcess from urllib.parse import urlparse as urllib_parse_urlparse +import shlex import stat import sys import portage @@ -201,8 +202,7 @@ class _BinpkgFetcherProcess(SpawnProcess): fetch_env = dict(settings.items()) fetch_args = [ - portage.util.varexpand(x, mydict=fcmd_vars) - for x in portage.util.shlex_split(fcmd) + portage.util.varexpand(x, mydict=fcmd_vars) for x in shlex.split(fcmd) ] if self.fd_pipes is None: diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index b93c08aa07..512e470ad6 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -6,6 +6,7 @@ import logging import operator import platform import re +import shlex import signal import subprocess import sys @@ -3083,7 +3084,7 @@ def nice(settings): def ionice(settings): ionice_cmd = settings.get("PORTAGE_IONICE_COMMAND") if ionice_cmd: - ionice_cmd = portage.util.shlex_split(ionice_cmd) + ionice_cmd = shlex.split(ionice_cmd) if not ionice_cmd: return diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py index 6a52404c88..465e20163d 100644 --- a/lib/_emerge/main.py +++ b/lib/_emerge/main.py @@ -4,6 +4,7 @@ import argparse import locale import platform +import shlex import sys import portage @@ -1295,7 +1296,7 @@ def emerge_main(args: Optional[list[str]] = None): tmpcmdline = [] if "--ignore-default-opts" not in myopts: tmpcmdline.extend( - portage.util.shlex_split( + shlex.split( emerge_config.target_config.settings.get("EMERGE_DEFAULT_OPTS", "") ) ) diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py index 013163f83f..c96d672b0d 100644 --- a/lib/portage/_emirrordist/FetchTask.py +++ b/lib/portage/_emirrordist/FetchTask.py @@ -5,6 +5,7 @@ import collections import errno import logging import random +import shlex import subprocess import portage @@ -471,7 +472,7 @@ class FetchTask(CompositeTask): except OSError: pass - args = portage.util.shlex_split(default_fetchcommand) + args = shlex.split(default_fetchcommand) args = [portage.util.varexpand(x, mydict=variables) for x in args] args = [ diff --git a/lib/portage/_sets/dbapi.py b/lib/portage/_sets/dbapi.py index 9c5b979a97..7c666b47c1 100644 --- a/lib/portage/_sets/dbapi.py +++ b/lib/portage/_sets/dbapi.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 import glob +import shlex import time from portage import os @@ -12,7 +13,6 @@ from portage.dep._slot_operator import strip_slots from portage.localization import _ from portage._sets.base import PackageSet from portage._sets import SetConfigError, get_boolean -import portage __all__ = [ "CategorySet", @@ -133,11 +133,11 @@ class OwnerSet(PackageSet): exclude_files = options.get("exclude-files") if exclude_files is not None: - exclude_files = frozenset(portage.util.shlex_split(exclude_files)) + exclude_files = frozenset(shlex.split(exclude_files)) return cls( vardb=trees["vartree"].dbapi, exclude_files=exclude_files, - files=frozenset(portage.util.shlex_split(options["files"])), + files=frozenset(shlex.split(options["files"])), ) singleBuilder = classmethod(singleBuilder) diff --git a/lib/portage/_sets/libs.py b/lib/portage/_sets/libs.py index 8608442352..f3e1ea220d 100644 --- a/lib/portage/_sets/libs.py +++ b/lib/portage/_sets/libs.py @@ -5,7 +5,7 @@ from portage.exception import InvalidData from portage.localization import _ from portage._sets.base import PackageSet from portage._sets import get_boolean, SetConfigError -import portage +import shlex class LibraryConsumerSet(PackageSet): @@ -57,7 +57,7 @@ class LibraryFileConsumerSet(LibraryConsumerSet): self._setAtoms(self.mapPathsToAtoms(consumers)) def singleBuilder(cls, options, settings, trees): - files = tuple(portage.util.shlex_split(options.get("files", ""))) + files = tuple(shlex.split(options.get("files", ""))) if not files: raise SetConfigError(_("no files given")) debug = get_boolean(options, "debug", False) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 7bc1f60f6d..22e2995c2f 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -63,6 +63,7 @@ import codecs import errno import io import re +import shlex import stat import subprocess import tempfile @@ -1474,9 +1475,7 @@ class binarytree: ssh_args.append(f"-p{port}") # NOTE: shlex evaluates embedded quotes ssh_args.extend( - portage.util.shlex_split( - self.settings.get("PORTAGE_SSH_OPTS", "") - ) + shlex.split(self.settings.get("PORTAGE_SSH_OPTS", "")) ) ssh_args.append(user_passwd + host) ssh_args.append("--") diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index de6aa5c82b..5a4d69f372 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -13,7 +13,7 @@ portage.proxy.lazyimport.lazyimport( "portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot", "portage.package.ebuild.doebuild:doebuild", "portage.package.ebuild.fetch:get_mirror_url,_download_suffix", - "portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level", + "portage.util:ensure_dirs,writemsg,writemsg_level", "portage.util.listdir:listdir", "portage.versions:best,catsplit,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str", ) @@ -48,6 +48,7 @@ import traceback import warnings import errno import functools +import shlex import collections from collections import OrderedDict @@ -1016,7 +1017,7 @@ class portdbapi(dbapi): existing_size = 0 ro_distdirs = self.settings.get("PORTAGE_RO_DISTDIRS") if ro_distdirs is not None: - for x in shlex_split(ro_distdirs): + for x in shlex.split(ro_distdirs): try: mystat = os.stat( portage.package.ebuild.fetch.get_mirror_url( diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index beee82f8b0..beb1a6486f 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -109,6 +109,7 @@ import pickle import platform import pwd import re +import shlex import stat import tempfile import textwrap @@ -927,13 +928,9 @@ class vardbapi(dbapi): env_file = self.getpath(cpv, filename="environment.bz2") if not os.path.isfile(env_file): return {} - bunzip2_cmd = portage.util.shlex_split( - self.settings.get("PORTAGE_BUNZIP2_COMMAND", "") - ) + bunzip2_cmd = shlex.split(self.settings.get("PORTAGE_BUNZIP2_COMMAND", "")) if not bunzip2_cmd: - bunzip2_cmd = portage.util.shlex_split( - self.settings["PORTAGE_BZIP2_COMMAND"] - ) + bunzip2_cmd = shlex.split(self.settings["PORTAGE_BZIP2_COMMAND"]) bunzip2_cmd.append("-d") args = bunzip2_cmd + ["-c", env_file] try: @@ -1082,7 +1079,7 @@ class vardbapi(dbapi): ) # Method parameters may override QUICKPKG_DEFAULT_OPTS. - opts_list = portage.util.shlex_split(settings.get("QUICKPKG_DEFAULT_OPTS", "")) + opts_list = shlex.split(settings.get("QUICKPKG_DEFAULT_OPTS", "")) if include_config is not None: opts_list.append(f"--include-config={'y' if include_config else 'n'}") if include_unmodified_config is not None: @@ -2694,9 +2691,7 @@ class dblink: # process symlinks second-to-last, directories last. mydirs = set() - uninstall_ignore = portage.util.shlex_split( - self.settings.get("UNINSTALL_IGNORE", "") - ) + uninstall_ignore = shlex.split(self.settings.get("UNINSTALL_IGNORE", "")) def unlink(file_name, lstatobj): if bsd_chflags: @@ -3868,7 +3863,7 @@ class dblink: real_relative_paths = {} collision_ignore = [] - for x in portage.util.shlex_split(self.settings.get("COLLISION_IGNORE", "")): + for x in shlex.split(self.settings.get("COLLISION_IGNORE", "")): if os.path.isdir(os.path.join(self._eroot, x.lstrip(os.sep))): x = normalize_path(x) x += "/*" diff --git a/lib/portage/dispatch_conf.py b/lib/portage/dispatch_conf.py index 2597cf9b45..b310cc07d5 100644 --- a/lib/portage/dispatch_conf.py +++ b/lib/portage/dispatch_conf.py @@ -7,6 +7,7 @@ import errno import functools +import shlex import stat import subprocess import sys @@ -16,7 +17,7 @@ import portage from portage import _encodings, os, shutil from portage.env.loaders import KeyValuePairFileLoader from portage.localization import _ -from portage.util import shlex_split, varexpand +from portage.util import varexpand from portage.util.hooks import perform_hooks from portage.util.path import iter_parents @@ -34,7 +35,7 @@ def diffstatusoutput(cmd, file1, file2): """ # Use Popen to emulate getstatusoutput(), since getstatusoutput() may # raise a UnicodeDecodeError which makes the output inaccessible. - args = shlex_split(cmd % (file1, file2)) + args = shlex.split(cmd % (file1, file2)) args = (portage._unicode_encode(x, errors="strict") for x in args) proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) diff --git a/lib/portage/emaint/modules/logs/logs.py b/lib/portage/emaint/modules/logs/logs.py index fefbe6edbc..bfd5a3dd94 100644 --- a/lib/portage/emaint/modules/logs/logs.py +++ b/lib/portage/emaint/modules/logs/logs.py @@ -3,7 +3,8 @@ import portage from portage import os -from portage.util import shlex_split, varexpand +from portage.util import varexpand +import shlex # default clean command from make.globals ## PORTAGE_LOGDIR_CLEAN = 'find "${PORTAGE_LOGDIR}" -type f ! -name "summary.log*" -mtime +7 -delete' @@ -55,7 +56,7 @@ class CleanLogs: clean_cmd = settings.get("PORTAGE_LOGDIR_CLEAN") if clean_cmd: - clean_cmd = shlex_split(clean_cmd) + clean_cmd = shlex.split(clean_cmd) if "-mtime" in clean_cmd and num_of_days is not None: if num_of_days == 0: i = clean_cmd.index("-mtime") diff --git a/lib/portage/emaint/modules/sync/sync.py b/lib/portage/emaint/modules/sync/sync.py index a1fbfc49e3..3c397c2a4b 100644 --- a/lib/portage/emaint/modules/sync/sync.py +++ b/lib/portage/emaint/modules/sync/sync.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 import os +import shlex import portage @@ -52,7 +53,7 @@ class SyncRepos: # Parse EMERGE_DEFAULT_OPTS, for settings like # --package-moves=n. - cmdline = portage.util.shlex_split( + cmdline = shlex.split( emerge_config.target_config.settings.get("EMERGE_DEFAULT_OPTS", "") ) emerge_config.opts = parse_opts(cmdline, silent=True)[1] diff --git a/lib/portage/getbinpkg.py b/lib/portage/getbinpkg.py index fca44f0525..af62b66aba 100644 --- a/lib/portage/getbinpkg.py +++ b/lib/portage/getbinpkg.py @@ -14,6 +14,7 @@ from portage.package.ebuild.fetch import _hide_url_passwd from _emerge.Package import _all_metadata_keys import pickle +import shlex import sys import socket import time @@ -525,7 +526,7 @@ def file_get( from portage.util import varexpand from portage.process import spawn - myfetch = [varexpand(x, mydict=variables) for x in portage.util.shlex_split(fcmd)] + myfetch = [varexpand(x, mydict=variables) for x in shlex.split(fcmd)] fd_pipes = { 0: portage._get_stdin().fileno(), 1: sys.__stdout__.fileno(), diff --git a/lib/portage/gpg.py b/lib/portage/gpg.py index d8a4cfcfc4..6b482eebd9 100644 --- a/lib/portage/gpg.py +++ b/lib/portage/gpg.py @@ -1,6 +1,7 @@ # Copyright 2001-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +import shlex import subprocess import sys import threading @@ -9,7 +10,7 @@ from portage import os from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS from portage.exception import GPGException from portage.output import colorize -from portage.util import shlex_split, varexpand, writemsg, writemsg_stdout +from portage.util import varexpand, writemsg, writemsg_stdout class GPG: @@ -60,7 +61,7 @@ class GPG: # GPG does not need to ask password, so can be ignored. writemsg(f"{colorize('WARN', str(e))}\n") - cmd = shlex_split(varexpand(self.GPG_unlock_command, mydict=self.settings)) + cmd = shlex.split(varexpand(self.GPG_unlock_command, mydict=self.settings)) return_code = subprocess.Popen(cmd, stdout=subprocess.DEVNULL).wait() if return_code == os.EX_OK: @@ -70,7 +71,7 @@ class GPG: raise GPGException("GPG unlock failed") if self.keepalive: - self.GPG_unlock_command = shlex_split( + self.GPG_unlock_command = shlex.split( varexpand(self.GPG_unlock_command, mydict=self.settings) ) self._terminated = threading.Event() diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py index 2b957d58c4..fc4d7b1fb6 100644 --- a/lib/portage/gpkg.py +++ b/lib/portage/gpkg.py @@ -9,6 +9,7 @@ import subprocess import errno import pwd import grp +import shlex import stat import sys import tempfile @@ -40,7 +41,7 @@ from portage.exception import ( from portage.output import colorize, EOutput from portage.util._urlopen import urlopen from portage.util import writemsg -from portage.util import shlex_split, varexpand +from portage.util import varexpand from portage.util.compression_probe import _compressors from portage.util.cpuinfo import makeopts_to_job_count from portage.process import find_binary @@ -464,7 +465,7 @@ class checksum_helper: "--batch --no-tty", ) - gpg_signing_command = shlex_split( + gpg_signing_command = shlex.split( varexpand(gpg_signing_command, mydict=self.settings) ) gpg_signing_command = [x for x in gpg_signing_command if x != ""] @@ -517,7 +518,7 @@ class checksum_helper: "[SIGNATURE]", "--output - -" ) - gpg_verify_command = shlex_split( + gpg_verify_command = shlex.split( varexpand(gpg_verify_command, mydict=self.settings) ) gpg_verify_command = [x for x in gpg_verify_command if x != ""] @@ -1756,13 +1757,13 @@ class gpkg: cmd = compressor[mode].replace( "{JOBS}", str(makeopts_to_job_count(self.settings.get("MAKEOPTS", "1"))) ) - cmd = shlex_split(varexpand(cmd, mydict=self.settings)) + cmd = shlex.split(varexpand(cmd, mydict=self.settings)) # Filter empty elements that make Popen fail cmd = [x for x in cmd if x != ""] if (not cmd) and ((mode + "_alt") in compressor): - cmd = shlex_split( + cmd = shlex.split( varexpand(compressor[mode + "_alt"], mydict=self.settings) ) cmd = [x for x in cmd if x != ""] diff --git a/lib/portage/package/ebuild/_config/LocationsManager.py b/lib/portage/package/ebuild/_config/LocationsManager.py index 6c54b80563..0e1881e1e5 100644 --- a/lib/portage/package/ebuild/_config/LocationsManager.py +++ b/lib/portage/package/ebuild/_config/LocationsManager.py @@ -3,6 +3,7 @@ __all__ = ("LocationsManager",) +import shlex import warnings import portage @@ -21,7 +22,6 @@ from portage.util import ( grabfile, normalize_path, read_corresponding_eapi_file, - shlex_split, writemsg, ) from portage.util._path import exists_raise_eaccess, isdir_raise_eaccess @@ -421,7 +421,7 @@ class LocationsManager: self.portdir_overlay = "" self.overlay_profiles = [] - for ov in shlex_split(self.portdir_overlay): + for ov in shlex.split(self.portdir_overlay): ov = normalize_path(ov) profiles_dir = os.path.join(ov, "profiles") if isdir_raise_eaccess(profiles_dir): diff --git a/lib/portage/package/ebuild/_config/env_var_validation.py b/lib/portage/package/ebuild/_config/env_var_validation.py index e00070926b..9f14529694 100644 --- a/lib/portage/package/ebuild/_config/env_var_validation.py +++ b/lib/portage/package/ebuild/_config/env_var_validation.py @@ -3,7 +3,7 @@ from portage import os from portage.process import find_binary -from portage.util import shlex_split +import shlex def validate_cmd_var(v): @@ -14,7 +14,7 @@ def validate_cmd_var(v): is the (possibly empty) list of tokens split by shlex. """ invalid = False - v_split = shlex_split(v) + v_split = shlex.split(v) if not v_split: invalid = True elif os.path.isabs(v_split[0]): diff --git a/lib/portage/package/ebuild/_parallel_manifest/ManifestTask.py b/lib/portage/package/ebuild/_parallel_manifest/ManifestTask.py index 87aa46de59..36b2a642cd 100644 --- a/lib/portage/package/ebuild/_parallel_manifest/ManifestTask.py +++ b/lib/portage/package/ebuild/_parallel_manifest/ManifestTask.py @@ -3,6 +3,7 @@ import errno import re +import shlex import subprocess from portage import os @@ -11,7 +12,7 @@ from portage.const import MANIFEST2_IDENTIFIERS from portage.dep import _repo_separator from portage.exception import InvalidDependString from portage.localization import _ -from portage.util import atomic_ofstream, grablines, shlex_split, varexpand, writemsg +from portage.util import atomic_ofstream, grablines, varexpand, writemsg from portage.util._async.AsyncTaskFuture import AsyncTaskFuture from portage.util._async.PipeLogger import PipeLogger from portage.util._async.PopenProcess import PopenProcess @@ -187,7 +188,7 @@ class ManifestTask(CompositeTask): gpg_vars = gpg_vars.copy() gpg_vars["FILE"] = self._manifest_path gpg_cmd = varexpand(self.gpg_cmd, mydict=gpg_vars) - gpg_cmd = shlex_split(gpg_cmd) + gpg_cmd = shlex.split(gpg_cmd) gpg_proc = PopenProcess( proc=subprocess.Popen( gpg_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py index 67fd1bb18d..84e8d8637a 100644 --- a/lib/portage/package/ebuild/config.py +++ b/lib/portage/package/ebuild/config.py @@ -15,6 +15,7 @@ import logging import platform import pwd import re +import shlex import sys import traceback import warnings @@ -78,7 +79,6 @@ from portage.util import ( grabfile_package, LazyItemsDict, normalize_path, - shlex_split, stack_dictlist, stack_dicts, stack_lists, @@ -628,7 +628,7 @@ class config: v = confs.get("PORTDIR_OVERLAY") if v is not None: portdir_overlay = v - known_repos.extend(shlex_split(v)) + known_repos.extend(shlex.split(v)) v = confs.get("SYNC") if v is not None: portdir_sync = v @@ -1580,7 +1580,7 @@ class config: ) try: - compression_binary = shlex_split( + compression_binary = shlex.split( portage.util.varexpand(compression["compress"], mydict=self) )[0] except IndexError as e: diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py index 6691db4e97..6641cd8341 100644 --- a/lib/portage/package/ebuild/doebuild.py +++ b/lib/portage/package/ebuild/doebuild.py @@ -13,6 +13,7 @@ import os as _os import platform import pwd import re +import shlex import signal import stat import sys @@ -107,7 +108,6 @@ from portage.util import ( apply_recursive_permissions, apply_secpass_permissions, noiselimit, - shlex_split, varexpand, writemsg, writemsg_stdout, @@ -677,7 +677,7 @@ def doebuild_environment( "{JOBS}", str(makeopts_to_job_count(mysettings.get("MAKEOPTS", "1"))), ) - compression_binary = shlex_split( + compression_binary = shlex.split( varexpand(compression_binary, mydict=settings) )[0] except IndexError as e: @@ -698,7 +698,7 @@ def doebuild_environment( ) cmd = [ varexpand(x, mydict=settings) - for x in shlex_split(compression_binary) + for x in shlex.split(compression_binary) ] # Filter empty elements cmd = [x for x in cmd if x != ""] @@ -2990,8 +2990,7 @@ def _post_src_install_soname_symlinks(mysettings, out): if qa_prebuilt: qa_prebuilt = re.compile( "|".join( - fnmatch.translate(x.lstrip(os.sep)) - for x in portage.util.shlex_split(qa_prebuilt) + fnmatch.translate(x.lstrip(os.sep)) for x in shlex.split(qa_prebuilt) ) ) diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py index bfa0c2b275..ed40cf6ede 100644 --- a/lib/portage/package/ebuild/fetch.py +++ b/lib/portage/package/ebuild/fetch.py @@ -11,6 +11,7 @@ import json import logging import random import re +import shlex import stat import sys import tempfile @@ -67,7 +68,6 @@ from portage.util import ( apply_secpass_permissions, ensure_dirs, grabdict, - shlex_split, varexpand, writemsg, writemsg_level, @@ -975,7 +975,7 @@ def fetch( ro_distdirs = [ x - for x in shlex_split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) + for x in shlex.split(mysettings.get("PORTAGE_RO_DISTDIRS", "")) if os.path.isdir(x) ] @@ -1736,7 +1736,7 @@ def fetch( variables[k] = v myfetch = varexpand(locfetch, mydict=variables) - myfetch = shlex_split(myfetch) + myfetch = shlex.split(myfetch) myret = -1 try: diff --git a/lib/portage/repository/config.py b/lib/portage/repository/config.py index a5b904dc6b..c9dfffa22c 100644 --- a/lib/portage/repository/config.py +++ b/lib/portage/repository/config.py @@ -6,6 +6,7 @@ import io import logging import warnings import re +import shlex import typing import portage @@ -21,7 +22,6 @@ from portage.env.loaders import KeyValuePairFileLoader from portage.util import ( normalize_path, read_corresponding_eapi_file, - shlex_split, stack_lists, writemsg, writemsg_level, @@ -656,7 +656,7 @@ class RepoConfigLoader: portdir_orig = portdir overlays.append(portdir) try: - port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)] + port_ov = [normalize_path(i) for i in shlex.split(portdir_overlay)] except ValueError as e: # File "/usr/lib/python3.2/shlex.py", line 168, in read_token # raise ValueError("No closing quotation") diff --git a/lib/portage/sync/modules/git/git.py b/lib/portage/sync/modules/git/git.py index a2830280fb..a06ca60a82 100644 --- a/lib/portage/sync/modules/git/git.py +++ b/lib/portage/sync/modules/git/git.py @@ -3,12 +3,13 @@ import logging import re +import shlex import subprocess import datetime import portage from portage import os -from portage.util import writemsg_level, shlex_split +from portage.util import writemsg_level from portage.util.futures import asyncio from portage.output import create_color_func, EOutput from portage.const import TIMESTAMP_FORMAT @@ -62,7 +63,7 @@ class GitSync(NewBase): git_cmd_opts = "" if self.repo.module_specific_options.get("sync-git-env"): - shlexed_env = shlex_split(self.repo.module_specific_options["sync-git-env"]) + shlexed_env = shlex.split(self.repo.module_specific_options["sync-git-env"]) env = { k: v for k, _, v in (assignment.partition("=") for assignment in shlexed_env) @@ -71,7 +72,7 @@ class GitSync(NewBase): self.spawn_kwargs["env"].update(env) if self.repo.module_specific_options.get("sync-git-clone-env"): - shlexed_env = shlex_split( + shlexed_env = shlex.split( self.repo.module_specific_options["sync-git-clone-env"] ) clone_env = { @@ -157,7 +158,7 @@ class GitSync(NewBase): self.add_safe_directory() if self.repo.module_specific_options.get("sync-git-env"): - shlexed_env = shlex_split(self.repo.module_specific_options["sync-git-env"]) + shlexed_env = shlex.split(self.repo.module_specific_options["sync-git-env"]) env = { k: v for k, _, v in (assignment.partition("=") for assignment in shlexed_env) @@ -166,7 +167,7 @@ class GitSync(NewBase): self.spawn_kwargs["env"].update(env) if self.repo.module_specific_options.get("sync-git-pull-env"): - shlexed_env = shlex_split( + shlexed_env = shlex.split( self.repo.module_specific_options["sync-git-pull-env"] ) pull_env = { diff --git a/lib/portage/sync/modules/mercurial/mercurial.py b/lib/portage/sync/modules/mercurial/mercurial.py index 6a3016a1f2..aad8ff94ab 100644 --- a/lib/portage/sync/modules/mercurial/mercurial.py +++ b/lib/portage/sync/modules/mercurial/mercurial.py @@ -2,11 +2,12 @@ # Distributed under the terms of the GNU General Public License v2 import logging +import shlex import subprocess import portage from portage import os -from portage.util import writemsg_level, shlex_split +from portage.util import writemsg_level from portage.sync.syncbase import NewBase @@ -46,7 +47,7 @@ class MercurialSync(NewBase): hg_cmd_opts = "" if self.repo.module_specific_options.get("sync-mercurial-env"): - shlexed_env = shlex_split( + shlexed_env = shlex.split( self.repo.module_specific_options["sync-mercurial-env"] ) env = { @@ -57,7 +58,7 @@ class MercurialSync(NewBase): self.spawn_kwargs["env"].update(env) if self.repo.module_specific_options.get("sync-mercurial-clone-env"): - shlexed_env = shlex_split( + shlexed_env = shlex.split( self.repo.module_specific_options["sync-mercurial-clone-env"] ) clone_env = { @@ -82,7 +83,7 @@ class MercurialSync(NewBase): writemsg_level(hg_cmd + "\n") exitcode = portage.process.spawn( - shlex_split(hg_cmd), + shlex.split(hg_cmd), cwd=portage._unicode_encode(self.repo.location), **self.spawn_kwargs, ) @@ -102,7 +103,7 @@ class MercurialSync(NewBase): hg_cmd_opts = "" if self.repo.module_specific_options.get("sync-mercurial-env"): - shlexed_env = shlex_split( + shlexed_env = shlex.split( self.repo.module_specific_options["sync-mercurial-env"] ) env = { @@ -113,7 +114,7 @@ class MercurialSync(NewBase): self.spawn_kwargs["env"].update(env) if self.repo.module_specific_options.get("sync-mercurial-pull-env"): - shlexed_env = shlex_split( + shlexed_env = shlex.split( self.repo.module_specific_options["sync-mercurial-pull-env"] ) pull_env = { @@ -139,7 +140,7 @@ class MercurialSync(NewBase): ) exitcode = portage.process.spawn( - shlex_split(hg_cmd), + shlex.split(hg_cmd), cwd=portage._unicode_encode(self.repo.location), **self.spawn_kwargs, ) diff --git a/lib/portage/sync/modules/rsync/rsync.py b/lib/portage/sync/modules/rsync/rsync.py index 5d442d2626..15c3eb4da5 100644 --- a/lib/portage/sync/modules/rsync/rsync.py +++ b/lib/portage/sync/modules/rsync/rsync.py @@ -5,6 +5,7 @@ import datetime import logging import random import re +import shlex import signal import socket import sys @@ -91,9 +92,7 @@ class RsyncSync(NewBase): self.extra_rsync_opts = list() if self.repo.module_specific_options.get("sync-rsync-extra-opts"): self.extra_rsync_opts.extend( - portage.util.shlex_split( - self.repo.module_specific_options["sync-rsync-extra-opts"] - ) + shlex.split(self.repo.module_specific_options["sync-rsync-extra-opts"]) ) exitcode = 0 @@ -599,9 +598,7 @@ class RsyncSync(NewBase): # defaults. portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n", 1) - rsync_opts.extend( - portage.util.shlex_split(self.settings.get("PORTAGE_RSYNC_OPTS", "")) - ) + rsync_opts.extend(shlex.split(self.settings.get("PORTAGE_RSYNC_OPTS", ""))) for opt in ("--recursive", "--times"): if opt not in rsync_opts: portage.writemsg( diff --git a/lib/portage/tests/ebuild/test_fetch.py b/lib/portage/tests/ebuild/test_fetch.py index 4812eb4306..1856bb52b8 100644 --- a/lib/portage/tests/ebuild/test_fetch.py +++ b/lib/portage/tests/ebuild/test_fetch.py @@ -3,6 +3,7 @@ import functools import io +import shlex import tempfile import types @@ -129,7 +130,7 @@ class EbuildFetchTestCase(TestCase): ), ) - fetchcommand = portage.util.shlex_split(playground.settings["FETCHCOMMAND"]) + fetchcommand = shlex.split(playground.settings["FETCHCOMMAND"]) fetch_bin = portage.process.find_binary(fetchcommand[0]) if fetch_bin is None: self.skipTest( @@ -137,7 +138,7 @@ class EbuildFetchTestCase(TestCase): ) eubin = os.path.join(playground.eprefix, "usr", "bin") os.symlink(fetch_bin, os.path.join(eubin, os.path.basename(fetch_bin))) - resumecommand = portage.util.shlex_split(playground.settings["RESUMECOMMAND"]) + resumecommand = shlex.split(playground.settings["RESUMECOMMAND"]) resume_bin = portage.process.find_binary(resumecommand[0]) if resume_bin is None: self.skipTest( diff --git a/lib/portage/tests/emerge/conftest.py b/lib/portage/tests/emerge/conftest.py index 356e09879c..ce86dc4fcd 100644 --- a/lib/portage/tests/emerge/conftest.py +++ b/lib/portage/tests/emerge/conftest.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 import argparse +import shlex from typing import Optional, Callable # , Self from portage.const import ( @@ -17,7 +18,7 @@ from portage import shutil from portage.util.futures import asyncio from portage.tests import cnf_bindir, cnf_sbindir from portage.process import find_binary -from portage.util import find_updated_config_files, shlex_split +from portage.util import find_updated_config_files import portage import pytest @@ -378,7 +379,7 @@ def _check_foo_file(pkgdir, filename, must_exist) -> None: def _check_number_of_protected_files(must_have, eroot, config_protect) -> None: assert must_have == len( - list(find_updated_config_files(eroot, shlex_split(config_protect))) + list(find_updated_config_files(eroot, shlex.split(config_protect))) ) @@ -798,7 +799,7 @@ def _generate_all_baseline_commands(playground, binhost): with open(binrepos_conf_file, "w") as f: f.write("[test-binhost]\n") f.write(f"sync-uri = {binhost_uri}\n") - fetchcommand = portage.util.shlex_split(settings["FETCHCOMMAND"]) + fetchcommand = shlex.split(settings["FETCHCOMMAND"]) fetch_bin = portage.process.find_binary(fetchcommand[0]) if fetch_bin is None: diff --git a/lib/portage/tests/emerge/test_config_protect.py b/lib/portage/tests/emerge/test_config_protect.py index e04fc1a920..59a2c23b90 100644 --- a/lib/portage/tests/emerge/test_config_protect.py +++ b/lib/portage/tests/emerge/test_config_protect.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 from functools import partial +import shlex import shutil import stat import subprocess @@ -15,7 +16,7 @@ from portage.const import BASH_BINARY, PORTAGE_PYM_PATH from portage.process import find_binary from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground -from portage.util import ensure_dirs, find_updated_config_files, shlex_split +from portage.util import ensure_dirs, find_updated_config_files class ConfigProtectTestCase(TestCase): @@ -146,7 +147,7 @@ src_install() { sum( len(x[1]) for x in find_updated_config_files( - eroot, shlex_split(config_protect) + eroot, shlex.split(config_protect) ) ), ) diff --git a/lib/portage/util/ExtractKernelVersion.py b/lib/portage/util/ExtractKernelVersion.py index 6a6501b5d5..af6b96ee36 100644 --- a/lib/portage/util/ExtractKernelVersion.py +++ b/lib/portage/util/ExtractKernelVersion.py @@ -4,10 +4,11 @@ __all__ = ["ExtractKernelVersion"] import logging +import shlex from portage import os, _encodings, _unicode_encode from portage.env.loaders import KeyValuePairFileLoader -from portage.util import grabfile, shlex_split, writemsg_level +from portage.util import grabfile, writemsg_level def ExtractKernelVersion(base_dir): @@ -81,6 +82,6 @@ def ExtractKernelVersion(base_dir): ) if kernelconfig and "CONFIG_LOCALVERSION" in kernelconfig: - version += "".join(shlex_split(kernelconfig["CONFIG_LOCALVERSION"])) + version += "".join(shlex.split(kernelconfig["CONFIG_LOCALVERSION"])) return (version, None) diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py index 1f8c9e94f1..0c88068dda 100644 --- a/lib/portage/util/__init__.py +++ b/lib/portage/util/__init__.py @@ -717,15 +717,6 @@ def writedict(mydict, myfilename, writekey=True): write_atomic(myfilename, "".join(lines)) -def shlex_split(s): - """ - This is equivalent to shlex.split, but if the current interpreter is - python2, it temporarily encodes unicode strings to bytes since python2's - shlex.split() doesn't handle unicode strings. - """ - return shlex.split(s) - - class _getconfig_shlex(shlex.shlex): def __init__(self, portage_tolerant=False, **kwargs): shlex.shlex.__init__(self, **kwargs) @@ -1968,7 +1959,7 @@ def find_updated_config_files(target_root, config_protect): % os.path.split(x.rstrip(os.path.sep)) ) mycommand += " ! -name '.*~' ! -iname '.*.bak' -print0" - cmd = shlex_split(mycommand) + cmd = shlex.split(mycommand) cmd = [ _unicode_encode(arg, encoding=encoding, errors="strict") for arg in cmd diff --git a/lib/portage/util/_async/BuildLogger.py b/lib/portage/util/_async/BuildLogger.py index 0cfc90a942..ed010f05dc 100644 --- a/lib/portage/util/_async/BuildLogger.py +++ b/lib/portage/util/_async/BuildLogger.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 import functools +import shlex import subprocess from _emerge.AsynchronousTask import AsynchronousTask @@ -9,7 +10,6 @@ from _emerge.AsynchronousTask import AsynchronousTask import portage from portage import os from portage.proxy.objectproxy import ObjectProxy -from portage.util import shlex_split from portage.util._async.PipeLogger import PipeLogger from portage.util._async.PopenProcess import PopenProcess from portage.util.futures import asyncio @@ -76,7 +76,7 @@ class BuildLogger(AsynchronousTask): if self.log_path is not None: log_filter_file = self.log_filter_file if log_filter_file is not None: - split_value = shlex_split(log_filter_file) + split_value = shlex.split(log_filter_file) log_filter_file = split_value if split_value else None if log_filter_file: filter_input, stdin = os.pipe() diff --git a/lib/portage/util/_dyn_libs/soname_deps.py b/lib/portage/util/_dyn_libs/soname_deps.py index 7d65209a4a..04d1f129fc 100644 --- a/lib/portage/util/_dyn_libs/soname_deps.py +++ b/lib/portage/util/_dyn_libs/soname_deps.py @@ -7,8 +7,8 @@ import functools from itertools import chain import os import re +import shlex -from portage.util import shlex_split from portage.util import ( normalize_path, varexpand, @@ -45,10 +45,10 @@ class SonameDepsProcessor: @staticmethod def _exclude_pattern(s): - # shlex_split enables quoted whitespace inside patterns + # shlex.split enables quoted whitespace inside patterns if s: pat = re.compile( - "|".join(fnmatch.translate(x.lstrip(os.sep)) for x in shlex_split(s)) + "|".join(fnmatch.translate(x.lstrip(os.sep)) for x in shlex.split(s)) ) else: pat = None