[gentoo-commits] proj/portage:master commit in: bin/install-qa-check.d/, bin/
commit: cc793fb5f0d7c39c904f9e209b804d8f762a96cd
Author: Kerin Millar plushkava net>
AuthorDate: Tue Jul 23 16:05:51 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Jul 26 10:22:40 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cc793fb5
Check whether xtrace is enabled sensibly
To write [[ ${-/x/} != $- ]] is horribly backwards and comparatively
expensive. Implement the test - and its inverse - in a sensible fashion.
Signed-off-by: Kerin Millar plushkava.net>
Signed-off-by: Sam James gentoo.org>
bin/ebuild.sh | 4 ++--
bin/install-qa-check.d/90cython-dep | 2 +-
bin/install-qa-check.d/90gcc-warnings | 2 +-
bin/phase-functions.sh| 6 +++---
4 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/bin/ebuild.sh b/bin/ebuild.sh
index 8b1e0861a8..a742397db9 100755
--- a/bin/ebuild.sh
+++ b/bin/ebuild.sh
@@ -473,7 +473,7 @@ __try_source() {
if [[ -r ${1} && -f ${1} ]]; then
local debug_on=false
- if [[ "${PORTAGE_DEBUG}" == "1" ]] && [[ "${-/x/}" == "$-" ]];
then
+ if [[ ${PORTAGE_DEBUG} == 1 ]] && [[ $- != *x* ]]; then
debug_on=true
fi
@@ -628,7 +628,7 @@ if ! has "${EBUILD_PHASE}" clean cleanrm ; then
unset E_RESTRICT PROVIDES_EXCLUDE REQUIRES_EXCLUDE
unset PORTAGE_EXPLICIT_INHERIT
- if [[ ${PORTAGE_DEBUG} != 1 || ${-/x/} != $- ]] ; then
+ if [[ ${PORTAGE_DEBUG} != 1 || $- == *x* ]] ; then
source "${EBUILD}" || die "error sourcing ebuild"
else
set -x
diff --git a/bin/install-qa-check.d/90cython-dep
b/bin/install-qa-check.d/90cython-dep
index 4932c02429..2458074c39 100644
--- a/bin/install-qa-check.d/90cython-dep
+++ b/bin/install-qa-check.d/90cython-dep
@@ -13,7 +13,7 @@ cython_dep_check() {
# In debug mode, the grep calls will produce false positives
# if they're shown in the trace.
local reset_debug=0
- if [[ ${-/x/} != $- ]] ; then
+ if [[ $- == *x* ]] ; then
set +x
reset_debug=1
fi
diff --git a/bin/install-qa-check.d/90gcc-warnings
b/bin/install-qa-check.d/90gcc-warnings
index 1060618dfa..2e728268e1 100644
--- a/bin/install-qa-check.d/90gcc-warnings
+++ b/bin/install-qa-check.d/90gcc-warnings
@@ -10,7 +10,7 @@ gcc_warn_check() {
# In debug mode, this variable definition and corresponding
grep calls
# will produce false positives if they're shown in the trace.
local reset_debug=0
- if [[ ${-/x/} != $- ]] ; then
+ if [[ $- == *x* ]] ; then
set +x
reset_debug=1
fi
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index d037cf6776..2c3f2d389f 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -1002,7 +1002,7 @@ __ebuild_main() {
ewarn "pkg_${1}() is not defined: '${EBUILD##*/}'"
fi
export SANDBOX_ON="0"
- if [[ "${PORTAGE_DEBUG}" != "1" || "${-/x/}" != "$-" ]]; then
+ if [[ ${PORTAGE_DEBUG} != 1 || $- == *x* ]]; then
__ebuild_phase_with_hooks pkg_${1}
else
set -x
@@ -1069,7 +1069,7 @@ __ebuild_main() {
;;
esac
- if [[ "${PORTAGE_DEBUG}" != "1" || "${-/x/}" != "$-" ]]; then
+ if [[ ${PORTAGE_DEBUG} != 1 || $- == *x* ]]; then
__dyn_${1}
else
set -x
@@ -1083,7 +1083,7 @@ __ebuild_main() {
# for example, awking and piping a file in /tmp requires a temp
file to be created
# in /etc. If pkg_setup is in the sandbox, both our lilo and
apache ebuilds break.
export SANDBOX_ON="0"
- if [[ "${PORTAGE_DEBUG}" != "1" || "${-/x/}" != "$-" ]]; then
+ if [[ ${PORTAGE_DEBUG} != 1 || $- == *x* ]]; then
__dyn_${1}
else
set -x
[gentoo-commits] proj/portage:master commit in: misc/, bin/
commit: f99eb91227918d5bf0cc531e78bd597c73b7ce1f
Author: Kerin Millar plushkava net>
AuthorDate: Wed Jul 24 16:09:10 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Jul 26 10:22:41 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f99eb912
emerge-webrsync: actually honour the sync-webrsync-verify-signature attribute
The repo_has_webrsync_verify variable was always being set as the empty
string, despite later being compared to 1 in the arithmetic context.
Have it work in the way that was originally intended. This also
eliminates a superfluous use of the has() function, which was the reason
for my inspecting the code in the first place.
Also, apply the exact same fix to emerge-delta-webrsync.
Fixes: b5cd549e07f3d835cbe04e5b718cdd47d7ee69f5
Fixes: 0347637fe5033d3d8eb4fdafa1e86de171934819
Signed-off-by: Kerin Millar plushkava.net>
Signed-off-by: Sam James gentoo.org>
bin/emerge-webrsync| 9 +
misc/emerge-delta-webrsync | 9 +
2 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync
index caa4986da2..124bcaddcf 100755
--- a/bin/emerge-webrsync
+++ b/bin/emerge-webrsync
@@ -105,16 +105,17 @@ do_debug=0
keep=false
handle_pgp_setup() {
+ local attr repo_has_webrsync_verify webrsync_gpg
+
# WEBRSYNC_VERIFY_SIGNATURE=0: disable PGP verification
# WEBRSYNC_VERIFY_SIGNATURE=1: use gemato for verification, fallback to
regular gpg
# WEBRSYNC_VERIFY_SIGNATURE=2: use legacy FEATURES="webrsync-gpg"
WEBRSYNC_VERIFY_SIGNATURE=1
- has webrsync-gpg ${FEATURES} && webrsync_gpg=1 || webrsync_gpg=0
+ has webrsync-gpg ${FEATURES} && webrsync_gpg=1
- repo_has_webrsync_verify=$(
- has $(__repo_attr "${repo_name}" sync-webrsync-verify-signature
| LC_ALL=C tr '[:upper:]' '[:lower:]') true yes
- )
+ attr=$(__repo_attr "${repo_name}" sync-webrsync-verify-signature)
+ [[ ${attr,,} == @(true|yes) ]] && repo_has_webrsync_verify=1
if [[ -n ${PORTAGE_TEMP_GPG_DIR} ]] || [[ ${repo_has_webrsync_verify}
-eq 1 ]]; then
# If FEATURES=webrsync-gpg is enabled then allow direct
emerge-webrsync
diff --git a/misc/emerge-delta-webrsync b/misc/emerge-delta-webrsync
index a788cdb0eb..8550c15fe0 100755
--- a/misc/emerge-delta-webrsync
+++ b/misc/emerge-delta-webrsync
@@ -165,16 +165,17 @@ if [[ ! -d $STATE_DIR ]]; then
fi
handle_pgp_setup() {
+ local attr repo_has_webrsync_verify webrsync_gpg
+
# WEBRSYNC_VERIFY_SIGNATURE=0: disable PGP verification
# WEBRSYNC_VERIFY_SIGNATURE=1: use gemato for verification, fallback to
regular gpg
# WEBRSYNC_VERIFY_SIGNATURE=2: use legacy FEATURES="webrsync-gpg"
WEBRSYNC_VERIFY_SIGNATURE=1
- has webrsync-gpg ${FEATURES} && webrsync_gpg=1 || webrsync_gpg=0
+ has webrsync-gpg ${FEATURES} && webrsync_gpg=1
- repo_has_webrsync_verify=$(
- has $(__repo_attr "${repo_name}" sync-webrsync-verify-signature
| LC_ALL=C tr '[:upper:]' '[:lower:]') true yes
- )
+ attr=$(__repo_attr "${repo_name}" sync-webrsync-verify-signature)
+ [[ ${attr,,} == @(true|yes) ]] && repo_has_webrsync_verify=1
if [[ -n ${PORTAGE_TEMP_GPG_DIR} ]] || [[ ${repo_has_webrsync_verify}
-eq 1 ]]; then
# If FEATURES=webrsync-gpg is enabled then allow direct
emerge-webrsync
[gentoo-commits] proj/portage:master commit in: cnf/, /
commit: 47cb029accbeadf092beecfb2cc5ea7d4c2b275d
Author: Florian Schmaus gentoo org>
AuthorDate: Mon Jun 17 19:54:16 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Jul 21 09:06:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=47cb029a
make.globals: add bcachefs_effective.* and bcachefs.* to PORTAGE_XATTR_EXCLUDE
Just like btrfs the bcachefs filesystem makes heaviy use of xattrs.
In my case
emerge --config gentoo-kernel
failed with
install-xattr: setxattr() failed: Operation not supported
A strace revealed
setxattr("/boot/efi/8410476acd4b1e12c678fc815c18f551/6.9.5-gentoo-dist/linux",
"bcachefs_effective.background_co"..., "zstd", 4, 0) = -1 EOPNOTSUPP (Operation
not supported)
and indeed, the source file to the install operation was under
/usr/src which had background compression enabled. As a result,
install-xattr tried to copy the attribute over to the destination file
and failed.
Signed-off-by: Florian Schmaus gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1346
Signed-off-by: Sam James gentoo.org>
NEWS | 2 ++
cnf/make.globals | 3 ++-
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/NEWS b/NEWS
index 53ba9f5fb1..e8ba29df68 100644
--- a/NEWS
+++ b/NEWS
@@ -20,6 +20,8 @@ Bug fixes:
working ebuilds. Future EAPIs will need to adjust the logic
added by this change. See bug #907061.
+* make.globals: add bcachefs_effective.* and bcachefs.* to
PORTAGE_XATTR_EXCLUDE
+
* vartree, movefile: Warn when rewriting a symlink (bug #934514).
* repository: config: Allow a repository to be configured using one of its
diff --git a/cnf/make.globals b/cnf/make.globals
index 2bb7a65593..33e99e9ec3 100644
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -159,7 +159,8 @@ PORTAGE_TRUST_HELPER="/usr/bin/getuto"
# system.nfs4_acl attributes are irrelevant, see bug #475496.
# user.* attributes are not supported on tmpfs (bug 640290), but
# user.pax.* is supported with the patch from bug 470644.
-PORTAGE_XATTR_EXCLUDE="btrfs.* security.evm security.ima
+PORTAGE_XATTR_EXCLUDE="bcachefs.* bcachefs_effective.*
+ btrfs.* security.evm security.ima
security.selinux system.nfs4_acl user.apache_handler
user.Beagle.* user.dublincore.* user.mime_encoding user.xdg.*"
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/update/, lib/portage/tests/resolver/, ...
commit: b65da90fe378b5aff48b0d11db376a26f4ca3c90
Author: Markus Meier gentoo org>
AuthorDate: Wed Jul 3 13:19:16 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Jul 19 05:41:29 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b65da90f
convert f-strings to normal strings
that do not contain any {}.
detected with ruff
Signed-off-by: Markus Meier gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1355
Signed-off-by: Sam James gentoo.org>
lib/_emerge/BinpkgFetcher.py | 2 +-
lib/_emerge/BinpkgVerifier.py| 2 +-
lib/portage/gpkg.py | 4 ++--
lib/portage/tests/resolver/test_runtime_cycle_merge_order.py | 4 ++--
lib/portage/tests/update/test_move_ent.py| 2 +-
5 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index a357bac82d..bb6997536b 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -30,7 +30,7 @@ class BinpkgFetcher(CompositeTask):
binpkg_path = bintree._remotepkgs[instance_key].get("PATH")
if not binpkg_path:
raise FileNotFound(
-f"PATH not found in the binpkg index, the binhost's portage is
probably out of date."
+"PATH not found in the binpkg index, the binhost's portage is
probably out of date."
)
binpkg_format = get_binpkg_format(binpkg_path)
diff --git a/lib/_emerge/BinpkgVerifier.py b/lib/_emerge/BinpkgVerifier.py
index 7e044c6c43..dd3a70d3ae 100644
--- a/lib/_emerge/BinpkgVerifier.py
+++ b/lib/_emerge/BinpkgVerifier.py
@@ -47,7 +47,7 @@ class BinpkgVerifier(CompositeTask):
if bintree.dbapi.cpv_exists(self.pkg.cpv):
self.scheduler.output(
f"!!! Tried to use non-existent binary for
'{self.pkg.cpv}'\n"
-+ f"!!! Likely caused by an outdated index. Run 'emaint
binhost -f'.\n",
++ "!!! Likely caused by an outdated index. Run 'emaint
binhost -f'.\n",
log_path=self.logfile,
background=self.background,
)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index fdb54c69b8..06e2283ce1 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -384,7 +384,7 @@ class tar_stream_reader:
try:
if self.proc.wait() != os.EX_OK:
if not self.killed:
-writemsg(colorize("BAD", f"GPKG external program
failed.\n"))
+writemsg(colorize("BAD", "GPKG external program
failed.\n"))
raise CompressorOperationFailed("decompression failed")
finally:
self.proc.stdout.close()
@@ -2120,7 +2120,7 @@ class gpkg:
return the first one.
"""
if self.gpkg_version not in (os.path.basename(f) for f in
tar.getnames()):
-raise InvalidBinaryPackageFormat(f"Invalid gpkg file")
+raise InvalidBinaryPackageFormat("Invalid gpkg file")
if self.basename and self.prefix and not
self.prefix.startswith(self.basename):
writemsg(
diff --git a/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
b/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
index a695b25198..fb1c52f0ea 100644
--- a/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
+++ b/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py
@@ -204,8 +204,8 @@ class RuntimeCycleMergeOrderTestCase(TestCase):
debug=False,
user_config={
"make.conf": (
-f'ACCEPT_KEYWORDS="~x86"',
-f'USE="http2"',
+'ACCEPT_KEYWORDS="~x86"',
+'USE="http2"',
),
},
)
diff --git a/lib/portage/tests/update/test_move_ent.py
b/lib/portage/tests/update/test_move_ent.py
index 0b938dd287..99e725e1c6 100644
--- a/lib/portage/tests/update/test_move_ent.py
+++ b/lib/portage/tests/update/test_move_ent.py
@@ -287,7 +287,7 @@ class MoveEntTestCase(TestCase):
user_config={
"make.conf": (
f'BINPKG_FORMAT="{binpkg_format}"',
-f'FEATURES="binpkg-multi-instance
pkgdir-index-trusted"',
+'FEATURES="binpkg-multi-instance
pkgdir-index-trusted"',
),
},
debug=False,
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/
commit: 45fceb11558e3363390a4b58ab067603b418773e
Author: Eli Schwartz gentoo org>
AuthorDate: Mon Jul 8 03:45:05 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Jul 19 05:41:19 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=45fceb11
ebuild: try to make maintainer mode checks a bit less prone to FP
We sometimes get weird results. For example, in binutils/gdb, we get:
```
* QA Notice: Automake "maintainer mode" detected:
*
* checking for aclocal... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing aclocal-1.15
* checking for autoconf... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing autoconf
* checking for autoheader... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing autoheader
* checking for aclocal... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing aclocal-1.15
* checking for autoconf... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing autoconf
* checking for autoheader... ${SHELL}
/var/tmp/portage/dev-debug/gdb-15.1/work/gdb-15.1/missing autoheader
```
The current pattern looks for the definition of the "missing" script
followed by known autotools commands. With a bit more effort, we can
hopefully match the actual `make` commands that get produced as
maintainer targets. As far as I can tell, they always include an
attempt to cd to the project root and then run the maintainer target.
Signed-off-by: Eli Schwartz gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1359
Signed-off-by: Sam James gentoo.org>
lib/portage/package/ebuild/doebuild.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index c7fa1d1e06..403836b80b 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -2383,7 +2383,7 @@ def _check_build_log(mysettings, out=None):
# Configuration:
# Automake: ${SHELL}
/var/tmp/portage/dev-libs/yaz-3.0.47/work/yaz-3.0.47/config/missing --run
automake-1.10
am_maintainer_mode_re = re.compile(
-r"/missing( --run|'|) (automake|autoconf|autoheader|aclocal)"
+r"\bcd\b.*&&.*/missing( --run|'|)
(automake|autoconf|autoheader|aclocal)"
)
am_maintainer_mode_exclude_re = re.compile(r"^\s*Automake:\s")
[gentoo-commits] proj/portage:master commit in: man/, lib/portage/dbapi/, lib/portage/binrepo/, lib/portage/, lib/_emerge/
commit: 20cd76664c11991e59b7d72b782fea96259ff9af
Author: Zac Medico gentoo org>
AuthorDate: Wed Jul 17 02:14:34 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Jul 17 02:14:34 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=20cd7666
binrepos.conf: Support custom download location
Download packages to a custom location if it is configured
in binrepos.conf, instead of PKGDIR. If a custom download
location is not configured then inject downloaded packages
into PKGDIR as usual.
The binarytree download_required method should now be used
instead of the isremote method to check if download is
required, since a remote package may or may not be cached
in the custom location. The get_local_repo_location method
should be used to check if there is a custom download
location, and if there is then downloaded packages must
not be injected into PKGDIR.
If any packages from a repo with a custom download location
were injected into PKGDIR in the past, their identity will
be recognized and will not be re-downloaded to the custom
location.
Bug: https://bugs.gentoo.org/934784
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/Binpkg.py | 7 +++--
lib/_emerge/BinpkgFetcher.py| 8 -
lib/_emerge/BinpkgPrefetcher.py | 7 +
lib/_emerge/Scheduler.py| 8 +++--
lib/_emerge/actions.py | 2 +-
lib/portage/binrepo/config.py | 3 ++
lib/portage/dbapi/bintree.py| 70 +++--
lib/portage/versions.py | 6 +++-
man/portage.5 | 7 -
9 files changed, 106 insertions(+), 12 deletions(-)
diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
index 299ae7fbc9..437111fa10 100644
--- a/lib/_emerge/Binpkg.py
+++ b/lib/_emerge/Binpkg.py
@@ -170,7 +170,7 @@ class Binpkg(CompositeTask):
pkg_count = self.pkg_count
fetcher = None
-if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv):
+if self.opts.getbinpkg and self._bintree.download_required(pkg.cpv):
fetcher = BinpkgFetcher(
background=self.background,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
@@ -245,7 +245,10 @@ class Binpkg(CompositeTask):
pkg = self.pkg
pkg_count = self.pkg_count
-if self._fetched_pkg:
+if self._fetched_pkg and
self._bintree.get_local_repo_location(pkg.cpv):
+os.rename(self._fetched_pkg, self._pkg_allocated_path)
+pkg_path = self._pkg_allocated_path
+elif self._fetched_pkg:
stdout_orig = sys.stdout
stderr_orig = sys.stderr
out = io.StringIO()
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 19d08359f0..a357bac82d 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -34,8 +34,14 @@ class BinpkgFetcher(CompositeTask):
)
binpkg_format = get_binpkg_format(binpkg_path)
+getname_kwargs = {}
+if not bintree.get_local_repo_location(pkg.cpv):
+getname_kwargs.update(
+dict(allocate_new=True, remote_binpkg_format=binpkg_format)
+)
+
self.pkg_allocated_path = pkg.root_config.trees["bintree"].getname(
-pkg.cpv, allocate_new=True, remote_binpkg_format=binpkg_format
+pkg.cpv, **getname_kwargs
)
self.pkg_path = self.pkg_allocated_path + ".partial"
diff --git a/lib/_emerge/BinpkgPrefetcher.py b/lib/_emerge/BinpkgPrefetcher.py
index a8af30ca80..f7204bcc17 100644
--- a/lib/_emerge/BinpkgPrefetcher.py
+++ b/lib/_emerge/BinpkgPrefetcher.py
@@ -51,6 +51,13 @@ class BinpkgPrefetcher(CompositeTask):
self.wait()
return
+if self._bintree.get_local_repo_location(self.pkg.cpv):
+os.rename(self.pkg_path, self.pkg_allocated_path)
+self._current_task = None
+self.returncode = os.EX_OK
+self.wait()
+return
+
injected_pkg = None
stdout_orig = sys.stdout
stderr_orig = sys.stderr
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index 614df9e783..e23ebeb7ac 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -830,7 +830,7 @@ class Scheduler(PollScheduler):
elif (
pkg.type_name == "binary"
and "--getbinpkg" in self.myopts
-and pkg.root_config.trees["bintree"].isremote(pkg.cpv)
+and pkg.root_config.trees["bintree"].download_required(pkg.cpv)
):
prefetcher = BinpkgPrefetcher(
background=True, pkg=pkg, scheduler=self._sched_iface
@@ -939,7 +939,7 @@ class Scheduler(PollScheduler):
# Display fetch on stdout, so that it's always clear what
# is consuming time here.
-if bintree.isremote(x.cpv):
+if
[gentoo-commits] proj/portage:master commit in: lib/portage/repository/, /
commit: ea04a583fab9739dbe05e8d8381ed2803e7dce88 Author: James Le Cuirot gentoo org> AuthorDate: Tue Jul 9 13:07:58 2024 + Commit: James Le Cuirot gentoo org> CommitDate: Wed Jul 10 16:55:20 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ea04a583 config: Allow a repository to be configured using one of its aliases Currently, the `[name]` in repos.conf can only match the primary name. This is inconvenient if a repository wants to change its name without breaking existing configurations. This raises the question of whether to then use the primary name or the alias in the UI and in the vardb. I went with the primary name because this is presumably the name that the repository actually wants you to use. If the configured name matches neither the primary name nor an alias, then emaint sync will simply claim that it is not found, but that behaviour is unchanged from before. Bug: https://bugs.gentoo.org/935830 Closes: https://github.com/gentoo/portage/pull/1358 Signed-off-by: James Le Cuirot gentoo.org> NEWS | 3 +++ lib/portage/repository/config.py | 10 ++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index d5a03533d9..53ba9f5fb1 100644 --- a/NEWS +++ b/NEWS @@ -22,6 +22,9 @@ Bug fixes: * vartree, movefile: Warn when rewriting a symlink (bug #934514). +* repository: config: Allow a repository to be configured using one of its + aliases rather than its primary name (bug #935830). + portage-3.0.65 (2024-06-04) -- diff --git a/lib/portage/repository/config.py b/lib/portage/repository/config.py index c9dfffa22c..46a2d83856 100644 --- a/lib/portage/repository/config.py +++ b/lib/portage/repository/config.py @@ -943,7 +943,9 @@ class RepoConfigLoader: del prepos[repo_name] continue -if repo.name != repo_name: +if repo.name != repo_name and ( +repo.aliases is None or repo_name not in repo.aliases +): writemsg_level( "!!! %s\n" % _( @@ -957,13 +959,13 @@ class RepoConfigLoader: del prepos[repo_name] continue -location_map[repo.location] = repo_name -treemap[repo_name] = repo.location +location_map[repo.location] = repo.name +treemap[repo.name] = repo.location # Add alias mappings, but never replace unaliased mappings. for repo_name, repo in list(prepos.items()): names = set() -names.add(repo_name) +names.add(repo.name) if repo.aliases: aliases = stack_lists([repo.aliases], incremental=True) names.update(aliases)
[gentoo-commits] proj/portage:master commit in: src/
commit: 918f4a4aa82f47291deac496d13c2761f306e01f
Author: Zac Medico gentoo org>
AuthorDate: Sun Jun 16 19:44:37 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Jun 28 18:15:38 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=918f4a4a
Allow GIL to be disabled in whirlpool C extension
In 3.13 python extensions need to declare support for GIL features, for
example if they don't declare Py_MOD_GIL_NOT_USED then it will cause the
GIL to be enabled even when python was launched in free-threaded mode.
This requires "multi-phase initialization" because Py_mod_create is
incompatible with m_slots. There's a PyUnstable_Module_SetGIL() function
that can be used with single-phase init, but it's an unstable API, so
it's better to use multi-phase init. There's no need to use
PyModule_GetState() because the whirlpool module has no mutable state.
Bug: https://bugs.gentoo.org/934220
Signed-off-by: Zac Medico gentoo.org>
src/portage_util__whirlpool.c | 46 +++
1 file changed, 29 insertions(+), 17 deletions(-)
diff --git a/src/portage_util__whirlpool.c b/src/portage_util__whirlpool.c
index 6c9421e56b..4af9307c71 100644
--- a/src/portage_util__whirlpool.c
+++ b/src/portage_util__whirlpool.c
@@ -1112,30 +1112,42 @@ static PyTypeObject WhirlpoolType = {
.tp_methods = Whirlpool_methods,
};
+static int
+whirlpool_exec(PyObject *mod)
+{
+if (PyType_Ready() < 0)
+return -1;
+
+Py_INCREF();
+if (PyModule_AddObject(mod, "Whirlpool", (PyObject *) ) < 0)
{
+Py_DECREF();
+return -1;
+}
+
+return 0;
+}
+
+static PyModuleDef_Slot _whirlpoolmodule_slots[] = {
+{Py_mod_exec, whirlpool_exec},
+#ifdef Py_MOD_PER_INTERPRETER_GIL_SUPPORTED
+{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+#endif
+#ifdef Py_MOD_GIL_NOT_USED
+{Py_mod_gil, Py_MOD_GIL_NOT_USED},
+#endif
+{0, NULL},
+};
+
static PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT,
.m_name = "_whirlpool",
.m_doc = "Reference Whirlpool implementation",
-.m_size = -1,
+.m_size = 0,
+.m_slots = _whirlpoolmodule_slots,
};
PyMODINIT_FUNC
PyInit__whirlpool(void)
{
-PyObject *m;
-if (PyType_Ready() < 0)
-return NULL;
-
-m = PyModule_Create();
-if (m == NULL)
-return NULL;
-
-Py_INCREF();
-if (PyModule_AddObject(m, "Whirlpool", (PyObject *) ) < 0) {
-Py_DECREF();
-Py_DECREF(m);
-return NULL;
-}
-
-return m;
+return PyModuleDef_Init();
}
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, /, lib/portage/util/
commit: 4d17764863896903df4d18fe7c6b45635a18f6c4
Author: Ulrich Müller gentoo org>
AuthorDate: Wed Jun 19 08:08:04 2024 +
Commit: Ulrich Müller gentoo org>
CommitDate: Wed Jun 19 18:17:06 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4d177648
vartree, movefile: Warn when rewriting a symlink
See PMS section 13.4.1 (Rewriting):
Any absolute symlink whose link starts with D must be rewritten with
the leading D removed. The package manager should issue a notice when
doing this.
Bug: https://bugs.gentoo.org/934514
Signed-off-by: Ulrich Müller gentoo.org>
NEWS | 2 ++
lib/portage/dbapi/vartree.py | 10 ++
lib/portage/util/movefile.py | 5 +
3 files changed, 17 insertions(+)
diff --git a/NEWS b/NEWS
index b4b378e7a0..d5a03533d9 100644
--- a/NEWS
+++ b/NEWS
@@ -20,6 +20,8 @@ Bug fixes:
working ebuilds. Future EAPIs will need to adjust the logic
added by this change. See bug #907061.
+* vartree, movefile: Warn when rewriting a symlink (bug #934514).
+
portage-3.0.65 (2024-06-04)
--
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index beb1a6486f..0c41d408c3 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -5563,6 +5563,16 @@ class dblink:
myabsto = myabsto.lstrip(sep)
if self.settings and self.settings["D"]:
if myto.startswith(self.settings["D"]):
+self._eqawarn(
+"preinst",
+[
+_(
+"QA Notice: Absolute symlink %s points to
%s inside the image directory.\n"
+"Removing the leading %s from its path."
+)
+% (mydest, myto, self.settings["D"])
+],
+)
myto = myto[len(self.settings["D"]) - 1 :]
# myrealto contains the path of the real file to which this
symlink points.
# we can simply test for existence of this file to see if the
target has been merged yet
diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py
index 75100a3acd..7b880d2e3e 100644
--- a/lib/portage/util/movefile.py
+++ b/lib/portage/util/movefile.py
@@ -210,6 +210,11 @@ def movefile(
try:
target = os.readlink(src)
if mysettings and "D" in mysettings and
target.startswith(mysettings["D"]):
+writemsg(
+f"!!! {_('Absolute symlink points to image
directory.')}\n",
+noiselevel=-1,
+)
+writemsg(f"!!! {dest} -> {target}\n", noiselevel=-1)
target = target[len(mysettings["D"]) - 1 :]
# Atomically update the path if it exists.
try:
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/
commit: 819c86325ce6b966d56d5f4848cf762767ab2bf9 Author: Sam James gentoo org> AuthorDate: Mon Jun 17 00:24:52 2024 + Commit: Sam James gentoo org> CommitDate: Mon Jun 17 00:24:52 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=819c8632 ebuild: fix typo in comment Signed-off-by: Sam James gentoo.org> lib/portage/package/ebuild/doebuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py index 21ff5a77fd..c7fa1d1e06 100644 --- a/lib/portage/package/ebuild/doebuild.py +++ b/lib/portage/package/ebuild/doebuild.py @@ -2390,7 +2390,7 @@ def _check_build_log(mysettings, out=None): make_jobserver_re = re.compile(r"g?make\[\d+\]: warning: jobserver unavailable:") make_jobserver = [] -# we deduplicate these since they is repeated for every setup.py call +# we deduplicate these since they are repeated for every setup.py call setuptools_warn = set() setuptools_warn_re = re.compile(r".*\/setuptools\/.*: .*Warning: (.*)") # skip useless version normalization warnings
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/
commit: 11c51d7c78f48d4c9842e0d475c26b7068f18c3e
Author: Eli Schwartz gmail com>
AuthorDate: Mon Jun 17 00:12:29 2024 +
Commit: Sam James gentoo org>
CommitDate: Mon Jun 17 00:20:41 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=11c51d7c
ebuild: fix maintainer mode checks to work with modern autotools
Modern autotools does not use the --run argument to "missing", so the
check essentially never ever ever ever fired anywhere.
The GNU "missing" script is actually allowed to be run by any software
at all, so checking for "missing --run" was always wrong. Indeed, there
are some packages which use it for their own purposes and added
suppressions for this FP.
The correct solution really is to check for *maintainer mode* by
checking whether *maintainer* programs are run (via "missing"). This
also means we get to check for specific programs which autotools.eclass
would be capable of handling, and don't need to arbitrarily exclude
stuff like help2man (???) which makes things somewhat simpler.
It should be noted that I have observed 3 scenarios for the missing
script to be run via:
- the missing script is surrounded by single quotes, followed by the
unquoted command
- the missing script is unquoted, and is followed by the unquoted
command
- legacy: the missing script is unquoted and is followed by --run
We have to handle all three cases via a regex group.
Signed-off-by: Eli Schwartz gmail.com>
Signed-off-by: Sam James gentoo.org>
lib/portage/package/ebuild/doebuild.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index 6641cd8341..21ff5a77fd 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -2382,10 +2382,10 @@ def _check_build_log(mysettings, out=None):
#
# Configuration:
# Automake: ${SHELL}
/var/tmp/portage/dev-libs/yaz-3.0.47/work/yaz-3.0.47/config/missing --run
automake-1.10
-am_maintainer_mode_re = re.compile(r"/missing --run ")
-am_maintainer_mode_exclude_re = re.compile(
-r"(/missing --run
(autoheader|autotest|help2man|makeinfo)|^\s*Automake:\s)"
+am_maintainer_mode_re = re.compile(
+r"/missing( --run|'|) (automake|autoconf|autoheader|aclocal)"
)
+am_maintainer_mode_exclude_re = re.compile(r"^\s*Automake:\s")
make_jobserver_re = re.compile(r"g?make\[\d+\]: warning: jobserver
unavailable:")
make_jobserver = []
[gentoo-commits] proj/portage:master commit in: /, bin/
commit: 19e750b6880c1ffba55c4d309fbf33d5e746ee33
Author: Sam James gentoo org>
AuthorDate: Thu Jun 13 02:20:27 2024 +
Commit: Sam James gentoo org>
CommitDate: Thu Jun 13 02:23:38 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=19e750b6
bin/ebuild.sh: disable globskipdots too in Bash 5.2
See 69cac73ba0a7bcf2e2cff88c60d389895a550623.
globskipdots is a new option in bash-5.2 that is also default-on
in that release. The default value is not gated by BASH_COMPAT (see bug
#907061),
hence we need to disable it for older Bashes to avoid behaviour changes in
ebuilds
and eclasses.
Bug: https://bugs.gentoo.org/907061
Signed-off-by: Sam James gentoo.org>
NEWS | 6 ++
bin/ebuild.sh | 8
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/NEWS b/NEWS
index 5aa485d264..b4b378e7a0 100644
--- a/NEWS
+++ b/NEWS
@@ -14,6 +14,12 @@ Features:
as well as / for all EAPIs rather than instead of / for EAPI 6 and below
(bug #435066).
+Bug fixes:
+* ebuild: Handle Bash 5.2's change in behavior which enables the shopt
+ 'globskipdots' by default. This is needed to avoid breaking existing
+ working ebuilds. Future EAPIs will need to adjust the logic
+ added by this change. See bug #907061.
+
portage-3.0.65 (2024-06-04)
--
diff --git a/bin/ebuild.sh b/bin/ebuild.sh
index c9f7c04e28..8b1e0861a8 100755
--- a/bin/ebuild.sh
+++ b/bin/ebuild.sh
@@ -18,7 +18,7 @@ source "${PORTAGE_BIN_PATH}/isolated-functions.sh" || exit 1
# used instead.
__check_bash_version() {
# Figure out which min version of bash we require.
- # Adjust patsub_replacement logic below on new EAPI!
+ # Adjust patsub_replacement/globskipdots logic below on new EAPI!
local maj min
if ___eapi_bash_3_2 ; then
maj=3 min=2
@@ -52,16 +52,16 @@ __check_bash_version() {
shopt -s compat32
fi
- # patsub_replacement is a new option in bash-5.2 that is also default-on
+ # patsub_replacement and globskipdots are new options in bash-5.2 that
are also default-on
# in that release. The default value is not gated by BASH_COMPAT (see
bug #881383),
# hence we need to disable it for older Bashes to avoid behaviour
changes in ebuilds
# and eclasses.
#
# New EAPI note: a newer EAPI (after 8) may well adopt Bash 5.2 as its
minimum version.
# If it does, this logic will need to be adjusted to only disable
patsub_replacement
- # for < ${new_api}!
+ # and globskipdots for < ${new_api}!
if (( BASH_VERSINFO[0] >= 6 || ( BASH_VERSINFO[0] == 5 &&
BASH_VERSINFO[1] >= 2 ) )) ; then
- shopt -u patsub_replacement
+ shopt -u patsub_replacement globskipdots
fi
}
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: f620a0769a509966295954c2b0c76e46e8fb4289
Author: Zac Medico gentoo org>
AuthorDate: Sun Jun 2 21:53:04 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Jun 9 17:53:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f620a076
tar_safe_extract: Use tarfile.fully_trusted_filter
This suppresses a DeprecationWarning triggered because the
tarfile.data_filter will become the new default in python3.14.
The fully_trusted filter should be suitable here because
tar_safe_extract already performs security validation on
tar members prior to extraction.
Bug: https://bugs.gentoo.org/933433
Signed-off-by: Zac Medico gentoo.org>
lib/portage/gpkg.py | 9 +
1 file changed, 9 insertions(+)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 9606f6d3c8..fdb54c69b8 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -628,6 +628,15 @@ class tar_safe_extract:
if self.closed:
raise OSError("Tar file is closed.")
temp_dir = tempfile.TemporaryDirectory(dir=dest_dir)
+# The below tar member security checks can be refactored as a filter
function
+# that raises an exception. Use tarfile.fully_trusted_filter for now,
which
+# is simply an identity function:
+# def fully_trusted_filter(member, dest_path):
+# return member
+try:
+self.tar.extraction_filter = tarfile.fully_trusted_filter
+except AttributeError:
+pass
try:
while True:
member = self.tar.next()
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/, lib/portage/sync/modules/
commit: 8bb7aecf7e5c922911192d0df63853c5c75d9f8a
Author: Alexey Gladkov kernel org>
AuthorDate: Tue Jun 4 15:31:06 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Jun 4 15:40:25 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8bb7aecf
sync/zipfile: Install zipfile sync method
Add files that were accidentally forgotten when adding zipfile sync
method.
Fixes: 80445d9b0 ("sync: Add method to download zip archives")
Signed-off-by: Alexey Gladkov kernel.org>
Closes: https://github.com/gentoo/portage/pull/1340
Signed-off-by: Zac Medico gentoo.org>
lib/portage/sync/modules/meson.build | 1 +
lib/portage/sync/modules/zipfile/meson.build | 8
2 files changed, 9 insertions(+)
diff --git a/lib/portage/sync/modules/meson.build
b/lib/portage/sync/modules/meson.build
index fab2878e92..ba0b6f278b 100644
--- a/lib/portage/sync/modules/meson.build
+++ b/lib/portage/sync/modules/meson.build
@@ -12,3 +12,4 @@ subdir('mercurial')
subdir('rsync')
subdir('svn')
subdir('webrsync')
+subdir('zipfile')
diff --git a/lib/portage/sync/modules/zipfile/meson.build
b/lib/portage/sync/modules/zipfile/meson.build
new file mode 100644
index 00..46006aea7e
--- /dev/null
+++ b/lib/portage/sync/modules/zipfile/meson.build
@@ -0,0 +1,8 @@
+py.install_sources(
+[
+'zipfile.py',
+'__init__.py',
+],
+subdir : 'portage/sync/modules/zipfile',
+pure : not native_extensions
+)
[gentoo-commits] proj/portage:master commit in: /
commit: 4dccee4b7f57be075e4d2d987181f0d88ae19b14 Author: Sam James gentoo org> AuthorDate: Tue Jun 4 01:27:02 2024 + Commit: Sam James gentoo org> CommitDate: Tue Jun 4 01:27:02 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=4dccee4b NEWS: prepare for Signed-off-by: Sam James gentoo.org> NEWS | 64 +++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 7c1eb99616..bb48e3819f 100644 --- a/NEWS +++ b/NEWS @@ -9,7 +9,69 @@ Release notes take the form of the following optional categories: portage-3.0.65 (UNRELEASED) -- -TODO +Features: +* binrepos.conf: Support "frozen" attribute (bug #932739). + + This avoids mutating the local index cache by fetching changes from a + binpkg host when running emerge repeatedly. It's useful for consistency + between emerge runs while completing an update. + + It should only be used temporarily, not set on a permanent basis. + +* bintree: Add REPO_REVISIONS to Packages index header (bug #924772). + + Portage will now store synced revisions at /var/lib/portage/repo_revisions. + + This associates a binpkg being built with the repository state / revision + at a time, which will ultimately later allow pinning binhost clients + to <= the last commit the binhost was updated at. + + It's also useful for pinning source builds from the same revision as + when binary builds were done for consistency and to minimize rebuilds and + maximize binpkg use. + +* mtimedb: Record binpkg BUILD_ID and BUILD_TIME (bug #933442). + + This fixes 'emerge --resume' to correctly pick up binpkgs and not + recalculate. + +Bug fixes: +* unpack: Skip unrecognized file formats silently, fixing PMS compliance. + +* estrip: Don't use splitdebug for .o files (bug #787623). + + This fixes splitdebug mangling debug information with Clang. + +* sync: git: Add safe.directory for getting HEAD commit (bug #930992). + +* install-qa-checks.d: 90config-impl-decl: Suppress known false positives + for gnulib on glibc and musl. + +* CONFIG_PROTECT: Don't use shlex.split (as it's a whitespace-separated list). + +* depgraph: Fix crash with --useoldpkg with subslot bumps (bug #932804). + +* depgraph: find_smallest_cycle: Revert optimization which caused + merge order regression (bug #922629). + + A new testcase is also added. + +* sync: Don't use IPv6 for rsync when disabled (bug #927241). + +* atomic_ofstream: Use mkstemp instead of PID for PID namespace safety (bug #851015). + +* MergeProcess: Pass bintree to subprocess (bug #933297). + + This fixes FEATURES="downgrade-backup". + +* tar_stream_writer: Improve error handling (bug #933385). + +Cleanups: +* Scheduler: Improve legibility. + +* portage.util.shlex_split: Drop legacy Python 2 wrapper. + +* eapi.sh: Drop obsolete ___eapi_has_dohtml_deprecated(). portage-3.0.64 (2024-04-28) --
[gentoo-commits] proj/portage:master commit in: /
commit: f03998622e0960388e903de0d6d24bdf6881e567 Author: Sam James gentoo org> AuthorDate: Tue Jun 4 01:27:26 2024 + Commit: Sam James gentoo org> CommitDate: Tue Jun 4 01:27:26 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f0399862 NEWS, meson.build: prepare for portage-3.0.65 Signed-off-by: Sam James gentoo.org> NEWS| 2 +- meson.build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index bb48e3819f..2dbe164617 100644 --- a/NEWS +++ b/NEWS @@ -6,7 +6,7 @@ Release notes take the form of the following optional categories: * Bug fixes * Cleanups -portage-3.0.65 (UNRELEASED) +portage-3.0.65 (2024-06-04) -- Features: diff --git a/meson.build b/meson.build index 3d1a27b70e..673c1e631c 100644 --- a/meson.build +++ b/meson.build @@ -1,7 +1,7 @@ project( 'portage', 'c', -version : '3.0.64', +version : '3.0.65', license : 'GPL-2.0-or-later', meson_version : '>=0.58.0' )
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/_emerge/
commit: 3a9f2c09eb75f47cf3ae15fa4ebe671548a66870
Author: Zac Medico gentoo org>
AuthorDate: Mon Jun 3 01:18:44 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Jun 3 01:55:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3a9f2c09
Add binpkgs info to mtimedb resume data
In order to fix emerge --resume and --keep-going to make
appropriate binary package selections, store a list of
binpkgs in the resume data. By adding the data as a new
key which older versions of portage will ignore, the
extension is backward compatible.
Without this fix, emerge --resume and --keep-going make
poor package selection choices which do not account for
the --binpkg-respect-use option.
Bug: https://bugs.gentoo.org/933442
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/Scheduler.py | 13 +
lib/_emerge/actions.py | 7 +++
lib/_emerge/depgraph.py| 21 +
lib/portage/tests/util/test_mtimedb.py | 30 +-
4 files changed, 66 insertions(+), 5 deletions(-)
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index d913cd2dc6..614df9e783 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -2116,6 +2116,19 @@ class Scheduler(PollScheduler):
for x in self._mergelist
if isinstance(x, Package) and x.operation == "merge"
]
+# Store binpkgs using the same keys as $PKGDIR/Packages plus EROOT.
+mtimedb["resume"]["binpkgs"] = [
+{
+"CPV": str(x.cpv),
+"BUILD_ID": x.cpv.build_id,
+"BUILD_TIME": x.cpv.build_time,
+"MTIME": x.cpv.mtime,
+"SIZE": x.cpv.file_size,
+"EROOT": x.root,
+}
+for x in self._mergelist
+if isinstance(x, Package) and x.type_name == "binary"
+]
mtimedb.commit()
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 512e470ad6..43d936fd14 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -216,6 +216,13 @@ def action_build(
if not isinstance(favorites, list):
del mtimedb[k]
continue
+binpkgs = resume_data.get("binpkgs")
+if binpkgs and (
+not isinstance(binpkgs, list)
+or any(not isinstance(x, dict) for x in binpkgs)
+):
+del mtimedb[k]
+continue
resume = False
if "--resume" in myopts and ("resume" in mtimedb or "resume_backup" in
mtimedb):
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 3adc04bcfb..a05404d9c2 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -11066,6 +11066,13 @@ class depgraph:
else:
args = []
+binpkgs_map = {}
+binpkgs = resume_data.get("binpkgs")
+if binpkgs:
+for x in binpkgs:
+if isinstance(x, dict) and "EROOT" in x and "CPV" in x:
+binpkgs_map[(x["EROOT"], x["CPV"])] = x
+
serialized_tasks = []
masked_tasks = []
for x in mergelist:
@@ -11096,8 +11103,22 @@ class depgraph:
except InvalidAtom:
continue
+if pkg_type == "binary":
+binpkg_info = binpkgs_map.get((myroot, pkg_key))
+else:
+binpkg_info = False
+
pkg = None
for pkg in self._iter_match_pkgs(root_config, pkg_type, atom):
+if binpkg_info:
+if not (
+pkg.cpv.build_id == binpkg_info.get("BUILD_ID")
+and pkg.cpv.build_time == binpkg_info.get("BUILD_TIME")
+and pkg.cpv.mtime == binpkg_info.get("MTIME")
+and pkg.cpv.file_size == binpkg_info.get("SIZE")
+):
+continue
+
if not self._pkg_visibility_check(
pkg
) or self._frozen_config.excluded_pkgs.findAtomForPackage(
diff --git a/lib/portage/tests/util/test_mtimedb.py
b/lib/portage/tests/util/test_mtimedb.py
index d80b4f1daf..6e3c2bee76 100644
--- a/lib/portage/tests/util/test_mtimedb.py
+++ b/lib/portage/tests/util/test_mtimedb.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Gentoo Foundation
+# Copyright 2022-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from unittest.mock import patch, mock_open
@@ -30,6 +30,16 @@ _ONE_RESUME_LIST_JSON = b"""{
"/usr/local/lib64": 1711784303
},
"resume": {
+ "binpkgs": [
+ {
+ "CPV": "another-cat/another-package-4.3.2-r1",
+ "BUILD_ID": 1,
+ "BUILD_TIME": 1710959527,
+ "MTIME": 1710966082,
+
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: eb855b8cd1248f49649003dcfb9bf009b70e88cb
Author: Zac Medico gentoo org>
AuthorDate: Sun Jun 2 17:56:44 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Jun 2 18:05:35 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=eb855b8c
tar_stream_writer: Add missing error attribute
This attribute was previously initialized only
in an exception handler.
Fixes: b8c3f38ec5ee ("Add more error handling for binpkgs")
Bug: https://bugs.gentoo.org/933385
Signed-off-by: Zac Medico gentoo.org>
lib/portage/gpkg.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index fc4d7b1fb6..9606f6d3c8 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -100,6 +100,7 @@ class tar_stream_writer:
self.closed = False
self.container = container
self.killed = False
+self.error = False
self.tar_format = tar_format
self.tarinfo = tarinfo
self.uid = uid
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 1a7fc63d20ad2e1292be3697c105c2d7e1691f91
Author: Zac Medico gentoo org>
AuthorDate: Sat Jun 1 04:56:45 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Jun 1 19:18:35 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1a7fc63d
MergeProcess: Pass bintree to subprocess
It's required for FEATURES=*-backup.
Fixes: b9a85ff987ea ("MergeProcess: Support QueryCommand with spawn start
method")
Bug: https://bugs.gentoo.org/933297
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dbapi/_MergeProcess.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/lib/portage/dbapi/_MergeProcess.py
b/lib/portage/dbapi/_MergeProcess.py
index d9ab2b47aa..34e39eb229 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2023 Gentoo Authors
+# Copyright 2010-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -183,11 +183,12 @@ class MergeProcess(ForkProcess):
# Since the entire QueryCommand._db is not required, only pass
# in tree types that QueryCommand specifically requires.
+# NOTE: For FEATURES=*-backup bintree is needed (bug 933297).
child_db = {}
parent_db = portage.db if QueryCommand._db is None else
QueryCommand._db
for root in parent_db:
child_db[root] = {}
-for tree_type in ("vartree", "porttree"):
+for tree_type in ("bintree", "porttree", "vartree"):
child_db[root][tree_type] = parent_db[root][tree_type]
self.target = functools.partial(
[gentoo-commits] proj/portage:master commit in: bin/ebuild-helpers/, bin/
commit: 5e1477724ec3e284c05dcbf71c45ca35f1ca100c
Author: Ulrich Müller gentoo org>
AuthorDate: Sun May 26 08:59:42 2024 +
Commit: Ulrich Müller gentoo org>
CommitDate: Sat Jun 1 11:11:35 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5e147772
eapi.sh: Drop ___eapi_has_dohtml_deprecated()
This was added because of a council decision 10 years ago:
https://projects.gentoo.org/council/meeting-logs/20140909-summary.txt
It has outlived its usefulness since dohtml is banned in EAPI 7 and
later.
Signed-off-by: Ulrich Müller gentoo.org>
bin/eapi.sh | 6 +-
bin/ebuild-helpers/dohtml | 6 +-
2 files changed, 2 insertions(+), 10 deletions(-)
diff --git a/bin/eapi.sh b/bin/eapi.sh
index ae815f3a63..5e894e17dd 100644
--- a/bin/eapi.sh
+++ b/bin/eapi.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# Copyright 2012-2023 Gentoo Authors
+# Copyright 2012-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# PHASES
@@ -80,10 +80,6 @@ ___eapi_has_dohtml() {
[[ ${1-${EAPI-0}} =~ ^(0|1|2|3|4|4-slot-abi|5|6)$ ]]
}
-___eapi_has_dohtml_deprecated() {
- [[ ${1-${EAPI-0}} == 6 ]]
-}
-
___eapi_has_dolib_libopts() {
[[ ${1-${EAPI-0}} =~ ^(0|1|2|3|4|4-slot-abi|5|6)$ ]]
}
diff --git a/bin/ebuild-helpers/dohtml b/bin/ebuild-helpers/dohtml
index 4d4efd496c..b8929ceff8 100755
--- a/bin/ebuild-helpers/dohtml
+++ b/bin/ebuild-helpers/dohtml
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# Copyright 2009-2018 Gentoo Foundation
+# Copyright 2009-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
@@ -9,10 +9,6 @@ if ! ___eapi_has_dohtml; then
exit 1
fi
-if ___eapi_has_dohtml_deprecated; then
- eqawarn "QA Notice: '${0##*/}' is deprecated in EAPI '${EAPI}'"
-fi
-
# Use safe cwd, avoiding unsafe import for bug #469338.
export __PORTAGE_HELPER_CWD=${PWD}
cd "${PORTAGE_PYM_PATH}" || die
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/, lib/portage/util/
commit: 120b2ec988eebf6cd90365d5b50a1a718eebb116
Author: Zac Medico gentoo org>
AuthorDate: Tue May 28 05:37:13 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue May 28 05:37:13 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=120b2ec9
atomic_ofstream: fix follow_symlinks fallback and default file mode
Handle OSError from mkstemp for (default) follow_symlinks mode,
not following the symlink if necessary (the target's parent may
not exist or may be readonly). This restores the fallback
behavior that existed before the introduction of mkstemp in
commit de19f3a7215d.
Handle missing _file and _tmp_name attributes during close.
Also set the default file mode respecting umask if a previous
file does not exist, which fixes the mode of CONTENTS files
since mkstemp.
Fixes: de19f3a7215d ("atomic_ofstream: Use mkstemp rather than getpid (pid
namespace safety)")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/meson.build | 1 +
lib/portage/tests/util/test_atomic_ofstream.py | 85 ++
lib/portage/util/__init__.py | 45 +-
3 files changed, 116 insertions(+), 15 deletions(-)
diff --git a/lib/portage/tests/util/meson.build
b/lib/portage/tests/util/meson.build
index 010dfa7849..7f4db871f4 100644
--- a/lib/portage/tests/util/meson.build
+++ b/lib/portage/tests/util/meson.build
@@ -1,5 +1,6 @@
py.install_sources(
[
+'test_atomic_ofstream.py',
'test_checksum.py',
'test_digraph.py',
'test_file_copier.py',
diff --git a/lib/portage/tests/util/test_atomic_ofstream.py
b/lib/portage/tests/util/test_atomic_ofstream.py
new file mode 100644
index 00..bbaf0f1b06
--- /dev/null
+++ b/lib/portage/tests/util/test_atomic_ofstream.py
@@ -0,0 +1,85 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import os
+import stat
+import tempfile
+
+from portage.tests import TestCase
+from portage.util import atomic_ofstream
+
+
+class AtomicOFStreamTestCase(TestCase):
+def test_enospc_rollback(self):
+file_name = "foo"
+start_dir = os.getcwd()
+with tempfile.TemporaryDirectory() as tempdir:
+try:
+os.chdir(tempdir)
+with self.assertRaises(OSError):
+with atomic_ofstream(file_name) as f:
+f.write("hello")
+raise OSError(errno.ENOSPC, "No space left on device")
+self.assertFalse(os.path.exists(file_name))
+self.assertEqual(os.listdir(tempdir), [])
+finally:
+os.chdir(start_dir)
+
+def test_open_failure(self):
+file_name = "bad/path"
+start_dir = os.getcwd()
+with tempfile.TemporaryDirectory() as tempdir:
+try:
+os.chdir(tempdir)
+with self.assertRaises(OSError):
+with atomic_ofstream(file_name):
+pass
+self.assertEqual(os.listdir(tempdir), [])
+finally:
+os.chdir(start_dir)
+
+def test_broken_symlink(self):
+content = "foo"
+broken_symlink = "symlink"
+symlink_targets = (("foo/bar/baz", False), ("baz", True))
+start_dir = os.getcwd()
+for symlink_target, can_follow in symlink_targets:
+with tempfile.TemporaryDirectory() as tempdir:
+try:
+os.chdir(tempdir)
+with open(broken_symlink, "w") as f:
+default_file_mode =
stat.S_IMODE(os.fstat(f.fileno()).st_mode)
+os.unlink(broken_symlink)
+os.symlink(symlink_target, broken_symlink)
+with atomic_ofstream(broken_symlink) as f:
+f.write(content)
+with open(broken_symlink) as f:
+self.assertEqual(f.read(), content)
+self.assertEqual(os.path.islink(broken_symlink),
can_follow)
+self.assertEqual(
+stat.S_IMODE(os.stat(broken_symlink).st_mode),
default_file_mode
+)
+finally:
+os.chdir(start_dir)
+
+def test_preserves_mode(self):
+file_name = "foo"
+file_mode = 0o604
+start_dir = os.getcwd()
+with tempfile.TemporaryDirectory() as tempdir:
+try:
+os.chdir(tempdir)
+with open(file_name, "wb"):
+pass
+self.assertNotEqual(stat.S_IMODE(os.stat(file_name).st_mode),
file_mode)
+os.chmod(file_name, file_mode)
+st_before = os.stat(file_name)
+self.assertEqual(stat.S_IMODE(st_before.st_mode), file_mode)
+with atomic_ofstream(file_name):
+
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: de19f3a7215d64d22dcc0f779314de1f1199963f Author: Zac Medico gentoo org> AuthorDate: Mon May 27 18:47:30 2024 + Commit: Zac Medico gentoo org> CommitDate: Mon May 27 20:01:49 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=de19f3a7 atomic_ofstream: Use mkstemp rather than getpid (pid namespace safety) Bug: https://bugs.gentoo.org/851015 Signed-off-by: Zac Medico gentoo.org> lib/portage/util/__init__.py | 47 ++-- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py index 0c88068dda..f338f274aa 100644 --- a/lib/portage/util/__init__.py +++ b/lib/portage/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2004-2023 Gentoo Authors +# Copyright 2004-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from portage.cache.mappings import UserDict @@ -70,6 +70,7 @@ import shlex import stat import string import sys +import tempfile import traceback import glob from typing import Optional, TextIO @@ -1446,21 +1447,22 @@ class atomic_ofstream(AbstractContextManager, ObjectProxy): if follow_links: canonical_path = os.path.realpath(filename) object.__setattr__(self, "_real_name", canonical_path) -tmp_name = "%s.%i" % (canonical_path, portage.getpid()) +parent, basename = os.path.split(canonical_path) +fd, tmp_name = tempfile.mkstemp(prefix=basename, dir=parent) +object.__setattr__(self, "_tmp_name", tmp_name) try: object.__setattr__( self, "_file", open_func( -_unicode_encode( -tmp_name, encoding=_encodings["fs"], errors="strict" -), +fd, mode=mode, **kargs, ), ) return -except OSError as e: +except OSError: +os.close(fd) if canonical_path == filename: raise # Ignore this error, since it's irrelevant @@ -1468,16 +1470,22 @@ class atomic_ofstream(AbstractContextManager, ObjectProxy): # new error if necessary. object.__setattr__(self, "_real_name", filename) -tmp_name = "%s.%i" % (filename, portage.getpid()) -object.__setattr__( -self, -"_file", -open_func( -_unicode_encode(tmp_name, encoding=_encodings["fs"], errors="strict"), -mode=mode, -**kargs, -), -) +parent, basename = os.path.split(filename) +fd, tmp_name = tempfile.mkstemp(prefix=basename, dir=parent) +object.__setattr__(self, "_tmp_name", tmp_name) +try: +object.__setattr__( +self, +"_file", +open_func( +fd, +mode=mode, +**kargs, +), +) +except OSError: +os.close(fd) +raise def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: @@ -1498,13 +1506,14 @@ class atomic_ofstream(AbstractContextManager, ObjectProxy): and performs the atomic replacement via os.rename(). If the abort() method has been called, then the temp file is closed and removed.""" f = object.__getattribute__(self, "_file") +tmp_name = object.__getattribute__(self, "_tmp_name") real_name = object.__getattribute__(self, "_real_name") if not f.closed: try: f.close() if not object.__getattribute__(self, "_aborted"): try: -apply_stat_permissions(f.name, os.stat(real_name)) +apply_stat_permissions(tmp_name, os.stat(real_name)) except OperationNotPermitted: pass except FileNotFound: @@ -1514,12 +1523,12 @@ class atomic_ofstream(AbstractContextManager, ObjectProxy): pass else: raise -os.rename(f.name, real_name) +os.rename(tmp_name, real_name) finally: # Make sure we cleanup the temp file # even if an exception is raised. try: -os.unlink(f.name) +os.unlink(tmp_name) except OSError as oe: pass
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 2db89a16ab87b85004216959ec6bc508575d96a0
Author: Zac Medico gentoo org>
AuthorDate: Mon May 27 18:03:32 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon May 27 18:04:41 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=2db89a16
binarytree: Rewrite remote index only on change
I noticed that the remote index was rewritten with a
new DOWNLOAD_TIMESTAMP even while frozen, and this
patch fixed it.
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dbapi/bintree.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 64dfee4faa..b32dea1eae 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1400,6 +1400,7 @@ class binarytree:
except OSError as e:
if e.errno != errno.ENOENT:
raise
+changed = True
local_timestamp = pkgindex.header.get("TIMESTAMP", None)
try:
download_timestamp =
float(pkgindex.header.get("DOWNLOAD_TIMESTAMP", 0))
@@ -1574,6 +1575,7 @@ class binarytree:
noiselevel=-1,
)
except UseCachedCopyOfRemoteIndex:
+changed = False
desc = "frozen" if repo.frozen else "up-to-date"
writemsg_stdout("\n")
writemsg_stdout(
@@ -1611,7 +1613,7 @@ class binarytree:
os.unlink(tmp_filename)
except OSError:
pass
-if pkgindex is rmt_idx:
+if pkgindex is rmt_idx and changed:
pkgindex.modified = False # don't update the header
pkgindex.header["DOWNLOAD_TIMESTAMP"] = "%d" % time.time()
try:
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: f1843024c34b8915786d3044f6f1b3a3b72284bf Author: Florian Schmaus gentoo org> AuthorDate: Tue May 21 07:35:09 2024 + Commit: Sam James gentoo org> CommitDate: Sun May 26 23:28:12 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f1843024 Scheduler: coalesce common code in _schedule_tasks_imp() Signed-off-by: Florian Schmaus gentoo.org> Signed-off-by: Sam James gentoo.org> lib/_emerge/Scheduler.py | 15 +-- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index 5c318f89b9..77318b0b3e 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -1985,15 +1985,6 @@ class Scheduler(PollScheduler): self._task_queues.merge.addFront(merge) merge.addExitListener(self._merge_exit) -elif pkg.built: -self._jobs += 1 -self._previous_job_start_time = time.time() -self._status_display.running = self._jobs -self._running_tasks[id(task)] = task -task.scheduler = self._sched_iface -self._task_queues.jobs.add(task) -task.addExitListener(self._extract_exit) - else: self._jobs += 1 self._previous_job_start_time = time.time() @@ -2001,7 +1992,11 @@ class Scheduler(PollScheduler): self._running_tasks[id(task)] = task task.scheduler = self._sched_iface self._task_queues.jobs.add(task) -task.addExitListener(self._build_exit) + +if pkg.built: +task.addExitListener(self._extract_exit) +else: +task.addExitListener(self._build_exit) return bool(state_change)
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: aa0a0c6f74ae28eb209ee933946e82917612b21e Author: Florian Schmaus gentoo org> AuthorDate: Tue May 21 07:33:05 2024 + Commit: Sam James gentoo org> CommitDate: Sun May 26 23:28:13 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=aa0a0c6f Scheduler: convert state_change to boolean in _schedule_tasks_imp() Signed-off-by: Florian Schmaus gentoo.org> Closes: https://github.com/gentoo/portage/pull/1324 Signed-off-by: Sam James gentoo.org> lib/_emerge/Scheduler.py | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index c547c309c2..d913cd2dc6 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -1953,11 +1953,11 @@ class Scheduler(PollScheduler): @return: True if state changed, False otherwise. """ -state_change = 0 +state_change = False while True: if not self._keep_scheduling(): -return bool(state_change) +return state_change if ( self._choose_pkg_return_early @@ -1966,13 +1966,13 @@ class Scheduler(PollScheduler): or not self._can_add_job() or self._job_delay() ): -return bool(state_change) +return state_change pkg = self._choose_pkg() if pkg is None: -return bool(state_change) +return state_change -state_change += 1 +state_change = True if not pkg.installed: self._pkg_count.curval += 1
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 305d843be43c79d6b09584280a67e16060e85df0 Author: Florian Schmaus gentoo org> AuthorDate: Tue May 21 16:11:53 2024 + Commit: Sam James gentoo org> CommitDate: Sun May 26 23:28:13 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=305d843b Scheduler: drop unreachable return statement in _schedule_tasks_imp() Signed-off-by: Florian Schmaus gentoo.org> Signed-off-by: Sam James gentoo.org> lib/_emerge/Scheduler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index 77318b0b3e..c547c309c2 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -1998,8 +1998,6 @@ class Scheduler(PollScheduler): else: task.addExitListener(self._build_exit) -return bool(state_change) - def _get_prefetcher(self, pkg): try: prefetcher = self._prefetchers.pop(pkg, None)
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/rsync/
commit: a671334b7c7b64bc779f1c2bfb4ed659d97c0d19
Author: Pavel Balaev void so>
AuthorDate: Mon Mar 18 11:29:34 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun May 26 23:27:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a671334b
sync: don't use ipv6 for rsync when it's disabled
socket.has_ipv6 gives a false result:
$ sysctl net.ipv6.conf.all.disable_ipv6=1
net.ipv6.conf.all.disable_ipv6 = 1
$ python
Python 3.11.8 (main, Feb 24 2024, 17:10:38) [GCC 13.2.1 20240210] on linux
>>> import socket
>>> socket.has_ipv6
True
This patch uses the portage.process.has_ipv6() function,
which returns the correct result.
Bug: https://bugs.gentoo.org/927241
Signed-off-by: Pavel Balaev void.so>
Closes: https://github.com/gentoo/portage/pull/1309
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/rsync/rsync.py | 7 +++
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/lib/portage/sync/modules/rsync/rsync.py
b/lib/portage/sync/modules/rsync/rsync.py
index 15c3eb4da5..e89221ebc9 100644
--- a/lib/portage/sync/modules/rsync/rsync.py
+++ b/lib/portage/sync/modules/rsync/rsync.py
@@ -19,6 +19,7 @@ from portage import _unicode_decode
from portage import os
from portage.const import VCS_DIRS, TIMESTAMP_FORMAT, RSYNC_PACKAGE_ATOM
from portage.output import create_color_func, yellow, blue, bold
+from portage.process import has_ipv6
from portage.sync.getaddrinfo_validate import getaddrinfo_validate
from portage.sync.syncbase import NewBase
from portage.util import writemsg, writemsg_level, writemsg_stdout
@@ -252,9 +253,7 @@ class RsyncSync(NewBase):
family = socket.AF_UNSPEC
if "-4" in all_rsync_opts or "--ipv4" in all_rsync_opts:
family = socket.AF_INET
-elif socket.has_ipv6 and (
-"-6" in all_rsync_opts or "--ipv6" in all_rsync_opts
-):
+elif has_ipv6() and ("-6" in all_rsync_opts or "--ipv6" in
all_rsync_opts):
family = socket.AF_INET6
addrinfos = None
@@ -278,7 +277,7 @@ class RsyncSync(NewBase):
if addrinfos:
AF_INET = socket.AF_INET
AF_INET6 = None
-if socket.has_ipv6:
+if has_ipv6():
AF_INET6 = socket.AF_INET6
ips_v4 = []
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/, lib/portage/
commit: d00ecdce359c8bc3d3520aef58d5808c7d15020b
Author: Pavel Balaev void so>
AuthorDate: Wed Mar 20 14:35:54 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun May 26 23:27:07 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d00ecdce
process: make has_ipv6 a public function
Signed-off-by: Pavel Balaev void.so>
Signed-off-by: Sam James gentoo.org>
lib/portage/process.py| 8
lib/portage/tests/process/test_unshare_net.py | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index 1bc0c507c2..6e4e0d7162 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -631,8 +631,8 @@ def spawn(
fd_pipes[1] = pw
fd_pipes[2] = pw
-# Cache _has_ipv6() result for use in child processes.
-_has_ipv6()
+# Cache has_ipv6() result for use in child processes.
+has_ipv6()
# This caches the libc library lookup and _unshare_validator results
# in the current process, so that results are cached for use in
@@ -751,7 +751,7 @@ def spawn(
__has_ipv6 = None
-def _has_ipv6():
+def has_ipv6():
"""
Test that both userland and kernel support IPv6, by attempting
to create a socket and listen on any unused port of the IPv6
@@ -812,7 +812,7 @@ def _configure_loopback_interface():
ifindex = rtnl.get_link_ifindex(b"lo")
rtnl.set_link_up(ifindex)
rtnl.add_address(ifindex, socket.AF_INET, "10.0.0.1", 8)
-if _has_ipv6():
+if has_ipv6():
rtnl.add_address(ifindex, socket.AF_INET6, "fd::1", 8)
except OSError as e:
writemsg(
diff --git a/lib/portage/tests/process/test_unshare_net.py
b/lib/portage/tests/process/test_unshare_net.py
index ad3b288ef4..56d96da05f 100644
--- a/lib/portage/tests/process/test_unshare_net.py
+++ b/lib/portage/tests/process/test_unshare_net.py
@@ -42,7 +42,7 @@ class UnshareNetTestCase(TestCase):
f"Unable to unshare:
{errno.errorcode.get(self.ABILITY_TO_UNSHARE, '?')}"
)
env = os.environ.copy()
-env["IPV6"] = "1" if portage.process._has_ipv6() else ""
+env["IPV6"] = "1" if portage.process.has_ipv6() else ""
self.assertEqual(
portage.process.spawn(
[BASH_BINARY, "-c", UNSHARE_NET_TEST_SCRIPT],
unshare_net=True, env=env
[gentoo-commits] proj/portage:master commit in: man/
commit: f3ae6bb0bd495b6b16d7e617515f4504ae2e1839 Author: Waldo Lemmer gmail com> AuthorDate: Sun May 26 14:30:32 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun May 26 19:09:06 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f3ae6bb0 emerge.1: Fix /var/log/emerge.log description /var/log/emerge.log doesn't "[contain] a log of all emerge output". This commit changes this description to better reflect what emerge.log contains: - Packages that were built - emerge's invocation (i.e. its command-line arguments) Signed-off-by: Waldo Lemmer gmail.com> Closes: https://github.com/gentoo/portage/pull/1331 Signed-off-by: Zac Medico gentoo.org> man/emerge.1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/man/emerge.1 b/man/emerge.1 index dd9cde5722..5c36dc1f0d 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -1,4 +1,4 @@ -.TH "EMERGE" "1" "Jan 2024" "Portage @VERSION@" "Portage" +.TH "EMERGE" "1" "May 2024" "Portage @VERSION@" "Portage" .SH "NAME" emerge \- Command\-line interface to the Portage system .SH "SYNOPSIS" @@ -1520,8 +1520,8 @@ Contains the default variables for the build process. \fBDo not edit this file\fR. .TP .B /var/log/emerge.log -Contains a log of all emerge output. This file is always appended to, so if you -want to clean it, you need to do so manually. +Contains a log of merged packages and invocations of \fBemerge\fR. This file +is always appended to, so if you want to clean it, you need to do so manually. .TP .B /var/log/emerge-fetch.log Contains a log of all the fetches in the previous emerge invocation.
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/resolver/
commit: ff22b5bf954574f1fad789fc148945d67c3a1215 Author: Zac Medico gentoo org> AuthorDate: Sun May 26 18:57:38 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun May 26 18:57:58 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ff22b5bf test_tar_merge_order.py: Fix unused pytest unused-import Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/resolver/test_tar_merge_order.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/portage/tests/resolver/test_tar_merge_order.py b/lib/portage/tests/resolver/test_tar_merge_order.py index 4bd9b4df4a..c66d01ee31 100644 --- a/lib/portage/tests/resolver/test_tar_merge_order.py +++ b/lib/portage/tests/resolver/test_tar_merge_order.py @@ -2,7 +2,6 @@ # Distributed under the terms of the GNU General Public License v2 import os -import pytest from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import (
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/resolver/
commit: 11bded10a2a4f143e8d7f8ccb2f91f2b6fed59b5
Author: Sam James gentoo org>
AuthorDate: Sun May 26 14:44:56 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun May 26 18:27:02 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=11bded10
tests: add testcase for app-arch/tar merge order with binpkgs
In the bug, dilfridge reports releng@ is hitting the following:
```
$ emerge -epvk world | grep -E "(app-arch/tar|sys-apps/acl)"
[ebuild N ] sys-apps/acl-2.3.2-r1::gentoo USE="nls -static-libs" 363 KiB
[binary R] app-arch/tar-1.35-1::gentoo USE="acl* nls* xattr* -minimal
(-selinux) -verify-sig" 0 KiB
[...]
```
Test for bug #922629 where binary app-arch/tar[acl] was merged
before its dependency sys-apps/acl (with virtual/acl merged but
unsatisfied).
It (appears to be) a bad interaction with @system containing
app-alternatives/tar
plus a circular dependency on app-arch/tar. The USE change is also important,
as e.g. dropping "sys-apps/attr nls" from package.use makes things okay.
XFAIL'd for now.
Bug: https://bugs.gentoo.org/922629
Signed-off-by: Sam James gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1332
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/resolver/meson.build | 1 +
lib/portage/tests/resolver/test_tar_merge_order.py | 495 +
2 files changed, 496 insertions(+)
diff --git a/lib/portage/tests/resolver/meson.build
b/lib/portage/tests/resolver/meson.build
index 8892c78131..ea948982e7 100644
--- a/lib/portage/tests/resolver/meson.build
+++ b/lib/portage/tests/resolver/meson.build
@@ -81,6 +81,7 @@ py.install_sources(
'test_slot_operator_update_probe_parent_downgrade.py',
'test_solve_non_slot_operator_slot_conflicts.py',
'test_targetroot.py',
+'test_tar_merge_order.py',
'test_unmerge_order.py',
'test_unnecessary_slot_upgrade.py',
'test_update.py',
diff --git a/lib/portage/tests/resolver/test_tar_merge_order.py
b/lib/portage/tests/resolver/test_tar_merge_order.py
new file mode 100644
index 00..7e1a18bc21
--- /dev/null
+++ b/lib/portage/tests/resolver/test_tar_merge_order.py
@@ -0,0 +1,495 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import pytest
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+ResolverPlayground,
+ResolverPlaygroundTestCase,
+)
+
+
+class TarMergeOrderTestCase(TestCase):
+@pytest.mark.xfail(reason="bug #922629 isn't yet fixed")
+def testTarMergeOrder(self):
+"""
+Test for bug #922629 where binary app-arch/tar[acl] was merged
+before its dependency sys-apps/acl (with virtual/acl merged but
+unsatisfied).
+
+It poorly interacted with @system containing app-alternatives/tar
+as a circular dependency on app-arch/tar.
+"""
+
+ebuilds = {
+"app-alternatives/tar-0-1": {
+"EAPI": "8",
+"RDEPEND": """
+!=app-arch/tar-1.34-r2 )
+libarchive? ( app-arch/libarchive )
+""",
+"IUSE": "+gnu libarchive",
+"REQUIRED_USE": "^^ ( gnu libarchive )",
+},
+"app-arch/libarchive-3.7.4": {"EAPI": "8"},
+"app-arch/tar-1.35": {
+"EAPI": "8",
+"RDEPEND": """
+acl? ( virtual/acl )
+""",
+"DEPEND": """
+acl? ( virtual/acl )
+xattr? ( sys-apps/attr )
+""",
+"BDEPEND": """
+nls? ( sys-devel/gettext )
+""",
+"IUSE": "acl nls xattr",
+},
+"virtual/acl-0-r2": {
+"EAPI": "8",
+"RDEPEND": ">=sys-apps/acl-2.2.52-r1",
+},
+"sys-devel/gettext-0.22.4": {
+"EAPI": "8",
+"RDEPEND": """
+acl? ( virtual/acl )
+xattr? ( sys-apps/attr )
+""",
+"DEPEND": """
+acl? ( virtual/acl )
+xattr? ( sys-apps/attr )
+""",
+"IUSE": "acl nls xattr",
+},
+"sys-apps/attr-2.5.2-r1": {
+"EAPI": "8",
+"BDEPEND": "nls? ( sys-devel/gettext )",
+"IUSE": "nls",
+},
+"sys-apps/acl-2.3.2-r1": {
+"EAPI": "8",
+"DEPEND": ">=sys-apps/attr-2.4.47-r1",
+"RDEPEND": ">=sys-apps/attr-2.4.47-r1",
+"BDEPEND": "nls? ( sys-devel/gettext )",
+"IUSE": "nls",
+},
+}
+
+installed = {
+"app-alternatives/tar-0-1": {
+"EAPI": "8",
+
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/tests/resolver/
commit: 96e4f95cc8c0d544d375b28394dafe8809c4bc9b Author: Zac Medico gentoo org> AuthorDate: Sun May 26 18:23:27 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun May 26 18:34:04 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=96e4f95c Revert "find_smallest_cycle: Optimize to traverse fewer nodes" This reverts commit 49e01d041c74680a81860b819daff812d83df02f in order to fix bug 922629. Later we can try to optimize it again but without breaking testTarMergeOrder. Bug: https://bugs.gentoo.org/922629 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/depgraph.py| 36 ++ .../tests/resolver/test_rebuild_ghostscript.py | 2 +- .../resolver/test_runtime_cycle_merge_order.py | 7 ++--- lib/portage/tests/resolver/test_tar_merge_order.py | 4 ++- 4 files changed, 9 insertions(+), 40 deletions(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 6b91d5c42d..3adc04bcfb 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -9313,14 +9313,7 @@ class depgraph: asap_nodes.extend(libc_pkgs) -def gather_deps( -ignore_priority, -mergeable_nodes, -selected_nodes, -node, -smallest_cycle=None, -traversed_nodes=None, -): +def gather_deps(ignore_priority, mergeable_nodes, selected_nodes, node): """ Recursively gather a group of nodes that RDEPEND on eachother. This ensures that they are merged as a group @@ -9340,24 +9333,10 @@ class depgraph: # RDEPENDs installed first, but ignore uninstalls # (these occur when new portage blocks an older package version). return False -if traversed_nodes is not None: -if node in traversed_nodes: -# Identical to a previously traversed cycle. -return False -traversed_nodes.add(node) selected_nodes.add(node) -if smallest_cycle is not None and len(selected_nodes) >= len( -smallest_cycle -): -return False for child in mygraph.child_nodes(node, ignore_priority=ignore_priority): if not gather_deps( -ignore_priority, -mergeable_nodes, -selected_nodes, -child, -smallest_cycle=smallest_cycle, -traversed_nodes=traversed_nodes, +ignore_priority, mergeable_nodes, selected_nodes, child ): return False return True @@ -9515,21 +9494,12 @@ class depgraph: local_priority_range.MEDIUM_SOFT + 1, ) ): -# Traversed nodes for current priority -traversed_nodes = set() for node in nodes: if not mygraph.parent_nodes(node): continue -if node in traversed_nodes: -continue selected_nodes = set() if gather_deps( -priority, -mergeable_nodes, -selected_nodes, -node, -smallest_cycle=smallest_cycle, -traversed_nodes=traversed_nodes, +priority, mergeable_nodes, selected_nodes, node ): if smallest_cycle is None or len(selected_nodes) < len( smallest_cycle diff --git a/lib/portage/tests/resolver/test_rebuild_ghostscript.py b/lib/portage/tests/resolver/test_rebuild_ghostscript.py index 8ee3349d6f..dad6a21322 100644 --- a/lib/portage/tests/resolver/test_rebuild_ghostscript.py +++ b/lib/portage/tests/resolver/test_rebuild_ghostscript.py @@ -137,9 +137,9 @@ class RebuildGhostscriptTestCase(TestCase): mergelist=[ "sys-apps/dbus-1.15.6", "x11-libs/gtk+-3.24.38", -"app-text/ghostscript-gpl-10.01.2", "net-dns/avahi-0.8-r7", "net-print/cups-2.4.6", +"app-text/ghostscript-gpl-10.01.2", "app-text/libspectre-0.2.12", "x11-libs/goffice-0.10.55", ], diff --git a/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py b/lib/portage/tests/resolver/test_runtime_cycle_merge_order.py index ed329aa097..a695b25198 100644 ---
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 685908978c48f6bfeab99f47ba7d78ee00294a64 Author: Tatsuyuki Ishi google com> AuthorDate: Wed May 15 07:46:36 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun May 26 18:01:22 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=68590897 Fix a depgraph crash with useoldpkg and subslot bumps This resolves a "list index out of range" error that could happen with when a package's subslot is bumped and --useoldpkg is used. Due to the subslot bump, the code will filter matched_packages to only contain the latest version. However, this didn't update matched_oldpkg, which would now contain stale packages and cause logic errors. Fix this by filtering matched_oldpkg again before its use. Signed-off-by: Tatsuyuki Ishi google.com> Bug: https://bugs.gentoo.org/932804 Closes: https://github.com/gentoo/portage/pull/1322 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/depgraph.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 13add990e6..6b91d5c42d 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -8177,6 +8177,7 @@ class depgraph: ): return pkg, existing_node +matched_oldpkg = [x for x in matched_oldpkg if x in matched_packages] visible_matches = [] if matched_oldpkg: visible_matches = [
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/sync/, lib/portage/dbapi/
commit: 5aed7289d516fab5b63557da46348125eabab368
Author: Zac Medico gentoo org>
AuthorDate: Thu Mar 14 04:09:34 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat May 25 22:08:15 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5aed7289
bintree: Add REPO_REVISIONS to package index header
As a means for binhost clients to select source repo
revisions which are consistent with binhosts, inject
REPO_REVISIONS from a package into the index header,
using a history of synced revisions to guarantee
forward progress. This queries the relevant repos to
check if any new revisions have appeared in the
absence of a proper sync operation.
Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dbapi/bintree.py | 67 -
lib/portage/tests/sync/test_sync_local.py | 71 +--
2 files changed, 124 insertions(+), 14 deletions(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 221afbd154..64dfee4faa 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -48,6 +48,7 @@ from portage.exception import (
from portage.localization import _
from portage.output import colorize
from portage.package.ebuild.profile_iuse import iter_iuse_vars
+from portage.sync.revision_history import get_repo_revision_history
from portage.util import ensure_dirs
from portage.util.file_copy import copyfile
from portage.util.futures import asyncio
@@ -62,6 +63,7 @@ from portage import _unicode_encode
import codecs
import errno
import io
+import json
import re
import shlex
import stat
@@ -135,13 +137,19 @@ class bindbapi(fakedbapi):
"USE",
"_mtime_",
}
+# Keys required only when initially adding a package.
+self._init_aux_keys = {
+"REPO_REVISIONS",
+}
self._aux_cache = {}
self._aux_cache_slot_dict_cache = None
@property
def _aux_cache_slot_dict(self):
if self._aux_cache_slot_dict_cache is None:
-self._aux_cache_slot_dict_cache =
slot_dict_class(self._aux_cache_keys)
+self._aux_cache_slot_dict_cache = slot_dict_class(
+chain(self._aux_cache_keys, self._init_aux_keys)
+)
return self._aux_cache_slot_dict_cache
def __getstate__(self):
@@ -1791,6 +1799,11 @@ class binarytree:
pkgindex = self._new_pkgindex()
d = self._inject_file(pkgindex, cpv, full_path)
+repo_revisions = (
+json.loads(d["REPO_REVISIONS"]) if d.get("REPO_REVISIONS")
else None
+)
+if repo_revisions:
+self._inject_repo_revisions(pkgindex.header, repo_revisions)
self._update_pkgindex_header(pkgindex.header)
self._pkgindex_write(pkgindex)
@@ -1872,7 +1885,7 @@ class binarytree:
@return: package metadata
"""
if keys is None:
-keys = self.dbapi._aux_cache_keys
+keys = chain(self.dbapi._aux_cache_keys, self.dbapi._init_aux_keys)
metadata = self.dbapi._aux_cache_slot_dict()
else:
metadata = {}
@@ -1916,6 +1929,56 @@ class binarytree:
return metadata
+def _inject_repo_revisions(self, header, repo_revisions):
+"""
+Inject REPO_REVISIONS from a package into the index header,
+using a history of synced revisions to guarantee forward
+progress. This queries the relevant repos to check if any
+new revisions have appeared in the absence of a proper sync
+operation.
+
+This does not expose REPO_REVISIONS that do not appear in
+the sync history, since such revisions suggest that the
+package was not built locally, and in this case its
+REPO_REVISIONS are not intended to be exposed.
+"""
+synced_repo_revisions = get_repo_revision_history(
+self.settings["EROOT"],
+[self.settings.repositories[repo_name] for repo_name in
repo_revisions],
+)
+header_repo_revisions = (
+json.loads(header["REPO_REVISIONS"]) if
header.get("REPO_REVISIONS") else {}
+)
+for repo_name, repo_revision in repo_revisions.items():
+rev_list = synced_repo_revisions.get(repo_name, [])
+header_rev = header_repo_revisions.get(repo_name)
+if not rev_list or header_rev in (repo_revision, rev_list[0]):
+continue
+try:
+header_rev_index = (
+None if header_rev is None else rev_list.index(header_rev)
+)
+except ValueError:
+header_rev_index = None
+try:
+repo_revision_index = rev_list.index(repo_revision)
+except ValueError:
+repo_revision_index = None
+if
[gentoo-commits] proj/portage:master commit in: bin/, lib/portage/tests/sync/, lib/portage/, lib/portage/emaint/modules/, man/, ...
commit: 71d9ce40be5bbf533a6d1b59c5a460621c3c91c4
Author: Zac Medico gentoo org>
AuthorDate: Thu Mar 14 04:09:21 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat May 25 22:08:15 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=71d9ce40
Add get_repo_revision_history function and repo_revisions file
The history of synced revisions is provided by a new
get_repo_revision_history function and corresponding
/var/lib/portage/repo_revisions file, with history
limit currently capped at 25 revisions. If a change
is detected and the current process has permission
to update the repo_revisions file, then the file will
be updated with any newly detected revisions.
For volatile repos the revisions may be unordered,
which makes them unusable for the purposes of the
revision history, so the revisions of volatile repos
are not tracked. This functions detects revisions
which are not yet visible to the current process due
to the sync-rcu option.
The emaint revisions --purgerepos and --purgeallrepos
options allow revisions for some or all repos to be
easily purged from the history. For example, the
emerge-webrsync script uses this emaint commmand to
purge the revision history of the gentoo repo when
the emerge-webrsync --revert option is used to roll
back to a previous snapshot:
emaint revisions --purgerepos="${repo_name}"
Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico gentoo.org>
bin/emerge-webrsync | 3 +-
lib/portage/const.py | 1 +
lib/portage/emaint/modules/meson.build| 1 +
lib/portage/emaint/modules/revisions/__init__.py | 36 ++
lib/portage/emaint/modules/revisions/meson.build | 8 ++
lib/portage/emaint/modules/revisions/revisions.py | 95
lib/portage/sync/controller.py| 8 +-
lib/portage/sync/meson.build | 1 +
lib/portage/sync/revision_history.py | 133 ++
lib/portage/tests/sync/test_sync_local.py | 75 +++-
man/emaint.1 | 18 ++-
man/portage.5 | 15 +++
12 files changed, 387 insertions(+), 7 deletions(-)
diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync
index 99da05543a..caa4986da2 100755
--- a/bin/emerge-webrsync
+++ b/bin/emerge-webrsync
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
# Author: Karl Trygve Kalleberg
# Rewritten from the old, Perl-based emerge-webrsync script
@@ -732,6 +732,7 @@ main() {
[[ ${do_debug} -eq 1 ]] && set -x
if [[ -n ${revert_date} ]] ; then
+ emaint revisions --purgerepos="${repo_name}"
do_snapshot 1 "${revert_date}"
else
do_latest_snapshot
diff --git a/lib/portage/const.py b/lib/portage/const.py
index 2154213b7b..c9a71009a7 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -51,6 +51,7 @@ PRIVATE_PATH = "var/lib/portage"
WORLD_FILE = f"{PRIVATE_PATH}/world"
WORLD_SETS_FILE = f"{PRIVATE_PATH}/world_sets"
CONFIG_MEMORY_FILE = f"{PRIVATE_PATH}/config"
+REPO_REVISIONS = f"{PRIVATE_PATH}/repo_revisions"
NEWS_LIB_PATH = "var/lib/gentoo"
# these variables get EPREFIX prepended automagically when they are
diff --git a/lib/portage/emaint/modules/meson.build
b/lib/portage/emaint/modules/meson.build
index 48f4f77d83..33b396be94 100644
--- a/lib/portage/emaint/modules/meson.build
+++ b/lib/portage/emaint/modules/meson.build
@@ -12,5 +12,6 @@ subdir('logs')
subdir('merges')
subdir('move')
subdir('resume')
+subdir('revisions')
subdir('sync')
subdir('world')
diff --git a/lib/portage/emaint/modules/revisions/__init__.py
b/lib/portage/emaint/modules/revisions/__init__.py
new file mode 100644
index 00..c51cbb4bf3
--- /dev/null
+++ b/lib/portage/emaint/modules/revisions/__init__.py
@@ -0,0 +1,36 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+doc = """Purge repo_revisions history file."""
+__doc__ = doc
+
+
+module_spec = {
+"name": "revisions",
+"description": doc,
+"provides": {
+"purgerevisions": {
+"name": "revisions",
+"sourcefile": "revisions",
+"class": "PurgeRevisions",
+"description": "Purge repo_revisions history",
+"functions": ["purgeallrepos", "purgerepos"],
+"func_desc": {
+"repo": {
+"long": "--purgerepos",
+"help": "(revisions module only): --purgerepos Purge
revisions for the specified repo(s)",
+"status": "Purging %s",
+"action": "store",
+"func": "purgerepos",
+},
+"allrepos": {
+
[gentoo-commits] proj/portage:master commit in: man/, lib/portage/binrepo/, lib/portage/dbapi/
commit: cd41fb9390fcca9dc8a565b5c52dbba73d9ae59c
Author: Zac Medico gentoo org>
AuthorDate: Sat May 25 21:53:52 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat May 25 21:53:52 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cd41fb93
binrepos.conf: Support "frozen" attribute
In order to allow consistent and reproducible dependency calculations
during mixed source and binary updates, add a "frozen" binrepos.conf
attribute which will freeze binrepo index updates and cause messsages
to indicate that the repo is frozen rather than up-to-date:
Local copy of remote index is frozen and will be used.
This should only be set temporarily in order to guarantee consistent
and reproducible dependency calculations for mixed binary and source
updates.
Bug: https://bugs.gentoo.org/932739
Signed-off-by: Zac Medico gentoo.org>
lib/portage/binrepo/config.py | 10 +-
lib/portage/dbapi/bintree.py | 5 +++--
man/portage.5 | 8 +++-
3 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py
index 5601a2e002..c744d1012f 100644
--- a/lib/portage/binrepo/config.py
+++ b/lib/portage/binrepo/config.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Gentoo Authors
+# Copyright 2020-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from collections import OrderedDict
@@ -12,6 +12,7 @@ from portage.util.configparser import SafeConfigParser,
ConfigParserError, read_
class BinRepoConfig:
__slots__ = (
+"frozen",
"name",
"name_fallback",
"fetchcommand",
@@ -19,6 +20,7 @@ class BinRepoConfig:
"resumecommand",
"sync_uri",
)
+_bool_opts = ("frozen",)
def __init__(self, opts):
"""
@@ -26,6 +28,9 @@ class BinRepoConfig:
"""
for k in self.__slots__:
setattr(self, k, opts.get(k.replace("_", "-")))
+for k in self._bool_opts:
+if isinstance(getattr(self, k, None), str):
+setattr(self, k, getattr(self, k).lower() in ("true", "yes"))
def info_string(self):
"""
@@ -38,6 +43,8 @@ class BinRepoConfig:
if self.priority is not None:
repo_msg.append(indent + "priority: " + str(self.priority))
repo_msg.append(indent + "sync-uri: " + self.sync_uri)
+if self.frozen:
+repo_msg.append(f"{indent}frozen: {str(self.frozen).lower()}")
repo_msg.append("")
return "\n".join(repo_msg)
@@ -48,6 +55,7 @@ class BinRepoConfigLoader(Mapping):
# Defaults for value interpolation.
parser_defaults = {
+"frozen": "false",
"EPREFIX": settings["EPREFIX"],
"EROOT": settings["EROOT"],
"PORTAGE_CONFIGROOT": settings["PORTAGE_CONFIGROOT"],
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 22e2995c2f..221afbd154 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1408,7 +1408,7 @@ class binarytree:
url = base_url.rstrip("/") + "/Packages"
f = None
-if not getbinpkg_refresh and local_timestamp:
+if local_timestamp and (repo.frozen or not getbinpkg_refresh):
raise UseCachedCopyOfRemoteIndex()
try:
@@ -1566,11 +1566,12 @@ class binarytree:
noiselevel=-1,
)
except UseCachedCopyOfRemoteIndex:
+desc = "frozen" if repo.frozen else "up-to-date"
writemsg_stdout("\n")
writemsg_stdout(
colorize(
"GOOD",
-_("Local copy of remote index is up-to-date and will
be used."),
+_("Local copy of remote index is %s and will be
used.") % desc,
)
+ "\n"
)
diff --git a/man/portage.5 b/man/portage.5
index 1f717c4cb8..3b8329bfb2 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1,4 +1,4 @@
-.TH "PORTAGE" "5" "Apr 2023" "Portage @VERSION@" "Portage"
+.TH "PORTAGE" "5" "May 2024" "Portage @VERSION@" "Portage"
.SH NAME
portage \- the heart of Gentoo
.SH "DESCRIPTION"
@@ -642,6 +642,12 @@ is intended to be used as a replacement for the
\fBmake.conf\fR(5)
.I Attributes supported in DEFAULT section:
.RS
.TP
+.B frozen = yes|no|true|false
+Use the most recently cached copy of the remote index, and do not
+attempt to refresh it. This should only be set temporarily in order to
+guarantee consistent and reproducible dependency calculations (for
+mixed binary and source updates).
+.TP
.B fetchcommand
Specifies a \fBFETCHCOMMAND\fR used to fetch files from a repository,
overriding the value from \fBmake.conf\fR(5).
[gentoo-commits] proj/portage:master commit in: bin/install-qa-check.d/
commit: 9728fbba80abeec5adad8ba6cf7580c16c039a45
Author: Eli Schwartz gmail com>
AuthorDate: Tue May 21 23:22:31 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed May 22 16:56:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=9728fbba
install-qa-checks.d: suppress some gnulib implicit decls on musl
These happen in tons of GNU packages because of using gnulib's regex.m4
specifically, which pulls in a macro that checks for some functionality
and spit out many implicit function declaration errors if regex.h isn't
GNU's specifically.
The compile tests do fail either way, it's just very dirty in the logs.
Bug: https://bugs.gentoo.org/906027
Signed-off-by: Eli Schwartz gmail.com>
Closes: https://github.com/gentoo/portage/pull/1327
Signed-off-by: Sam James gentoo.org>
bin/install-qa-check.d/90config-impl-decl | 12
1 file changed, 12 insertions(+)
diff --git a/bin/install-qa-check.d/90config-impl-decl
b/bin/install-qa-check.d/90config-impl-decl
index adea7d4051..8768c99c62 100644
--- a/bin/install-qa-check.d/90config-impl-decl
+++ b/bin/install-qa-check.d/90config-impl-decl
@@ -58,6 +58,18 @@ add_default_skips() {
# also gnulib, but checks both linux/non-linux headers
MIN
)
+ if [[ ${CHOST} = *musl* ]]; then
+ QA_CONFIG_IMPL_DECL_SKIP+=(
+ # gnulib checks for functions that aren't available on
musl.
+
+ # regex.m4 always emits these warnings, but they are
noisy to fix
+ # and the check will correctly fail due to missing
macros anyway.
+ re_set_syntax
+ re_compile_pattern
+ re_search
+ re_match
+ )
+ fi
}
find_log_targets() {
[gentoo-commits] proj/portage:master commit in: bin/
commit: 7fa294b66c78b9a9000d161f3b4bacbad158489b
Author: Mike Gilbert gentoo org>
AuthorDate: Wed May 22 15:35:07 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Wed May 22 15:35:07 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7fa294b6
unpack: skip unrecognized file formats *silently*
See PMS section 12.3.15.
Signed-off-by: Mike Gilbert gentoo.org>
bin/phase-helpers.sh | 13 -
1 file changed, 13 deletions(-)
diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh
index 45a1639c42..77132eb066 100644
--- a/bin/phase-helpers.sh
+++ b/bin/phase-helpers.sh
@@ -446,8 +446,6 @@ unpack() {
echo "${my_output}" >&2
die "${myfail}"
fi
- else
- __vecho "unpack ${x}: file format not
recognized. Ignoring."
fi
;;
rar)
@@ -459,8 +457,6 @@ unpack() {
fi
if ___eapi_unpack_supports_rar; then
unrar x -idq -o+ "${srcdir}${x}" || die
"${myfail}"
- else
- __vecho "unpack ${x}: file format not
recognized. Ignoring."
fi
;;
lha|lzh)
@@ -473,8 +469,6 @@ unpack() {
fi
if ___eapi_unpack_supports_lha; then
lha xfq "${srcdir}${x}" || die
"${myfail}"
- else
- __vecho "unpack ${x}: file format not
recognized. Ignoring."
fi
;;
a)
@@ -545,8 +539,6 @@ unpack() {
fi
if ___eapi_unpack_supports_xz; then
__unpack_tar "xz -T$(___makeopts_jobs)
-d"
- else
- __vecho "unpack ${x}: file format not
recognized. Ignoring."
fi
;;
txz)
@@ -558,13 +550,8 @@ unpack() {
fi
if ___eapi_unpack_supports_txz; then
XZ_OPT="-T$(___makeopts_jobs)" tar xof
"${srcdir}${x}" || die "${myfail}"
- else
- __vecho "unpack ${x}: file format not
recognized. Ignoring."
fi
;;
- *)
- __vecho "unpack ${x}: file format not
recognized. Ignoring."
- ;;
esac
done
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/repository/, lib/portage/sync/modules/mercurial/, ...
commit: cdf328f5629c20ecf792a6200c8c2c24d932ee68
Author: Mike Gilbert gentoo org>
AuthorDate: Mon May 13 19:21:47 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue May 21 17:27:30 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cdf328f5
Drop portage.util.shlex_split
This has been a trivial wrapper since we dropped support for python2.
Signed-off-by: Mike Gilbert gentoo.org>
bin/dispatch-conf | 5 +++--
bin/ebuild | 5 ++---
bin/egencache | 5 ++---
bin/quickpkg| 9 +
lib/_emerge/BinpkgExtractorAsync.py | 14 +-
lib/_emerge/BinpkgFetcher.py| 4 ++--
lib/_emerge/actions.py | 3 ++-
lib/_emerge/main.py | 3 ++-
lib/portage/_emirrordist/FetchTask.py | 3 ++-
lib/portage/_sets/dbapi.py | 6 +++---
lib/portage/_sets/libs.py | 4 ++--
lib/portage/dbapi/bintree.py| 5 ++---
lib/portage/dbapi/porttree.py | 5 +++--
lib/portage/dbapi/vartree.py| 17 ++---
lib/portage/dispatch_conf.py| 5 +++--
lib/portage/emaint/modules/logs/logs.py | 5 +++--
lib/portage/emaint/modules/sync/sync.py | 3 ++-
lib/portage/getbinpkg.py| 3 ++-
lib/portage/gpg.py | 7 ---
lib/portage/gpkg.py | 11 ++-
lib/portage/package/ebuild/_config/LocationsManager.py | 4 ++--
.../package/ebuild/_config/env_var_validation.py| 4 ++--
.../package/ebuild/_parallel_manifest/ManifestTask.py | 5 +++--
lib/portage/package/ebuild/config.py| 6 +++---
lib/portage/package/ebuild/doebuild.py | 9 -
lib/portage/package/ebuild/fetch.py | 6 +++---
lib/portage/repository/config.py| 4 ++--
lib/portage/sync/modules/git/git.py | 11 ++-
lib/portage/sync/modules/mercurial/mercurial.py | 15 ---
lib/portage/sync/modules/rsync/rsync.py | 9 +++--
lib/portage/tests/ebuild/test_fetch.py | 5 +++--
lib/portage/tests/emerge/conftest.py| 7 ---
lib/portage/tests/emerge/test_config_protect.py | 5 +++--
lib/portage/util/ExtractKernelVersion.py| 5 +++--
lib/portage/util/__init__.py| 11 +--
lib/portage/util/_async/BuildLogger.py | 4 ++--
lib/portage/util/_dyn_libs/soname_deps.py | 6 +++---
37 files changed, 116 insertions(+), 122 deletions(-)
diff --git a/bin/dispatch-conf b/bin/dispatch-conf
index e34e9587f7..93164d909e 100755
--- a/bin/dispatch-conf
+++ b/bin/dispatch-conf
@@ -14,6 +14,7 @@
import atexit
import errno
import re
+import shlex
import subprocess
import sys
import termios
@@ -89,7 +90,7 @@ def cmd_var_is_valid(cmd):
Return true if the first whitespace-separated token contained
in cmd is an executable file, false otherwise.
"""
-cmd = portage.util.shlex_split(cmd)
+cmd = shlex.split(cmd)
if not cmd:
return False
@@ -130,7 +131,7 @@ class dispatch:
if pager is None or not cmd_var_is_valid(pager):
pager = "cat"
-pager_basename = os.path.basename(portage.util.shlex_split(pager)[0])
+pager_basename = os.path.basename(shlex.split(pager)[0])
if pager_basename == "less":
less_opts = self.options.get("less-opts")
if less_opts is not None and less_opts.strip():
diff --git a/bin/ebuild b/bin/ebuild
index 043e5bc476..113a6214d6 100755
--- a/bin/ebuild
+++ b/bin/ebuild
@@ -34,6 +34,7 @@ signal.signal(signal.SIGUSR1, debug_signal)
import argparse
from os import path as osp
+import shlex
import sys
import textwrap
@@ -107,9 +108,7 @@ def main():
parser.error("missing required args")
if not opts.ignore_default_opts:
-default_opts = portage.util.shlex_split(
-portage.settings.get("EBUILD_DEFAULT_OPTS", "")
-)
+default_opts = shlex.split(portage.settings.get("EBUILD_DEFAULT_OPTS",
""))
opts, pargs = parser.parse_known_args(default_opts + sys.argv[1:])
debug = opts.debug
diff --git a/bin/egencache b/bin/egencache
index dbe8d27fee..36477e1abf 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -33,6 +33,7 @@ try:
signal.signal(signal.SIGUSR1, debug_signal)
import argparse
+import shlex
import stat
import sys
import
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/_emerge/, lib/portage/, bin/, ...
commit: a33065dab4bebd476d0dcba3c1659fbe0e515469
Author: Mike Gilbert gentoo org>
AuthorDate: Wed May 15 17:25:18 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Tue May 21 17:24:00 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a33065da
Do not use shlex.split for CONFIG_PROTECT
PMS says this is a whitespace-separated list, so we should not treat it
as a shell expression.
Signed-off-by: Mike Gilbert gentoo.org>
bin/dispatch-conf | 4 ++--
bin/portageq| 8
lib/_emerge/depgraph.py | 6 +++---
lib/_emerge/post_emerge.py | 2 +-
lib/portage/_global_updates.py | 6 +++---
lib/portage/dbapi/vartree.py| 8
lib/portage/emaint/modules/sync/sync.py | 4 +---
7 files changed, 18 insertions(+), 20 deletions(-)
diff --git a/bin/dispatch-conf b/bin/dispatch-conf
index 601110ce87..e34e9587f7 100755
--- a/bin/dispatch-conf
+++ b/bin/dispatch-conf
@@ -202,7 +202,7 @@ class dispatch:
protect_obj = portage.util.ConfigProtect(
config_root,
config_paths,
-
portage.util.shlex_split(portage.settings.get("CONFIG_PROTECT_MASK", "")),
+portage.settings.get("CONFIG_PROTECT_MASK", "").split(),
case_insensitive=("case-insensitive-fs" in
portage.settings.features),
)
@@ -616,4 +616,4 @@ if len(sys.argv) > 1:
# for testing
d.grind(sys.argv[1:])
else:
-d.grind(portage.util.shlex_split(portage.settings.get("CONFIG_PROTECT",
"")))
+d.grind(portage.settings.get("CONFIG_PROTECT", "").split())
diff --git a/bin/portageq b/bin/portageq
index 93fa4edeba..9ef0cb7d62 100755
--- a/bin/portageq
+++ b/bin/portageq
@@ -410,8 +410,8 @@ try:
from portage.util import ConfigProtect
settings = portage.settings
-protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))
-protect_mask =
portage.util.shlex_split(settings.get("CONFIG_PROTECT_MASK", ""))
+protect = settings.get("CONFIG_PROTECT", "").split()
+protect_mask = settings.get("CONFIG_PROTECT_MASK", "").split()
protect_obj = ConfigProtect(
root,
protect,
@@ -449,8 +449,8 @@ try:
from portage.util import ConfigProtect
settings = portage.settings
-protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT", ""))
-protect_mask =
portage.util.shlex_split(settings.get("CONFIG_PROTECT_MASK", ""))
+protect = settings.get("CONFIG_PROTECT", "").split()
+protect_mask = settings.get("CONFIG_PROTECT_MASK", "").split()
protect_obj = ConfigProtect(
root,
protect,
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 9673d85f87..13add990e6 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -55,7 +55,7 @@ from portage.package.ebuild.getmaskingstatus import
_getmaskingstatus, _MaskReas
from portage._sets import SETPREFIX
from portage._sets.base import InternalPackageSet
from portage.dep._slot_operator import evaluate_slot_operator_equal_deps
-from portage.util import ConfigProtect, shlex_split, new_protect_filename
+from portage.util import ConfigProtect, new_protect_filename
from portage.util import cmp_sort_key, writemsg, writemsg_stdout
from portage.util import ensure_dirs, normalize_path
from portage.util import writemsg_level, write_atomic
@@ -10650,8 +10650,8 @@ class depgraph:
settings = self._frozen_config.roots[root].settings
protect_obj[root] = ConfigProtect(
settings["PORTAGE_CONFIGROOT"],
-shlex_split(settings.get("CONFIG_PROTECT", "")),
-shlex_split(settings.get("CONFIG_PROTECT_MASK", "")),
+settings.get("CONFIG_PROTECT", "").split(),
+settings.get("CONFIG_PROTECT_MASK", "").split(),
case_insensitive=("case-insensitive-fs" in
settings.features),
)
diff --git a/lib/_emerge/post_emerge.py b/lib/_emerge/post_emerge.py
index 37e2c3cc80..6f1f1c243d 100644
--- a/lib/_emerge/post_emerge.py
+++ b/lib/_emerge/post_emerge.py
@@ -93,7 +93,7 @@ def post_emerge(myaction, myopts, myfiles, target_root,
trees, mtimedb, retval):
settings.regenerate()
settings.lock()
-config_protect = portage.util.shlex_split(settings.get("CONFIG_PROTECT",
""))
+config_protect = settings.get("CONFIG_PROTECT", "").split()
infodirs = settings.get("INFOPATH", "").split(":") + settings.get(
"INFODIR", ""
).split(":")
diff --git a/lib/portage/_global_updates.py b/lib/portage/_global_updates.py
index f7997fc37c..4ed8a3f9a7 100644
--- a/lib/portage/_global_updates.py
+++ b/lib/portage/_global_updates.py
@@ -15,7 +15,7 @@ from portage.update import (
update_config_files,
update_dbentry,
)
-from portage.util
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: cd755af8347c32ea5247b68e6193c8428c9ffd7b Author: Florian Schmaus gentoo org> AuthorDate: Tue May 21 12:55:16 2024 + Commit: Zac Medico gentoo org> CommitDate: Tue May 21 16:07:45 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=cd755af8 SchedulerInterface/PollScheduler: _loadavg_latency's unit is seconds The unit of _loadavg_latency is seconds, not milliseconds, as it is used as first argument to an eventloop's call_later() function. Signed-off-by: Florian Schmaus gentoo.org> Closes: https://github.com/gentoo/portage/pull/1325 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/PollScheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_emerge/PollScheduler.py b/lib/_emerge/PollScheduler.py index b5bfd20b76..fdf88dd04d 100644 --- a/lib/_emerge/PollScheduler.py +++ b/lib/_emerge/PollScheduler.py @@ -11,7 +11,7 @@ from _emerge.getloadavg import getloadavg class PollScheduler: -# max time between loadavg checks (milliseconds) +# max time between loadavg checks (seconds) _loadavg_latency = None def __init__(self, main=False, event_loop=None):
[gentoo-commits] proj/portage:master commit in: bin/install-qa-check.d/
commit: 8256473c6a9fa93e7cf81c46fa920cd522507c21
Author: Eli Schwartz gmail com>
AuthorDate: Fri May 17 05:52:53 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri May 17 06:28:24 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8256473c
install-qa-checks.d: suppress some gnulib implicit configure declarations
These happen in tons of GNU packages because of using gnulib, which
pulls in macros that check for some functionality and spit out an
implicit function declaration error if they aren't supported, which is
*expected*.
Bug: https://bugs.gentoo.org/906027
Signed-off-by: Eli Schwartz gmail.com>
Closes: https://github.com/gentoo/portage/pull/1323
Signed-off-by: Sam James gentoo.org>
bin/install-qa-check.d/90config-impl-decl | 9 +
1 file changed, 9 insertions(+)
diff --git a/bin/install-qa-check.d/90config-impl-decl
b/bin/install-qa-check.d/90config-impl-decl
index 7969dff175..adea7d4051 100644
--- a/bin/install-qa-check.d/90config-impl-decl
+++ b/bin/install-qa-check.d/90config-impl-decl
@@ -49,6 +49,15 @@ add_default_skips() {
res_ndestroy
statacl
)
+
+ QA_CONFIG_IMPL_DECL_SKIP+=(
+ # Available in c23, these gnulib checks are expected to fail
+ alignof
+ static_assert
+ unreachable
+ # also gnulib, but checks both linux/non-linux headers
+ MIN
+ )
}
find_log_targets() {
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: b16ab6c7088cfde7664f1728542545e6dafce99d Author: Bryan Gardiner khumba net> AuthorDate: Sat May 11 03:18:56 2024 + Commit: Sam James gentoo org> CommitDate: Mon May 13 05:43:48 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b16ab6c7 depgraph.py: Fix the URL for the changed-deps wiki page Signed-off-by: Bryan Gardiner khumba.net> Signed-off-by: Sam James gentoo.org> lib/_emerge/depgraph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 6853ec4911..9673d85f87 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -1181,7 +1181,7 @@ class depgraph: "NOTE: Refer to the following page for more information about dependency", " change(s) without revision bump:", "", -" https://wiki.gentoo.org/wiki/Project:Portage/Changed_Deps;, +" https://wiki.gentoo.org/wiki/Project:Portage/Changed_dependencies;, "", " In order to suppress reports about dependency changes, add", " --changed-deps-report=n to the EMERGE_DEFAULT_OPTS variable in",
[gentoo-commits] proj/portage:master commit in: man/
commit: 8a98670a4a108814aaecbe3192b93505e8b38d50 Author: Bryan Gardiner khumba net> AuthorDate: Sat May 11 03:18:57 2024 + Commit: Sam James gentoo org> CommitDate: Mon May 13 05:43:48 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8a98670a ebuild.5: Escape carats which are special to groff Raw carats are converted to circumflex characters which are not searchable, so they need to be escaped. See groff_char(7). Signed-off-by: Bryan Gardiner khumba.net> Signed-off-by: Sam James gentoo.org> man/ebuild.5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/ebuild.5 b/man/ebuild.5 index a32ba4828c..36d65eb18f 100644 --- a/man/ebuild.5 +++ b/man/ebuild.5 @@ -734,7 +734,7 @@ If flag1 enabled then flag2 enabled flag1? ( flag2 ) If flag1 disabled then flag2 enabled !flag1? ( flag2 ) If flag1 disabled then flag2 disabled !flag1? ( !flag2 ) Must enable any one or more (inclusive or) || ( flag1 flag2 flag3 ) -Must enable exactly one but not more (exclusive or)^^ ( flag1 flag2 flag3 ) +Must enable exactly one but not more (exclusive or)\(ha\(ha ( flag1 flag2 flag3 ) May enable at most one (EAPI 5 or later) ?? ( flag1 flag2 flag3 ) .TE .TP
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/git/
commit: 95b4337d376a146db8fda7717393366175cbd285
Author: Alex Xu (Hello71) yahoo ca>
AuthorDate: Sun May 5 20:24:52 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun May 12 16:20:02 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=95b4337d
sync: git: add safe.directory for get head commit
Closes: https://bugs.gentoo.org/930992
Fixes: 1339a02103 ("sync: git: include signing key and git revision in log
output")
Signed-off-by: Alex Xu (Hello71) yahoo.ca>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/git/git.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/portage/sync/modules/git/git.py
b/lib/portage/sync/modules/git/git.py
index 8fdbf97de0..a2830280fb 100644
--- a/lib/portage/sync/modules/git/git.py
+++ b/lib/portage/sync/modules/git/git.py
@@ -606,6 +606,7 @@ class GitSync(NewBase):
if self.bin_command is None:
# return quietly so that we don't pollute emerge --info output
return (1, False)
+self.add_safe_directory()
rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
try:
ret = (
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 7c9bb2caeaff59232cb44e84b585db3feb2dd5fe Author: Sam James gentoo org> AuthorDate: Sat May 4 12:05:17 2024 + Commit: Sam James gentoo org> CommitDate: Sat May 4 12:05:17 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c9bb2ca actions: handle pkgmoves for autoconf, automake, libtool Signed-off-by: Sam James gentoo.org> lib/_emerge/actions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index d36a799244..b93c08aa07 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -2070,11 +2070,11 @@ def action_info(settings, trees, myopts, myfiles): append(ccache_str) myvars = [ -"sys-devel/autoconf", -"sys-devel/automake", +"dev-build/autoconf", +"dev-build/automake", "virtual/os-headers", "sys-devel/binutils", -"sys-devel/libtool", +"dev-build/libtool", "dev-lang/python", ] myvars += portage.util.grabfile(settings["PORTDIR"] + "/profiles/info_pkgs")
[gentoo-commits] proj/portage:master commit in: bin/
commit: 51f6ad158fc3de14df3f87d1242ff950b2189f05
Author: Alex Xu (Hello71) yahoo ca>
AuthorDate: Sat Mar 11 17:27:23 2023 +
Commit: Sam James gentoo org>
CommitDate: Sat May 4 01:32:47 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=51f6ad15
estrip: Don't use splitdebug for .o files
As the process_ar comment explained, object files cannot be splitdebug.
Bug: https://bugs.gentoo.org/787623 ("www-client/firefox[clang]: .gnu_debuglink
is busted when using LLD")
Fixes: 51579fb34c19 ("prepstrip: add support for elfutils strip")
Signed-off-by: Alex Xu (Hello71) yahoo.ca>
Signed-off-by: Sam James gentoo.org>
bin/estrip | 18 --
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/bin/estrip b/bin/estrip
index 2d9d50922a..640645be63 100755
--- a/bin/estrip
+++ b/bin/estrip
@@ -280,9 +280,6 @@ dedup_elf_debug() {
# Usage: save_elf_debug [splitdebug]
save_elf_debug() {
- ${FEATURES_splitdebug} || return 0
- ${PORTAGE_RESTRICT_splitdebug} && return 0
-
debug-print-function "${FUNCNAME}" "$@"
# NOTE: Debug files must be installed in
@@ -390,7 +387,7 @@ process_elf() {
if ${strip_this} ; then
# See if we can split & strip at the same time
- if [[ -n ${SPLIT_STRIP_FLAGS} ]] ; then
+ if ${splitdebug} && [[ -n ${SPLIT_STRIP_FLAGS} ]] ; then
local shortname="${x##*/}.debug"
local
splitdebug="${tmpdir}/splitdebug/${shortname}.${BASHPID:-$(__bashpid)}"
@@ -401,7 +398,9 @@ process_elf() {
"${x}"
save_elf_debug "${x}" "${inode_link}_debug"
"${splitdebug}"
else
- save_elf_debug "${x}" "${inode_link}_debug"
+ if ${splitdebug} ; then
+ save_elf_debug "${x}" "${inode_link}_debug"
+ fi
${already_stripped} || ${STRIP} ${strip_flags} "${x}"
fi
fi
@@ -431,7 +430,7 @@ process_ar() {
# There is no concept of splitdebug for objects not yet
# linked in (only for finally linked ELFs), so we have to
# retain the debug info in the archive itself.
- if ! ${FEATURES_splitdebug} || ${PORTAGE_RESTRICT_splitdebug} ;
then
+ if ! ${splitdebug} ; then
${STRIP} -g "${x}" && ${RANLIB} "${x}"
fi
fi
@@ -542,6 +541,12 @@ do
set +o noglob
fi
+ if ${FEATURES_splitdebug} && ! ${PORTAGE_RESTRICT_splitdebug} ; then
+ splitdebug=true
+ else
+ splitdebug=false
+ fi
+
# In Prefix we are usually an unprivileged user, so we can't strip
# unwritable objects. Make them temporarily writable for the
# stripping.
@@ -564,6 +569,7 @@ do
${f} == *"SB shared object"* ]] ; then
process_elf "${x}" "${inode_link}" ${PORTAGE_STRIP_FLAGS}
elif [[ ${f} == *"SB relocatable"* ]] ; then
+ [[ ${x} == *.ko ]] || splitdebug=false
process_elf "${x}" "${inode_link}" ${SAFE_STRIP_FLAGS}
fi
[gentoo-commits] proj/portage:master commit in: /
commit: 5c16a8108f21ce64284697af89e6075e2c513fbf Author: Sam James gentoo org> AuthorDate: Sun Apr 28 00:34:04 2024 + Commit: Sam James gentoo org> CommitDate: Sun Apr 28 00:34:23 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=5c16a810 NEWS: fix typo in placeholder Fixes: 60d3ffc6b9168b0bfa0cd601b3664e261fc90685 Signed-off-by: Sam James gentoo.org> NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index d0032cb557..7c1eb99616 100644 --- a/NEWS +++ b/NEWS @@ -6,7 +6,7 @@ Release notes take the form of the following optional categories: * Bug fixes * Cleanups -portage-3.0.64 (UNRELEASED) +portage-3.0.65 (UNRELEASED) -- TODO
[gentoo-commits] proj/portage:master commit in: /
commit: 248de090a42ab42261bc2234afdeeb874c53ec7a Author: Sam James gentoo org> AuthorDate: Sun Apr 28 00:33:50 2024 + Commit: Sam James gentoo org> CommitDate: Sun Apr 28 00:33:50 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=248de090 NEWS: update Signed-off-by: Sam James gentoo.org> NEWS | 5 + 1 file changed, 5 insertions(+) diff --git a/NEWS b/NEWS index f91470175d..d0032cb557 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,11 @@ Release notes take the form of the following optional categories: * Bug fixes * Cleanups +portage-3.0.64 (UNRELEASED) +-- + +TODO + portage-3.0.64 (2024-04-28) --
[gentoo-commits] proj/portage:master commit in: /
commit: 08af84b13342c86aa4c99c3d6532d26fa5703ec8 Author: Sam James gentoo org> AuthorDate: Sun Apr 28 00:28:50 2024 + Commit: Sam James gentoo org> CommitDate: Sun Apr 28 00:28:50 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=08af84b1 meson.build, NEWS: prepare for portage-3.0.64 Signed-off-by: Sam James gentoo.org> NEWS| 2 +- meson.build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 8ec71ffeb9..f91470175d 100644 --- a/NEWS +++ b/NEWS @@ -6,7 +6,7 @@ Release notes take the form of the following optional categories: * Bug fixes * Cleanups -portage-3.0.64 (UNRELEASED) +portage-3.0.64 (2024-04-28) -- Features: diff --git a/meson.build b/meson.build index b06e1f4881..3d1a27b70e 100644 --- a/meson.build +++ b/meson.build @@ -1,7 +1,7 @@ project( 'portage', 'c', -version : '3.0.63', +version : '3.0.64', license : 'GPL-2.0-or-later', meson_version : '>=0.58.0' )
[gentoo-commits] proj/portage:master commit in: /
commit: 909991ee9d18418d68ec8d1aa853f329df30e18c Author: Sam James gentoo org> AuthorDate: Sun Apr 28 00:25:42 2024 + Commit: Sam James gentoo org> CommitDate: Sun Apr 28 00:27:47 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=909991ee NEWS: update Signed-off-by: Sam James gentoo.org> NEWS | 93 1 file changed, 93 insertions(+) diff --git a/NEWS b/NEWS index ef59577f66..8ec71ffeb9 100644 --- a/NEWS +++ b/NEWS @@ -9,11 +9,104 @@ Release notes take the form of the following optional categories: portage-3.0.64 (UNRELEASED) -- +Features: +* Scheduler: Support parallel-install with merge-wait (bug #256616, bug #925213). + + This makes the (future) improvement mentioned in portage-3.0.62's NEWS. + +* phase-functions: prematurely delete WORKDIR if FEATURES=merge-wait + + The 'merge-wait' feature could lead to running out of disk space on + PORTAGE_TMPDIR's filesystem because many more workdirs were kept around + until merging was queued up. We now purge the workdirs but keep the image + around until merging. + +* Start to record REPO_REVISIONS in build-info in VDB (bug #924772). + + Ultimately the intention is to expose this information in binhost + metadata so that clients can select consistent revisions of source + repositories. + +* util: file_copy: Support btrfs's FICLONE ioctl for fast file clones. + + Added when implementing the below 'src: Drop linux_reflink C module.' + cleanup. + +* sync: Add new 'zipfile' sync-type. + + This allows fetching ebuild repositories via snapshots from e.g. GitHub, + gitweb, cgit. + Bug fixes: * ctypes: Add workaround for loading libc on musl. * util: set a timeout for urlopen calls (bug #926221). +* SpawnProcess: Fix noise / warnings caused by set_term_size (bug #923750, bug #925456). + +* binpkg, gpkg: Add missing newlines to error messages. + +* writable_check: Improve whitespace handling when parsing /proc/self/mountinfo (bug #925888). + +* doebuild: Call _setup_locale (bug #925863). + + This is needed to more robustly handle invalid or inappropriate locale settings. + +* binarytree._populate_remote: Fix UnboundLocalError for binpkg-request-signature (bug #926048). + +* elog/mod_custom: Spawn processes in background (bug #925907). + + Fixes a noisy crash on exit. + +* Fix REQUIRED_USE error with --fetchonly --pretend but not --fetchonly (bug #675748). + +* socks5: Fix exit with FEATURES="network-sandbox-proxy" or FEATURES="distcc" (bug #925240). + +* meson.build: Install dist-info METADATA (bug #920330). + + This is needed for 'pip' to not be confused when Portage is installed system-wide. + +* EbuildPhase/EbuildBinpkg: Ensure PKGDIR subdirectory permissions (bug #71). + + A warning (failure) was observed with FEATURES="installsources" otherwise. + +* config: Sort FEATURES to avoid non-determinism (bug #914441). + +* man: Document autoenabled options by --getbinpkgonly. + +* BinpkgPrefetcher: Use eerror for binarytree inject failures (bug #927632). + + This improves the readability of the error substantially. + +* install-qa-check.d: Remove QA warning when no bash completions are found (bug #928599, bug #928869). + + This check had too many false positives. + +* lib: use more pure git-describe output for --version (bug #912209). + + This makes 'emerge --version' info far nicer when using portage-. + +* lib: adhere to python package version conventions (bug #926966). + + (Note that this needed fixing even before the change for bug #912209). + +Cleanups: +* cnf: sets: Migrate @golang-rebuild to dev-lang/go (removed from Portage) (bug #919751). + +* src: Drop libc C module. + + This was only used for `find_library` on musl. We've now implemented + an alternative workaround for this. + +* src: Drop linux_reflink C module. + + This is now implemented in pure Python, which also means PyPy can + benefit. + + Testsuite coverage was also improved for this area. + +* Delete compat_coroutine module. + portage-3.0.63 (2024-02-25) --
[gentoo-commits] proj/portage:master commit in: /
commit: 87ee964c3e6d73bc537cc410426c999a1cc4ed91 Author: Alfred Wingate protonmail com> AuthorDate: Sat Apr 6 09:18:55 2024 + Commit: Sam James gentoo org> CommitDate: Sun Apr 28 00:06:03 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=87ee964c pylintrc: give fully qualified name to overgeneral-exceptions * Ability to give unqualified names will be removed in pylint-3.1.0. https://pylint.pycqa.org/en/latest/whatsnew/3/3.0/index.html#changes-requiring-user-actions Signed-off-by: Alfred Wingate protonmail.com> Signed-off-by: Sam James gentoo.org> pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index 398c04475d..9d3dae6212 100644 --- a/pylintrc +++ b/pylintrc @@ -482,4 +482,4 @@ preferred-modules= # Exceptions that will emit a warning when being caught. Defaults to # "BaseException, Exception". -overgeneral-exceptions=BaseException +overgeneral-exceptions=builtins.BaseException
[gentoo-commits] proj/portage:master commit in: /
commit: 7c144a192b91cf623830e882b1025d63c415ada7 Author: Alfred Wingate protonmail com> AuthorDate: Sat Apr 6 09:26:20 2024 + Commit: Sam James gentoo org> CommitDate: Sun Apr 28 00:06:04 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c144a19 .pre-commit-config.yaml: bump pylint to 3.1.0 Signed-off-by: Alfred Wingate protonmail.com> Closes: https://github.com/gentoo/portage/pull/1315 Signed-off-by: Sam James gentoo.org> .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f6fca10cba..4646bb3ff1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: hooks: - id: black - repo: https://github.com/PyCQA/pylint -rev: v3.0.0a7 +rev: v3.1.0 hooks: - id: pylint additional_dependencies:
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: 8c6e5d06afbf6fca1893cff5ed777e44f93b7a5d
Author: Alexey Gladkov kernel org>
AuthorDate: Sun Mar 3 16:41:08 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8c6e5d06
sync/zipfile: Handle ETag header
Most services add an ETag header and determine whether the locally
cached version of the URL has expired. So we can add ETag processing to
avoid unnecessary downloading and unpacking of the zip archive.
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/zipfile.py | 36 +++--
1 file changed, 29 insertions(+), 7 deletions(-)
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
index 1762d2c8f1..bb78b39243 100644
--- a/lib/portage/sync/modules/zipfile/zipfile.py
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -10,7 +10,7 @@ import tempfile
import urllib.request
import portage
-from portage.util import writemsg_level
+from portage.util import writemsg_level, writemsg_stdout
from portage.sync.syncbase import SyncBase
@@ -31,13 +31,31 @@ class ZipFile(SyncBase):
if kwargs:
self._kwargs(kwargs)
-# initial checkout
-zip_uri = self.repo.sync_uri
+req = urllib.request.Request(url=self.repo.sync_uri)
-with urllib.request.urlopen(zip_uri) as response:
-with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
-shutil.copyfileobj(response, tmp_file)
-zip_file = tmp_file.name
+info = portage.grabdict(os.path.join(self.repo.location, ".info"))
+if "etag" in info:
+req.add_header("If-None-Match", info["etag"][0])
+
+try:
+with urllib.request.urlopen(req) as response:
+with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+shutil.copyfileobj(response, tmp_file)
+
+zip_file = tmp_file.name
+etag = response.headers.get("etag")
+
+except urllib.error.HTTPError as resp:
+if resp.code == 304:
+writemsg_stdout(">>> The repository has not changed.\n",
noiselevel=-1)
+return (os.EX_OK, False)
+
+writemsg_level(
+f"!!! Unable to obtain zip archive: {resp}\n",
+noiselevel=-1,
+level=logging.ERROR,
+)
+return (1, False)
if not zipfile.is_zipfile(zip_file):
msg = "!!! file is not a zip archive."
@@ -77,6 +95,10 @@ class ZipFile(SyncBase):
with open(dstpath, "wb") as dstfile:
shutil.copyfileobj(srcfile, dstfile)
+with open(os.path.join(self.repo.location, ".info"), "w") as infofile:
+if etag:
+infofile.write(f"etag {etag}\n")
+
os.unlink(zip_file)
return (os.EX_OK, True)
[gentoo-commits] proj/portage:master commit in: man/
commit: 221fa725deb03a65308864d812d3f8eddec6c930 Author: Alexey Gladkov kernel org> AuthorDate: Sat Apr 27 13:29:47 2024 + Commit: Sam James gentoo org> CommitDate: Sun Apr 28 00:04:10 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=221fa725 zipfile: Add an example to portage(5) Signed-off-by: Alexey Gladkov kernel.org> Closes: https://github.com/gentoo/portage/pull/1264 Signed-off-by: Sam James gentoo.org> man/portage.5 | 11 ++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/man/portage.5 b/man/portage.5 index d8990bf416..1f717c4cb8 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -1140,7 +1140,7 @@ expire while it is in use by a running process. .B sync\-type Specifies type of synchronization performed by `emerge \-\-sync`. .br -Valid non\-empty values: cvs, git, mercurial, rsync, svn, webrsync +Valid non\-empty values: cvs, git, mercurial, rsync, zipfile, svn, webrsync (emerge\-webrsync) .br This attribute can be set to empty value to disable synchronization of given @@ -1165,6 +1165,8 @@ cvs: [cvs://]:access_method:[username@]hostname[:port]:/path git: (git|git+ssh|http|https)://[username@]hostname[:port]/path .br rsync: (rsync|ssh)://[username@]hostname[:port]/(module|path) +.br +zipfile: (http|https)://hostname[:port]/path/to/zipfile.zip .TP Examples: .RS @@ -1327,6 +1329,13 @@ location = /var/lib/layman/voip sync\-type = laymanator sync\-uri = git://anongit.gentoo.org/proj/voip.git auto\-sync = yes + +# Overlay with latest ebuild repository snapshot. +[snapshot] +location = /var/db/repos/snapshot +sync\-type = zipfile +sync\-uri = https://github.com/gentoo/gentoo/archive/refs/heads/master.zip +auto\-sync = yes .fi .RE .TP
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: 7e93192fda22594b9e9d223c54a39b4bad0554f9
Author: Alexey Gladkov kernel org>
AuthorDate: Mon Mar 11 00:25:07 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:08 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=7e93192f
sync/zipfile: Add retrieve_head to return archive checksum
Since we have an ETag, we can return the checksum of the archive. It
will be a replacement for the head commit of the repository.
Suggested-by: Zac Medico gentoo.org>
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/__init__.py | 3 ++-
lib/portage/sync/modules/zipfile/zipfile.py | 9 +
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/lib/portage/sync/modules/zipfile/__init__.py
b/lib/portage/sync/modules/zipfile/__init__.py
index 19fe3af412..e44833088c 100644
--- a/lib/portage/sync/modules/zipfile/__init__.py
+++ b/lib/portage/sync/modules/zipfile/__init__.py
@@ -21,10 +21,11 @@ module_spec = {
"sourcefile": "zipfile",
"class": "ZipFile",
"description": doc,
-"functions": ["sync"],
+"functions": ["sync", "retrieve_head"],
"func_desc": {
"sync": "Performs an archived http download of the "
+ "repository, then unpacks it.",
+"retrieve_head": "Returns the checksum of the unpacked
archive.",
},
"validate_config": CheckSyncConfig,
"module_specific_options": (),
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
index bb78b39243..3cd210a64b 100644
--- a/lib/portage/sync/modules/zipfile/zipfile.py
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -26,6 +26,15 @@ class ZipFile(SyncBase):
def __init__(self):
SyncBase.__init__(self, "emerge", ">=sys-apps/portage-2.3")
+def retrieve_head(self, **kwargs):
+"""Get information about the checksum of the unpacked archive"""
+if kwargs:
+self._kwargs(kwargs)
+info = portage.grabdict(os.path.join(self.repo.location, ".info"))
+if "etag" in info:
+return (os.EX_OK, info["etag"][0])
+return (1, False)
+
def sync(self, **kwargs):
"""Sync the repository"""
if kwargs:
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: ced2e6d4f4ac95b8e17cf7dae964a64037a85bf0
Author: Alexey Gladkov kernel org>
AuthorDate: Mon Mar 11 17:09:05 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:09 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=ced2e6d4
sync/zipfile: Recycle files that have not changed
We can check whether the content of files from the archive differs from
the current revision. This will give us several advantages:
* This will give us some meaning to the mtime of files, since it will
prevent the timestamps of unmodified files from being changed.
* This will also get rid of recreatiing self.repo.location, which will
allow sync with FEATURES=usersync because self.repo.location is reused.
Suggested-by: Zac Medico gentoo.org>
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/zipfile.py | 32 -
1 file changed, 31 insertions(+), 1 deletion(-)
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
index 3cd210a64b..edfb5aa681 100644
--- a/lib/portage/sync/modules/zipfile/zipfile.py
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -35,6 +35,16 @@ class ZipFile(SyncBase):
return (os.EX_OK, info["etag"][0])
return (1, False)
+def _do_cmp(self, f1, f2):
+bufsize = 8 * 1024
+while True:
+b1 = f1.read(bufsize)
+b2 = f2.read(bufsize)
+if b1 != b2:
+return False
+if not b1:
+return True
+
def sync(self, **kwargs):
"""Sync the repository"""
if kwargs:
@@ -76,7 +86,15 @@ class ZipFile(SyncBase):
return (1, False)
# Drop previous tree
-shutil.rmtree(self.repo.location)
+tempdir = tempfile.mkdtemp(prefix=".temp", dir=self.repo.location)
+tmpname = os.path.basename(tempdir)
+
+for name in os.listdir(self.repo.location):
+if name != tmpname:
+os.rename(
+os.path.join(self.repo.location, name),
+os.path.join(tempdir, name),
+)
with zipfile.ZipFile(zip_file) as archive:
strip_comp = 0
@@ -101,9 +119,21 @@ class ZipFile(SyncBase):
continue
with archive.open(n) as srcfile:
+prvpath = os.path.join(tempdir, *parts[strip_comp:])
+
+if os.path.exists(prvpath):
+with open(prvpath, "rb") as prvfile:
+if self._do_cmp(prvfile, srcfile):
+os.rename(prvpath, dstpath)
+continue
+srcfile.seek(0)
+
with open(dstpath, "wb") as dstfile:
shutil.copyfileobj(srcfile, dstfile)
+# Drop previous tree
+shutil.rmtree(tempdir)
+
with open(os.path.join(self.repo.location, ".info"), "w") as infofile:
if etag:
infofile.write(f"etag {etag}\n")
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/sync/
commit: b01cd4208a17a141311d490788aff11537312575
Author: Zac Medico gentoo org>
AuthorDate: Tue Mar 12 10:20:56 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:09 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b01cd420
sync/zipfile: Add testcase for etag
Signed-off-by: Zac Medico gentoo.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/tests/sync/meson.build | 1 +
lib/portage/tests/sync/test_sync_zipfile.py | 99 +
2 files changed, 100 insertions(+)
diff --git a/lib/portage/tests/sync/meson.build
b/lib/portage/tests/sync/meson.build
index b78583021f..8c566080e3 100644
--- a/lib/portage/tests/sync/meson.build
+++ b/lib/portage/tests/sync/meson.build
@@ -1,6 +1,7 @@
py.install_sources(
[
'test_sync_local.py',
+'test_sync_zipfile.py',
'__init__.py',
'__test__.py',
],
diff --git a/lib/portage/tests/sync/test_sync_zipfile.py
b/lib/portage/tests/sync/test_sync_zipfile.py
new file mode 100644
index 00..4fbde8a351
--- /dev/null
+++ b/lib/portage/tests/sync/test_sync_zipfile.py
@@ -0,0 +1,99 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import http.server
+import os
+import shutil
+import socketserver
+import subprocess
+import tempfile
+import textwrap
+import threading
+from functools import partial
+
+import portage
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import ResolverPlayground
+
+
+class test_sync_zipfile_case(TestCase):
+def test_sync_zipfile(self):
+cpv = "dev-libs/A-0"
+ebuilds = {
+cpv: {"EAPI": "8"},
+}
+etag = "foo"
+
+server = None
+playground = None
+tmpdir = tempfile.mkdtemp()
+try:
+
+class Handler(http.server.SimpleHTTPRequestHandler):
+def end_headers(self):
+self.send_header("etag", etag)
+super().end_headers()
+
+server = socketserver.TCPServer(
+("127.0.0.1", 0),
+partial(Handler, directory=tmpdir),
+)
+threading.Thread(target=server.serve_forever, daemon=True).start()
+
+playground = ResolverPlayground(
+ebuilds=ebuilds,
+)
+settings = playground.settings
+
+env = settings.environ()
+
+repos_conf = textwrap.dedent(
+"""
+[test_repo]
+location = %(location)s
+sync-type = zipfile
+sync-uri = %(sync-uri)s
+auto-sync = true
+"""
+)
+
+repo_location = f"{playground.eprefix}/var/repositories/test_repo"
+
+env["PORTAGE_REPOSITORIES"] = repos_conf % {
+"location": repo_location,
+"sync-uri":
"http://{}:{}/test_repo.zip".format(*server.server_address),
+}
+
+shutil.make_archive(os.path.join(tmpdir, "test_repo"), "zip",
repo_location)
+
+ebuild =
playground.trees[playground.eroot]["porttree"].dbapi.findname(cpv)
+self.assertTrue(os.path.exists(ebuild))
+shutil.rmtree(repo_location)
+self.assertFalse(os.path.exists(ebuild))
+
+result = subprocess.run(
+[
+"emerge",
+"--sync",
+],
+env=env,
+stdout=subprocess.PIPE,
+stderr=subprocess.STDOUT,
+)
+output = result.stdout.decode(errors="replace")
+try:
+self.assertEqual(result.returncode, os.EX_OK)
+except Exception:
+print(output)
+raise
+
+repo = settings.repositories["test_repo"]
+sync_mod = portage.sync.module_controller.get_class("zipfile")
+status, repo_revision = sync_mod().retrieve_head(options={"repo":
repo})
+self.assertEqual(status, os.EX_OK)
+self.assertEqual(repo_revision, etag)
+finally:
+if server is not None:
+server.shutdown()
+shutil.rmtree(tmpdir)
+playground.cleanup()
[gentoo-commits] proj/portage:master commit in: lib/portage/sync/modules/zipfile/
commit: 80445d9b00bfcd1eb4955cf3ecb397b4c02663ba
Author: Alexey Gladkov kernel org>
AuthorDate: Mon Feb 12 13:59:40 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Apr 28 00:04:07 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=80445d9b
sync: Add method to download zip archives
Add a simple method for synchronizing repository as a snapshot in a zip
archive. The implementation does not require external utilities to
download and unpack archive. This makes the method very cheap.
The main usecase being considered is obtaining snapshots of github
repositories, but many other web interfaces for git also support
receiving snapshots in zip format.
For example, to get a snapshot of the master branch:
https://github.com/gentoo/portage/archive/refs/heads/master.zip
https://gitweb.gentoo.org/proj/portage.git/snapshot/portage-master.zip
or a link to a snapshot of the tag:
https://github.com/gentoo/portage/archive/refs/tags/portage-3.0.61.zip
Signed-off-by: Alexey Gladkov kernel.org>
Signed-off-by: Sam James gentoo.org>
lib/portage/sync/modules/zipfile/__init__.py | 33 +++
lib/portage/sync/modules/zipfile/zipfile.py | 82
2 files changed, 115 insertions(+)
diff --git a/lib/portage/sync/modules/zipfile/__init__.py
b/lib/portage/sync/modules/zipfile/__init__.py
new file mode 100644
index 00..19fe3af412
--- /dev/null
+++ b/lib/portage/sync/modules/zipfile/__init__.py
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2024 Alexey Gladkov
+
+doc = """Zipfile plug-in module for portage.
+Performs a http download of a portage snapshot and unpacks it to the repo
+location."""
+__doc__ = doc[:]
+
+
+import os
+
+from portage.sync.config_checks import CheckSyncConfig
+
+
+module_spec = {
+"name": "zipfile",
+"description": doc,
+"provides": {
+"zipfile-module": {
+"name": "zipfile",
+"sourcefile": "zipfile",
+"class": "ZipFile",
+"description": doc,
+"functions": ["sync"],
+"func_desc": {
+"sync": "Performs an archived http download of the "
++ "repository, then unpacks it.",
+},
+"validate_config": CheckSyncConfig,
+"module_specific_options": (),
+},
+},
+}
diff --git a/lib/portage/sync/modules/zipfile/zipfile.py
b/lib/portage/sync/modules/zipfile/zipfile.py
new file mode 100644
index 00..1762d2c8f1
--- /dev/null
+++ b/lib/portage/sync/modules/zipfile/zipfile.py
@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2024 Alexey Gladkov
+
+import os
+import os.path
+import logging
+import zipfile
+import shutil
+import tempfile
+import urllib.request
+
+import portage
+from portage.util import writemsg_level
+from portage.sync.syncbase import SyncBase
+
+
+class ZipFile(SyncBase):
+"""ZipFile sync module"""
+
+short_desc = "Perform sync operations on GitHub repositories"
+
+@staticmethod
+def name():
+return "ZipFile"
+
+def __init__(self):
+SyncBase.__init__(self, "emerge", ">=sys-apps/portage-2.3")
+
+def sync(self, **kwargs):
+"""Sync the repository"""
+if kwargs:
+self._kwargs(kwargs)
+
+# initial checkout
+zip_uri = self.repo.sync_uri
+
+with urllib.request.urlopen(zip_uri) as response:
+with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+shutil.copyfileobj(response, tmp_file)
+zip_file = tmp_file.name
+
+if not zipfile.is_zipfile(zip_file):
+msg = "!!! file is not a zip archive."
+self.logger(self.xterm_titles, msg)
+writemsg_level(msg + "\n", noiselevel=-1, level=logging.ERROR)
+
+os.unlink(zip_file)
+
+return (1, False)
+
+# Drop previous tree
+shutil.rmtree(self.repo.location)
+
+with zipfile.ZipFile(zip_file) as archive:
+strip_comp = 0
+
+for f in archive.namelist():
+f = os.path.normpath(f)
+if os.path.basename(f) == "profiles":
+strip_comp = f.count("/")
+break
+
+for n in archive.infolist():
+p = os.path.normpath(n.filename)
+
+if os.path.isabs(p):
+continue
+
+parts = p.split("/")
+dstpath = os.path.join(self.repo.location, *parts[strip_comp:])
+
+if n.is_dir():
+os.makedirs(dstpath, mode=0o755, exist_ok=True)
+continue
+
+with archive.open(n) as srcfile:
+with open(dstpath, "wb") as dstfile:
+shutil.copyfileobj(srcfile, dstfile)
+
+os.unlink(zip_file)
+
+return (os.EX_OK, True)
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 36235596e061bf8cf4729b3915f9aaa6ec80baa3
Author: Alfred Wingate protonmail com>
AuthorDate: Sat Apr 6 08:28:43 2024 +
Commit: Sam James gentoo org>
CommitDate: Fri Apr 26 22:05:48 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=36235596
lib: adhere to python package version conventions
* Commit metadata isn't valid version that python tooling is expected to
parse. Follow python ecosystem conventions and make it a local
version.
https://packaging.python.org/en/latest/specifications/version-specifiers/#local-version-segments
Example:
* Old: 3.0.63-g08a2bc380
* New: 3.0.63+g08a2bc380
Bug: https://bugs.gentoo.org/926966
Signed-off-by: Alfred Wingate protonmail.com>
Closes: https://github.com/gentoo/portage/pull/1314
Signed-off-by: Sam James gentoo.org>
lib/portage/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index a468eeaff3..21bf993170 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -732,7 +732,7 @@ if installation.TYPE == installation.TYPES.SOURCE:
output = _unicode_decode(proc.communicate()[0],
encoding=encoding)
status = proc.wait()
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
-VERSION = output.lstrip('portage-').strip()
+VERSION = output.lstrip("portage-").strip().replace("-g",
"+g")
else:
VERSION = "HEAD"
return VERSION
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 381fad5e3554ec94ec5626e8c17874f32b30b752
Author: Sam James gentoo org>
AuthorDate: Tue Aug 29 07:26:36 2023 +
Commit: Sam James gentoo org>
CommitDate: Fri Apr 26 22:05:48 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=381fad5e
lib: use more pure git-describe output for --version
Use `git describe --dirty` output rather than mangling git-describe and
reinventing
--dirty by manually checking for changes post-commit.
We no longer mangle the 7th commit post-tag into _p7, but instead do:
${tag}-7-${last_commit}.
This is similar to gnulib's git-version-gen (which we may still want to import,
not sure, this seems enough for now) and is familiar output for developers.
Example:
* Old: 3.0.51_p7
* New: 3.0.51-7-g098b30548
Bug: https://bugs.gentoo.org/912209
Signed-off-by: Sam James gentoo.org>
lib/portage/__init__.py | 35 ---
1 file changed, 4 insertions(+), 31 deletions(-)
diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index aa81bdb4c2..a468eeaff3 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -720,10 +720,7 @@ if installation.TYPE == installation.TYPES.SOURCE:
BASH_BINARY,
"-c",
(
-f"cd {_shell_quote(PORTAGE_BASE_PATH)} ; git describe
--match 'portage-*' || exit $? ; "
-'if [ -n "`git diff-index --name-only --diff-filter=M
HEAD`" ] ; '
-"then echo modified ; git rev-list --format=%%ct -n 1
HEAD ; fi ; "
-"exit 0"
+f"cd {_shell_quote(PORTAGE_BASE_PATH)} ; git describe
--dirty --match 'portage-*' || exit $? ; "
),
]
cmd = [
@@ -735,33 +732,9 @@ if installation.TYPE == installation.TYPES.SOURCE:
output = _unicode_decode(proc.communicate()[0],
encoding=encoding)
status = proc.wait()
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK:
-output_lines = output.splitlines()
-if output_lines:
-version_split = output_lines[0].split("-")
-if len(version_split) > 1:
-VERSION = version_split[1]
-patchlevel = False
-if len(version_split) > 2:
-patchlevel = True
-VERSION = f"{VERSION}_p{version_split[2]}"
-if len(output_lines) > 1 and output_lines[1] ==
"modified":
-head_timestamp = None
-if len(output_lines) > 3:
-try:
-head_timestamp = int(output_lines[3])
-except ValueError:
-pass
-timestamp = int(time.time())
-if (
-head_timestamp is not None
-and timestamp > head_timestamp
-):
-timestamp = timestamp - head_timestamp
-if not patchlevel:
-VERSION = f"{VERSION}_p0"
-VERSION = f"{VERSION}_p{timestamp}"
-return VERSION
-VERSION = "HEAD"
+VERSION = output.lstrip('portage-').strip()
+else:
+VERSION = "HEAD"
return VERSION
VERSION = _LazyVersion()
[gentoo-commits] proj/portage:master commit in: bin/install-qa-check.d/
commit: 8bbdc814485d2990630d4217f68830fe1148801c
Author: Mike Gilbert gentoo org>
AuthorDate: Wed Apr 24 15:35:32 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Wed Apr 24 15:35:32 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8bbdc814
Remove QA warning when no bash completions are found
This triggers unwanted warnings in at least two known cases.
Bug: https://bugs.gentoo.org/928599
Bug: https://bugs.gentoo.org/928869
Signed-off-by: Mike Gilbert gentoo.org>
bin/install-qa-check.d/60bash-completion | 3 ---
1 file changed, 3 deletions(-)
diff --git a/bin/install-qa-check.d/60bash-completion
b/bin/install-qa-check.d/60bash-completion
index 85573df0e5..d3cb52d1c1 100644
--- a/bin/install-qa-check.d/60bash-completion
+++ b/bin/install-qa-check.d/60bash-completion
@@ -77,9 +77,6 @@ bashcomp_check() {
fi
if [[ -z ${completions[@]} ]]; then
- qa_warnings+=(
- "${f##*/}: does not define any
completions (failed to source?)."
- )
continue
fi
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: 08a2bc3800ea2e997716903244ec041339b45b06
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 24 22:11:54 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Mar 28 14:47:49 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=08a2bc38
BinpkgPrefetcher: Emit eerror message for binarytree inject failure
Since it is confusing when emerge exits due to an error that only shows
in the fetch log, emit an eerror message when binarytree inject fails:
>>> Running pre-merge checks for sys-libs/glibc-2.38-r10
* Fetching in the background:
* /var/cache/binpkgs/sys-libs/glibc/glibc-2.38-r10-5.gpkg.tar.partial
* To view fetch progress, run in another terminal:
* tail -f /var/log/emerge-fetch.log
>>> Failed to emerge sys-libs/glibc-2.38-r10
* Messages for package sys-libs/glibc-2.38-r10:
* Binary package is not usable:
* !!!
* gpg: keyblock resource '/etc/portage/gnupg/pubring.kbx': No such file
or directory
* [GNUPG:] ERROR add_keyblock_resource 33587281
* [GNUPG:] PLAINTEXT 74 0
* [GNUPG:] NEWSIG
* gpg: Signature made Wed 20 Mar 2024 10:34:45 PM CET
* gpg:using RSA key
534E4209AB49EEE1C19D96162C44695DB9F6043D
* [GNUPG:] ERROR keydb_search 33554445
* [GNUPG:] ERROR keydb_search 33554445
* [GNUPG:] ERRSIG 2C44695DB9F6043D 1 10 01 1710970485 9
534E4209AB49EEE1C19D96162C44695DB9F6043D
* [GNUPG:] NO_PUBKEY 2C44695DB9F6043D
* !!! Invalid binary package:
'/var/cache/binpkgs/sys-devel/binutils/binutils-2.41-r5-4.gpkg.tar.partial',
GPG verify failed
Fixes: 0ff49114cec7 ("binarytree: Handle inject failures")
Bug: https://bugs.gentoo.org/927632
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/BinpkgPrefetcher.py | 35 ++-
1 file changed, 30 insertions(+), 5 deletions(-)
diff --git a/lib/_emerge/BinpkgPrefetcher.py b/lib/_emerge/BinpkgPrefetcher.py
index ed68d2852c..a8af30ca80 100644
--- a/lib/_emerge/BinpkgPrefetcher.py
+++ b/lib/_emerge/BinpkgPrefetcher.py
@@ -7,7 +7,10 @@ import sys
from _emerge.BinpkgFetcher import BinpkgFetcher
from _emerge.CompositeTask import CompositeTask
from _emerge.BinpkgVerifier import BinpkgVerifier
+import portage
from portage import os
+from portage.elog import messages as elog_messages
+from portage.util import no_color
class BinpkgPrefetcher(CompositeTask):
@@ -48,6 +51,7 @@ class BinpkgPrefetcher(CompositeTask):
self.wait()
return
+injected_pkg = None
stdout_orig = sys.stdout
stderr_orig = sys.stderr
out = io.StringIO()
@@ -67,12 +71,33 @@ class BinpkgPrefetcher(CompositeTask):
output_value = out.getvalue()
if output_value:
-self.scheduler.output(
-output_value,
-log_path=self.scheduler.fetch.log_file,
-background=self.background,
-)
+if injected_pkg is None:
+msg = ["Binary package is not usable:"]
+msg.extend("\t" + line for line in
output_value.splitlines())
+self._elog("eerror", msg)
+else:
+self.scheduler.output(
+output_value,
+log_path=self.scheduler.fetch.log_file,
+background=self.background,
+)
self._current_task = None
self.returncode = 1 if injected_pkg is None else os.EX_OK
self.wait()
+
+def _elog(self, elog_funcname, lines, phase="other"):
+out = io.StringIO()
+elog_func = getattr(elog_messages, elog_funcname)
+global_havecolor = portage.output.havecolor
+try:
+portage.output.havecolor = not no_color(self._bintree.settings)
+for line in lines:
+elog_func(line, phase=phase, key=self.pkg.cpv, out=out)
+finally:
+portage.output.havecolor = global_havecolor
+msg = out.getvalue()
+if msg:
+self.scheduler.output(
+msg, background=self.background,
log_path=self.scheduler.fetch.log_file
+)
[gentoo-commits] proj/portage:master commit in: man/
commit: 3bc61292f9d44f8fd4d8655ab10e2064c70c912f Author: Gábor Oszkár Dénes protonmail com> AuthorDate: Thu Feb 29 20:56:00 2024 + Commit: Sam James gentoo org> CommitDate: Thu Mar 28 04:26:12 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3bc61292 man: Document autoenabled options by --getbinpkgonly Signed-off-by: Gábor Oszkár Dénes protonmail.com> Closes: https://github.com/gentoo/portage/pull/1291 Signed-off-by: Sam James gentoo.org> man/emerge.1 | 8 +--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/man/emerge.1 b/man/emerge.1 index 43dc3f26b1..dd9cde5722 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -447,15 +447,17 @@ Tells emerge to ignore binary packages for which the corresponding ebuild dependencies have changed since the packages were built. In order to help avoid issues with resolving inconsistent dependencies, this option is automatically enabled unless the \fB\-\-usepkgonly\fR -option is enabled. Behavior with respect to changed build\-time +(or \fB\-\-getbinpkgonly\fR) option is enabled. +Behavior with respect to changed build\-time dependencies is controlled by the \fB\-\-with\-bdeps\fR option. .TP .BR "\-\-binpkg\-respect\-use [ y | n ]" Tells emerge to ignore binary packages if their USE flags don't match the current configuration. In order to help avoid issues with resolving inconsistent USE flag settings, this option is -automatically enabled unless the \fB\-\-usepkgonly\fR option -is enabled. If \fB\-\-binpkg\-respect\-use\fR is given explicitly, +automatically enabled unless the \fB\-\-usepkgonly\fR +(or \fB\-\-getbinpkgonly\fR) option is enabled. +If \fB\-\-binpkg\-respect\-use\fR is given explicitly, then it implies \fB\-\-autounmask\-use=n\fR, because these options naturally oppose each other. .TP
[gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/
commit: 72f41d07b5396195a98691fafeb3e5b207a76564 Author: Raul E Rangel chromium org> AuthorDate: Thu Mar 21 20:29:59 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 24 22:18:34 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=72f41d07 Reapply "config: Don't directly modify FEATURES" This reverts commit b150419d28bd7afb98404a829c639584d34efc03. It turns out we need to keep the open coded version to avoid creating a persistent setting. See https://github.com/gentoo/portage/pull/1098#issuecomment-1761638611 This change just adds a sorted() so we get deterministic ordering. Bug: https://bugs.gentoo.org/914441 Signed-off-by: Raul E Rangel chromium.org> Closes: https://github.com/gentoo/portage/pull/1312 Signed-off-by: Zac Medico gentoo.org> lib/portage/package/ebuild/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py index bafdc55a08..67fd1bb18d 100644 --- a/lib/portage/package/ebuild/config.py +++ b/lib/portage/package/ebuild/config.py @@ -2206,7 +2206,9 @@ class config: # "test" is in IUSE and USE=test is masked, so execution # of src_test() probably is not reliable. Therefore, # temporarily disable FEATURES=test just for this package. -self["FEATURES"] = " ".join(x for x in self.features if x != "test") +self["FEATURES"] = " ".join( +x for x in sorted(self.features) if x != "test" +) # Allow _* flags from USE_EXPAND wildcards to pass through here. use.difference_update(
[gentoo-commits] proj/portage:master commit in: .github/workflows/
commit: b276077666c2fca47c91eae19669f1bf6c0c702f
Author: Mike Gilbert gentoo org>
AuthorDate: Fri Mar 15 20:27:06 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 15 21:34:16 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2760776
github: update actions to avoid deprecation warnings
Signed-off-by: Mike Gilbert gentoo.org>
.github/workflows/ci.yml | 6 +++---
.github/workflows/lint.yml | 13 ++---
.github/workflows/pre-commit.yml | 8 +---
3 files changed, 14 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e4168203d4..2ac66970e5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -18,7 +18,7 @@ jobs:
- '3.9'
- '3.10'
- '3.11'
- - '3.12-dev'
+ - '3.12'
- '3.13-dev'
- 'pypy-3.10'
exclude:
@@ -34,9 +34,9 @@ jobs:
start-method: 'spawn'
fail-fast: false
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
-uses: actions/setup-python@v4
+uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 2091796b7c..ddcd9d486d 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -6,13 +6,13 @@ jobs:
black:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: "Collect stragglers that Black misses"
id: stragglers
run: |
- echo "::set-output name=missed::$(
+ echo "missed=$(
find bin -type f -not -name '*.py' -not -name '*.sh' | \
- xargs grep -l '#!/usr/bin/env python' | tr $'\n' ' ')"
+ xargs grep -l '#!/usr/bin/env python' | tr $'\n' ' ')" >>
$GITHUB_OUTPUT
- uses: psf/black@24.1.1
with:
src: . ${{ steps.stragglers.outputs.missed }}
@@ -25,13 +25,12 @@ jobs:
- '3.9'
- '3.10'
- '3.11'
- # pylint currently broken under 3.12
- # - '3.12-dev'
+ - '3.12'
fail-fast: false
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
-uses: actions/setup-python@v4
+uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install python dependencies
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
index ad71791c0b..1c3fbc48b0 100644
--- a/.github/workflows/pre-commit.yml
+++ b/.github/workflows/pre-commit.yml
@@ -9,6 +9,8 @@ jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
-- uses: actions/checkout@v3
-- uses: actions/setup-python@v4
-- uses: pre-commit/action@v3.0.0
+- uses: actions/checkout@v4
+- uses: actions/setup-python@v5
+ with:
+python-version: '3.x'
+- uses: pre-commit/action@v3.0.1
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/util/file_copy/, src/
commit: 23529ee81964665107400e87fc3d49c256e915c0
Author: Mike Gilbert gentoo org>
AuthorDate: Fri Mar 1 15:45:58 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 15 20:05:34 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=23529ee8
Replace linux_reflink extension module
Python 3.8 added support for copy_file_range in the os module,
so we can just call that directly.
Also, we can use the FICLONE ioctl for fast file clones on supported
filesystems (btrfs).
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/file_copy.py | 137 ++
lib/portage/util/file_copy/__init__.py | 36 ---
lib/portage/util/file_copy/meson.build | 7 -
lib/portage/util/meson.build | 2 +-
src/meson.build| 20 --
src/portage_util_file_copy_reflink_linux.c | 396 -
6 files changed, 138 insertions(+), 460 deletions(-)
diff --git a/lib/portage/util/file_copy.py b/lib/portage/util/file_copy.py
new file mode 100644
index 00..e3926d8ef6
--- /dev/null
+++ b/lib/portage/util/file_copy.py
@@ -0,0 +1,137 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import errno
+import fcntl
+import logging
+import os
+import platform
+import shutil
+import sys
+
+
+logger = logging.getLogger(__name__)
+
+# Added in Python 3.12
+FICLONE = getattr(fcntl, "FICLONE", 0x40049409)
+
+# Unavailable in PyPy
+SEEK_DATA = getattr(os, "SEEK_DATA", 3)
+SEEK_HOLE = getattr(os, "SEEK_HOLE", 4)
+
+
+def _get_chunks(src):
+try:
+offset_hole = 0
+while True:
+try:
+# Find the next bit of data
+offset_data = os.lseek(src, offset_hole, SEEK_DATA)
+except OSError as e:
+# Re-raise for unexpected errno values
+if e.errno not in (errno.EINVAL, errno.ENXIO):
+raise
+
+offset_end = os.lseek(src, 0, os.SEEK_END)
+
+if e.errno == errno.ENXIO:
+# End of file
+if offset_end > offset_hole:
+# Hole at end of file
+yield (offset_end, 0)
+else:
+# SEEK_DATA failed with EINVAL, return the whole file
+yield (0, offset_end)
+
+break
+else:
+offset_hole = os.lseek(src, offset_data, SEEK_HOLE)
+yield (offset_data, offset_hole - offset_data)
+
+except OSError:
+logger.warning("_get_chunks failed unexpectedly",
exc_info=sys.exc_info())
+raise
+
+
+def _do_copy_file_range(src, dst, offset, count):
+while count > 0:
+# count must fit in ssize_t
+c = min(count, sys.maxsize)
+written = os.copy_file_range(src, dst, c, offset, offset)
+if written == 0:
+# https://bugs.gentoo.org/828844
+raise OSError(errno.EOPNOTSUPP, os.strerror(errno.EOPNOTSUPP))
+offset += written
+count -= written
+
+
+def _do_sendfile(src, dst, offset, count):
+os.lseek(dst, offset, os.SEEK_SET)
+while count > 0:
+# count must fit in ssize_t
+c = min(count, sys.maxsize)
+written = os.sendfile(dst, src, offset, c)
+offset += written
+count -= written
+
+
+def _fastcopy(src, dst):
+with (
+open(src, "rb", buffering=0) as srcf,
+open(dst, "wb", buffering=0) as dstf,
+):
+srcfd = srcf.fileno()
+dstfd = dstf.fileno()
+
+if platform.system() == "Linux":
+try:
+fcntl.ioctl(dstfd, FICLONE, srcfd)
+return
+except OSError:
+pass
+
+try_cfr = hasattr(os, "copy_file_range")
+
+for offset, count in _get_chunks(srcfd):
+if count == 0:
+os.ftruncate(dstfd, offset)
+else:
+if try_cfr:
+try:
+_do_copy_file_range(srcfd, dstfd, offset, count)
+continue
+except OSError as e:
+try_cfr = False
+if e.errno not in (errno.EXDEV, errno.ENOSYS,
errno.EOPNOTSUPP):
+logger.warning(
+"_do_copy_file_range failed unexpectedly",
+exc_info=sys.exc_info(),
+)
+try:
+_do_sendfile(srcfd, dstfd, offset, count)
+except OSError:
+logger.warning(
+"_do_sendfile failed unexpectedly",
exc_info=sys.exc_info()
+)
+raise
+
+
+def copyfile(src, dst):
+"""
+Copy the contents (no metadata) of the file named src to a file
+named dst.
+
+If possible,
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/file_copy/
commit: 0cc3f9e269b26173c2f81d731c5fe208b758270b
Author: Mike Gilbert gentoo org>
AuthorDate: Sun Mar 3 02:28:42 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 15 20:05:34 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0cc3f9e2
Improve testCopyFileSparse
Actually create sparse blocks at the start and end.
Check file size before/after copying.
Ensure sparse output when _fastcopy succeeds.
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/tests/util/file_copy/test_copyfile.py | 35 ---
1 file changed, 24 insertions(+), 11 deletions(-)
diff --git a/lib/portage/tests/util/file_copy/test_copyfile.py
b/lib/portage/tests/util/file_copy/test_copyfile.py
index e91a47bed8..e114e6ae35 100644
--- a/lib/portage/tests/util/file_copy/test_copyfile.py
+++ b/lib/portage/tests/util/file_copy/test_copyfile.py
@@ -3,13 +3,14 @@
import shutil
import tempfile
+from unittest.mock import patch
import pytest
from portage import os
from portage.tests import TestCase
from portage.checksum import perform_md5
-from portage.util.file_copy import copyfile
+from portage.util.file_copy import copyfile, _fastcopy
class CopyFileTestCase(TestCase):
@@ -42,25 +43,37 @@ class CopyFileSparseTestCase(TestCase):
# files too big, in case the filesystem doesn't support
# sparse files.
with open(src_path, "wb") as f:
+f.seek(2**16, os.SEEK_SET)
f.write(content)
-f.seek(2**17, 1)
-f.write(content)
-f.seek(2**18, 1)
+f.seek(2**17, os.SEEK_SET)
f.write(content)
# Test that sparse blocks are handled correctly at
-# the end of the file (involves seek and truncate).
-f.seek(2**17, 1)
+# the end of the file.
+f.truncate(2**18)
-copyfile(src_path, dest_path)
+fastcopy_success = False
+
+def mock_fastcopy(src, dst):
+nonlocal fastcopy_success
+_fastcopy(src, dst)
+fastcopy_success = True
+
+with patch("portage.util.file_copy._fastcopy", new=mock_fastcopy):
+copyfile(src_path, dest_path)
self.assertEqual(perform_md5(src_path), perform_md5(dest_path))
-# This last part of the test is expected to fail when sparse
-# copy is not implemented, so mark it xfail:
-pytest.xfail(reason="sparse copy is not implemented")
+src_stat = os.stat(src_path)
+dest_stat = os.stat(dest_path)
+
+self.assertEqual(src_stat.st_size, dest_stat.st_size)
# If sparse blocks were preserved, then both files should
# consume the same number of blocks.
-self.assertEqual(os.stat(src_path).st_blocks,
os.stat(dest_path).st_blocks)
+# This is expected to fail when sparse copy is not implemented.
+if src_stat.st_blocks != dest_stat.st_blocks:
+if fastcopy_success:
+pytest.fail(reason="sparse copy failed with _fastcopy")
+pytest.xfail(reason="sparse copy is not implemented")
finally:
shutil.rmtree(tempdir)
[gentoo-commits] proj/portage:master commit in: bin/
commit: 1223086a043483c15700dd7781f1710b05c4c86c
Author: Zac Medico gentoo org>
AuthorDate: Fri Mar 15 04:20:01 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Mar 15 04:22:02 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1223086a
bin/ebuild: Discard merge-wait from FEATURES
We don't implement merge-wait for the ebuild command, so discard
it from FEATURES. This prevents premature WORKDIR removal.
Fixes: 8eb2502bf264 ("phase-functions: prematurely delete WORKDIR if
FEATURES=merge-wait")
Signed-off-by: Zac Medico gentoo.org>
bin/ebuild | 4
1 file changed, 4 insertions(+)
diff --git a/bin/ebuild b/bin/ebuild
index 69db474c0b..043e5bc476 100755
--- a/bin/ebuild
+++ b/bin/ebuild
@@ -304,6 +304,10 @@ def main():
tmpsettings.features.discard("fail-clean")
+# We don't implement merge-wait for the ebuild command, so discard
+# it from FEATURES. This prevents premature WORKDIR removal.
+tmpsettings.features.discard("merge-wait")
+
if "merge" in pargs and "noauto" in tmpsettings.features:
print("Disabling noauto in features... merge disables it. (qmerge
doesn't)")
tmpsettings.features.discard("noauto")
[gentoo-commits] proj/portage:master commit in: bin/
commit: 8ac72ee300c1e9a55f9c1ee96438e8f12d64dd20
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 10 20:12:22 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 10 20:12:22 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8ac72ee3
install_qa_check: prematurely delete WORKDIR if FEATURES=merge-wait
Delay this until after the install qa hooks so that WORKDIR is
accessible to these hooks:
* bin/install-qa-check.d/90config-impl-decl
* gentoo/metadata/install-qa-check.d/60python-tests
Bug: https://github.com/gentoo/portage/pull/1302#issuecomment-1987303879
Fixes: 8eb2502bf264 ("phase-functions: prematurely delete WORKDIR if
FEATURES=merge-wait")
Signed-off-by: Zac Medico gentoo.org>
bin/misc-functions.sh | 7 +++
bin/phase-functions.sh | 7 ---
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index 696f2a6133..e7ca2a8a92 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -255,6 +255,13 @@ install_qa_check() {
"${PORTAGE_BIN_PATH}"/estrip --prepallstrip
fi
fi
+
+ # Prematurely delete WORKDIR in case merge-wait is enabled to
+ # decrease the space used by portage build directories until the
+ # packages are merged and cleaned.
+ if has merge-wait ${FEATURES} && ! has keepwork ${FEATURES}; then
+ rm -rf "${WORKDIR}"
+ fi
}
__dyn_instprep() {
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index 9ff5554405..d037cf6776 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -724,13 +724,6 @@ __dyn_install() {
>> DEBUGBUILD
fi
trap - SIGINT SIGQUIT
-
- # Prematurely delete WORKDIR in case merge-wait is enabled to
- # decrease the space used by portage build directories until the
- # packages are merged and cleaned.
- if has merge-wait ${FEATURES} && ! has keepwork ${FEATURES}; then
- rm -rf "${WORKDIR}"
- fi
}
__dyn_help() {
[gentoo-commits] proj/portage:master commit in: lib/_emerge/
commit: c3ed026be40b4cf857b33854e70f59f7e016e3b5
Author: Zac Medico gentoo org>
AuthorDate: Sat Mar 9 23:23:28 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Mar 9 23:24:44 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c3ed026b
_setup_repo_revisions: Skip async_aux_get for INHERITED
Since INHERITED is always available for the ebuild environment,
get it from settings.configdict["pkg"]["INHERITED"].
Fixes: d55d415a4f68 ("EbuildPhase/EbuildBinpkg: Ensure PKGDIR subdirectory
permissions")
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/EbuildPhase.py | 13 +++--
1 file changed, 3 insertions(+), 10 deletions(-)
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 73c284fe41..9e5268ff73 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -92,17 +92,10 @@ async def _setup_repo_revisions(settings):
and repo_name
and "PORTAGE_REPO_REVISIONS" not in settings.configdict["pkg"]
):
-inherits = frozenset(
-(
-await db.async_aux_get(
-settings.mycpv,
-["INHERITED"],
-myrepo=repo_name,
-)
-)[0].split()
-)
repo = db.repositories[repo_name]
-ec_dict = repo.eclass_db.get_eclass_data(inherits)
+ec_dict = repo.eclass_db.get_eclass_data(
+settings.configdict["pkg"]["INHERITED"].split()
+)
referenced_repos = {repo.name: repo}
for ec_info in ec_dict.values():
ec_repo = db.repositories.get_repo_for_location(
[gentoo-commits] proj/portage:master commit in: bin/, lib/portage/dbapi/, lib/_emerge/
commit: d55d415a4f68b0d2b0c4dca00cce2d9aee934faa
Author: Zac Medico gentoo org>
AuthorDate: Sat Mar 9 22:01:18 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Mar 9 22:49:03 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d55d415a
EbuildPhase/EbuildBinpkg: Ensure PKGDIR subdirectory permissions
Bug: https://bugs.gentoo.org/71
Signed-off-by: Zac Medico gentoo.org>
bin/misc-functions.sh| 8 ++--
lib/_emerge/EbuildBinpkg.py | 3 ++-
lib/_emerge/EbuildPhase.py | 3 +++
lib/portage/dbapi/bintree.py | 18 +-
4 files changed, 24 insertions(+), 8 deletions(-)
diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index 4ce3acbfdc..696f2a6133 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
#
# Miscellaneous shell functions that make use of the ebuild env but don't need
@@ -511,7 +511,11 @@ __dyn_package() {
export SANDBOX_ON="0"
[[ -z "${PORTAGE_BINPKG_TMPFILE}" ]] && \
die "PORTAGE_BINPKG_TMPFILE is unset"
- mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed"
+ if [[ ! -d ${PORTAGE_BINPKG_TMPFILE%/*} ]]; then
+ # Warn because we don't set PKGDIR directory permissions here.
+ ewarn "__dyn_package: Creating PORTAGE_BINPKG_TMPFILE parent
dir: ${PORTAGE_BINPKG_TMPFILE%/*}"
+ mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed"
+ fi
if [[ ! -z "${BUILD_ID}" ]]; then
echo -n "${BUILD_ID}" >
"${PORTAGE_BUILDDIR}"/build-info/BUILD_ID
diff --git a/lib/_emerge/EbuildBinpkg.py b/lib/_emerge/EbuildBinpkg.py
index 97b69ae018..26d828e4d2 100644
--- a/lib/_emerge/EbuildBinpkg.py
+++ b/lib/_emerge/EbuildBinpkg.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import io
@@ -29,6 +29,7 @@ class EbuildBinpkg(CompositeTask):
pkg_allocated_path, build_id = bintree.getname_build_id(
pkg.cpv, allocate_new=True
)
+bintree._ensure_dir(os.path.dirname(pkg_allocated_path))
self.pkg_allocated_path = pkg_allocated_path
self._binpkg_tmpfile = self.pkg_allocated_path + "." +
str(portage.getpid())
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 86a64be78e..73c284fe41 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -254,6 +254,9 @@ class EbuildPhase(CompositeTask):
)
else:
raise InvalidBinaryPackageFormat(binpkg_format)
+self.settings.mycpv._db.bintree._ensure_dir(
+os.path.dirname(self.settings["PORTAGE_BINPKG_TMPFILE"])
+)
def _async_start_exit(self, task):
task.future.cancelled() or task.future.result()
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 4ba1407cda..7bc1f60f6d 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -72,6 +72,7 @@ import traceback
import warnings
from gzip import GzipFile
from itertools import chain
+from pathlib import PurePath
from urllib.parse import urlparse
@@ -839,11 +840,18 @@ class binarytree:
return
pkgdir_gid = pkgdir_st.st_gid
pkgdir_grp_mode = 0o2070 & pkgdir_st.st_mode
-try:
-ensure_dirs(path, gid=pkgdir_gid, mode=pkgdir_grp_mode, mask=0)
-except PortageException:
-if not os.path.isdir(path):
-raise
+
+components = []
+for component in PurePath(path).relative_to(self.pkgdir).parts:
+components.append(component)
+component_path = os.path.join(self.pkgdir, *components)
+try:
+ensure_dirs(
+component_path, gid=pkgdir_gid, mode=pkgdir_grp_mode,
mask=0
+)
+except PortageException:
+if not os.path.isdir(component_path):
+raise
def _file_permissions(self, path):
try:
[gentoo-commits] proj/portage:master commit in: bin/, lib/_emerge/, lib/portage/package/ebuild/_config/
commit: eea598a20b2db5ecbe3975dc96885f529ae54c1c
Author: Zac Medico gentoo org>
AuthorDate: Sat Mar 9 21:22:35 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Mar 9 21:22:35 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=eea598a2
__dyn_install: Record REPO_REVISIONS in build-info
Record REPO_REVISIONS as a json object that maps repo name to
revision for an ebuild's source repository and any repositories
that eclasses were inherited from:
$ cat /var/tmp/portage/sys-apps/portage-3.0.63/build-info/REPO_REVISIONS
{"gentoo": "34875e30e73e33d3597d1101cdf97dc22729b268"}
Ultimately the intention is to expose this information in binhost
metadata so that clients can select consistent revisions of source
repositories.
Bug: https://bugs.gentoo.org/924772
Signed-off-by: Zac Medico gentoo.org>
bin/phase-functions.sh | 1 +
lib/_emerge/EbuildPhase.py | 46 ++
.../package/ebuild/_config/special_env_vars.py | 1 +
3 files changed, 48 insertions(+)
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index ab130a3be5..9ff5554405 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -719,6 +719,7 @@ __dyn_install() {
cp "${EBUILD}" "${PF}.ebuild"
[[ -n "${PORTAGE_REPO_NAME}" ]] && echo "${PORTAGE_REPO_NAME}" >
repository
+ [[ -n ${PORTAGE_REPO_REVISIONS} ]] && echo "${PORTAGE_REPO_REVISIONS}"
> REPO_REVISIONS
if has nostrip ${FEATURES} ${PORTAGE_RESTRICT} || has strip
${PORTAGE_RESTRICT}; then
>> DEBUGBUILD
fi
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index b472803438..86a64be78e 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -4,6 +4,7 @@
import functools
import gzip
import io
+import json
import sys
import tempfile
@@ -83,6 +84,50 @@ async def _setup_locale(settings):
raise AssertionError("C locale did not pass the test!")
+async def _setup_repo_revisions(settings):
+repo_name = settings.configdict["pkg"].get("PORTAGE_REPO_NAME")
+db = getattr(settings.mycpv, "_db", None)
+if (
+isinstance(db, portage.portdbapi)
+and repo_name
+and "PORTAGE_REPO_REVISIONS" not in settings.configdict["pkg"]
+):
+inherits = frozenset(
+(
+await db.async_aux_get(
+settings.mycpv,
+["INHERITED"],
+myrepo=repo_name,
+)
+)[0].split()
+)
+repo = db.repositories[repo_name]
+ec_dict = repo.eclass_db.get_eclass_data(inherits)
+referenced_repos = {repo.name: repo}
+for ec_info in ec_dict.values():
+ec_repo = db.repositories.get_repo_for_location(
+os.path.dirname(os.path.dirname(ec_info.location))
+)
+referenced_repos.setdefault(ec_repo.name, ec_repo)
+repo_revisions = {}
+for repo_ref in referenced_repos.values():
+if repo_ref.sync_type:
+sync =
portage.sync.module_controller.get_class(repo_ref.sync_type)()
+try:
+# TODO: Wait for subprocesses asynchronously here.
+status, repo_revision = sync.retrieve_head(
+options={"repo": repo_ref}
+)
+except NotImplementedError:
+pass
+else:
+if status == os.EX_OK:
+repo_revisions[repo_ref.name] = repo_revision.strip()
+settings.configdict["pkg"]["PORTAGE_REPO_REVISIONS"] = json.dumps(
+repo_revisions, ensure_ascii=False, sort_keys=True
+)
+
+
class EbuildPhase(CompositeTask):
__slots__ = ("actionmap", "fd_pipes", "phase", "settings") +
("_ebuild_lock",)
@@ -120,6 +165,7 @@ class EbuildPhase(CompositeTask):
async def _async_start(self):
await _setup_locale(self.settings)
+await _setup_repo_revisions(self.settings)
need_builddir = self.phase not in
EbuildProcess._phases_without_builddir
diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py
b/lib/portage/package/ebuild/_config/special_env_vars.py
index 6020029e35..1a66192c96 100644
--- a/lib/portage/package/ebuild/_config/special_env_vars.py
+++ b/lib/portage/package/ebuild/_config/special_env_vars.py
@@ -165,6 +165,7 @@ environ_whitelist = frozenset(
"PORTAGE_PYTHON",
"PORTAGE_PYTHONPATH",
"PORTAGE_QUIET",
+"PORTAGE_REPO_REVISIONS",
"PORTAGE_REPO_NAME",
"PORTAGE_REPOSITORIES",
"PORTAGE_RESTRICT",
[gentoo-commits] proj/portage:master commit in: bin/
commit: 8eb2502bf264d54dfbf180222f162f7a593b947d
Author: Florian Schmaus gentoo org>
AuthorDate: Wed Mar 6 20:52:08 2024 +
Commit: Sam James gentoo org>
CommitDate: Sat Mar 9 18:46:00 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8eb2502b
phase-functions: prematurely delete WORKDIR if FEATURES=merge-wait
Using the merge-wait feature together with many parallel emerge jobs
potentially leads to a high disk space usage due to the created
portage build directories waiting for their packages to be merged into
the live filesystem prior portage will clean them. This can easily
lead to out-of-space errors.
Prematurely deleting WORKDIR at the end of src_install() helps
reducing the temporarily used disk space. All that portage need to
merge a package into the live filesystem are the files in the image/
directory (D). (At least in theory)
Signed-off-by: Florian Schmaus gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1302
Signed-off-by: Sam James gentoo.org>
bin/phase-functions.sh | 7 +++
1 file changed, 7 insertions(+)
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index ebcf5f242a..ab130a3be5 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -723,6 +723,13 @@ __dyn_install() {
>> DEBUGBUILD
fi
trap - SIGINT SIGQUIT
+
+ # Prematurely delete WORKDIR in case merge-wait is enabled to
+ # decrease the space used by portage build directories until the
+ # packages are merged and cleaned.
+ if has merge-wait ${FEATURES} && ! has keepwork ${FEATURES}; then
+ rm -rf "${WORKDIR}"
+ fi
}
__dyn_help() {
[gentoo-commits] proj/portage:master commit in: cnf/, /
commit: 32e4a588cfd9548a28c74476d586a6b66b15bace
Author: Zac Medico gentoo org>
AuthorDate: Fri Mar 8 05:04:10 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Mar 8 05:04:10 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=32e4a588
meson.build: Install dist-info METADATA
Install dist-info METADATA for pip to resolve dependencies:
# pip freeze | grep portage
portage==3.0.63
Suggested-by: Eli Schwartz gmail.com>
Bug: https://bugs.gentoo.org/920330
Signed-off-by: Zac Medico gentoo.org>
cnf/METADATA | 3 +++
meson.build | 14 ++
2 files changed, 17 insertions(+)
diff --git a/cnf/METADATA b/cnf/METADATA
new file mode 100644
index 00..57060b0856
--- /dev/null
+++ b/cnf/METADATA
@@ -0,0 +1,3 @@
+Metadata-Version: 2.1
+Name: portage
+Version: @VERSION@
diff --git a/meson.build b/meson.build
index e826f6ffad..b06e1f4881 100644
--- a/meson.build
+++ b/meson.build
@@ -97,6 +97,20 @@ if native_extensions
subdir('src')
endif
+if system_wide
+ METADATA = configure_file(
+ input : 'cnf/METADATA',
+ output : 'METADATA',
+ configuration : conf_data
+ )
+ install_data(
+ [
+ METADATA
+ ],
+ install_dir : py.get_install_dir() /
'portage-@0@.dist-info'.format(conf_data.get('VERSION'))
+ )
+endif
+
test(
'pytest',
py,
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: b7e89f866a9a1d73ab72670d74e2292b05893849
Author: Mike Gilbert gentoo org>
AuthorDate: Wed Mar 6 04:01:27 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Wed Mar 6 18:19:31 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b7e89f86
util: set a timeout for urlopen calls
A hung urlopen call can cause emerge to produce no output when fetching
binhost data.
Bug: https://bugs.gentoo.org/926221
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/_urlopen.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/portage/util/_urlopen.py b/lib/portage/util/_urlopen.py
index 22f0e08df0..d451a94a89 100644
--- a/lib/portage/util/_urlopen.py
+++ b/lib/portage/util/_urlopen.py
@@ -26,10 +26,10 @@ def have_pep_476():
return hasattr(__import__("ssl"), "_create_unverified_context")
-def urlopen(url, if_modified_since=None, headers={}, proxies=None):
+def urlopen(url, timeout=10, if_modified_since=None, headers={}, proxies=None):
parse_result = urllib_parse.urlparse(url)
if parse_result.scheme not in ("http", "https"):
-return _urlopen(url)
+return _urlopen(url, timeout=timeout)
netloc = parse_result.netloc.rpartition("@")[-1]
url = urllib_parse.urlunparse(
@@ -59,7 +59,7 @@ def urlopen(url, if_modified_since=None, headers={},
proxies=None):
handlers.append(urllib_request.ProxyHandler(proxies))
opener = urllib_request.build_opener(*handlers)
-hdl = opener.open(request)
+hdl = opener.open(request, timeout=timeout)
if hdl.headers.get("last-modified", ""):
try:
add_header = hdl.headers.add_header
[gentoo-commits] proj/portage:master commit in: /
commit: 3a1e39943ea43ec22b6cf7cfdf2673ed30f97d56 Author: Mike Gilbert gentoo org> AuthorDate: Thu Mar 7 20:42:16 2024 + Commit: Mike Gilbert gentoo org> CommitDate: Thu Mar 7 20:42:16 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=3a1e3994 Update NEWS Signed-off-by: Mike Gilbert gentoo.org> NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 5da71f776e..ef59577f66 100644 --- a/NEWS +++ b/NEWS @@ -12,6 +12,8 @@ portage-3.0.64 (UNRELEASED) Bug fixes: * ctypes: Add workaround for loading libc on musl. +* util: set a timeout for urlopen calls (bug #926221). + portage-3.0.63 (2024-02-25) --
[gentoo-commits] proj/portage:master commit in: lib/_emerge/, lib/portage/util/_async/
commit: 6f9a10d38259dd61b948837e193b047464791845
Author: Zac Medico gentoo org>
AuthorDate: Mon Mar 4 05:31:08 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Mar 4 05:33:44 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6f9a10d3
SpawnProcess: Optimize away null input for create_pipe=False
When create_pipe=False support was added in commit e8b31c86eaed,
a null input file descriptor was used for PipeLogger and BuildLogger
instances. Optimize this away, eliminating the unnecessary loggers.
Fixes: e8b31c86eaed ("ForkProcess: Prevent redundant pipe and set_term_size
recursion")
Bug: https://bugs.gentoo.org/916566
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/SpawnProcess.py| 51 +-
lib/portage/util/_async/ForkProcess.py | 7 ++---
2 files changed, 28 insertions(+), 30 deletions(-)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 9fc12c42e5..513a7b2fe4 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -79,10 +79,7 @@ class SpawnProcess(SubProcess):
# SpawnProcess will have created a pipe earlier, so it
# would be redundant to do it here (it could also trigger
# spawn recursion via set_term_size as in bug 923750).
-# Use /dev/null for master_fd, triggering early return
-# of _main, followed by _async_waitpid.
-# TODO: Optimize away the need for master_fd here.
-master_fd = os.open(os.devnull, os.O_RDONLY)
+master_fd = None
slave_fd = None
can_log = False
@@ -166,23 +163,27 @@ class SpawnProcess(SubProcess):
self._registered = True
def _start_main_task(self, pr, log_file_path=None, stdout_fd=None):
-build_logger = BuildLogger(
-env=self.env,
-log_path=log_file_path,
-log_filter_file=self.log_filter_file,
-scheduler=self.scheduler,
-)
-build_logger.start()
-
-pipe_logger = PipeLogger(
-background=self.background,
-scheduler=self.scheduler,
-input_fd=pr,
-log_file_path=build_logger.stdin,
-stdout_fd=stdout_fd,
-)
-
-pipe_logger.start()
+if pr is None:
+build_logger = None
+pipe_logger = None
+else:
+build_logger = BuildLogger(
+env=self.env,
+log_path=log_file_path,
+log_filter_file=self.log_filter_file,
+scheduler=self.scheduler,
+)
+build_logger.start()
+
+pipe_logger = PipeLogger(
+background=self.background,
+scheduler=self.scheduler,
+input_fd=pr,
+log_file_path=build_logger.stdin,
+stdout_fd=stdout_fd,
+)
+
+pipe_logger.start()
self._main_task_cancel = functools.partial(
self._main_cancel, build_logger, pipe_logger
@@ -198,18 +199,18 @@ class SpawnProcess(SubProcess):
await self._pty_ready
self._pty_ready = None
try:
-if pipe_logger.poll() is None:
+if pipe_logger is not None and pipe_logger.poll() is None:
await pipe_logger.async_wait()
-if build_logger.poll() is None:
+if build_logger is not None and build_logger.poll() is None:
await build_logger.async_wait()
except asyncio.CancelledError:
self._main_cancel(build_logger, pipe_logger)
raise
def _main_cancel(self, build_logger, pipe_logger):
-if pipe_logger.poll() is None:
+if pipe_logger is not None and pipe_logger.poll() is None:
pipe_logger.cancel()
-if build_logger.poll() is None:
+if build_logger is not None and build_logger.poll() is None:
build_logger.cancel()
def _main_exit(self, main_task):
diff --git a/lib/portage/util/_async/ForkProcess.py
b/lib/portage/util/_async/ForkProcess.py
index ebcbd94107..e6cfdefb88 100644
--- a/lib/portage/util/_async/ForkProcess.py
+++ b/lib/portage/util/_async/ForkProcess.py
@@ -91,11 +91,8 @@ class ForkProcess(SpawnProcess):
# When called via process.spawn, SpawnProcess
# will have created a pipe earlier, so it would be
# redundant to do it here (it could also trigger spawn
-# recursion via set_term_size as in bug 923750). Use
-# /dev/null for master_fd, triggering early return
-# of _main, followed by _async_waitpid.
-# TODO: Optimize away the need for master_fd here.
-master_fd = os.open(os.devnull, os.O_RDONLY)
+# recursion via
[gentoo-commits] proj/portage:master commit in: lib/portage/util/_eventloop/, lib/portage/util/, lib/portage/tests/util/
commit: 11c65496bd951c3b7778a3c1ea240e347b1f4c38 Author: Zac Medico gentoo org> AuthorDate: Sun Mar 3 21:06:13 2024 + Commit: Zac Medico gentoo org> CommitDate: Sun Mar 3 21:10:49 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=11c65496 socks5: Use run_coroutine_exitfuncs() Since commit c3ebdbb42e72 the atexit_register(proxy.stop) call in the get_socks5_proxy function can accept a coroutine function to execute in run_coroutine_exitfuncs(), so convert the ProxyManager stop method to a coroutine and use run_coroutine_exitfuncs(). Bug: https://bugs.gentoo.org/925240 Signed-off-by: Zac Medico gentoo.org> lib/portage/tests/util/test_socks5.py | 5 ++- lib/portage/util/_eventloop/asyncio_event_loop.py | 12 --- lib/portage/util/socks5.py| 39 +++ 3 files changed, 7 insertions(+), 49 deletions(-) diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py index 4a6d08169d..a8cd0c46c4 100644 --- a/lib/portage/tests/util/test_socks5.py +++ b/lib/portage/tests/util/test_socks5.py @@ -216,10 +216,9 @@ class Socks5ServerTestCase(TestCase): self.assertEqual(result, content) finally: try: -# Also run_exitfuncs to test atexit hook cleanup. -await socks5.proxy.stop() +# Also run_coroutine_exitfuncs to test atexit hook cleanup. self.assertNotEqual(portage.process._exithandlers, []) -portage.process.run_exitfuncs() +await portage.process.run_coroutine_exitfuncs() self.assertEqual(portage.process._exithandlers, []) finally: portage.process._exithandlers = previous_exithandlers diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index a598b1b516..821cc7f102 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -15,7 +15,6 @@ except ImportError: PidfdChildWatcher = None import portage -from portage.util import socks5 class AsyncioEventLoop(_AbstractEventLoop): @@ -75,17 +74,6 @@ class AsyncioEventLoop(_AbstractEventLoop): self._closing = False async def _close_main(self): -# Even though this has an exit hook, invoke it here so that -# we can properly wait for it and avoid messages like this: -# [ERROR] Task was destroyed but it is pending! -if socks5.proxy.is_running(): -# TODO: Convert socks5.proxy.stop() to a regular coroutine -# function so that it doesn't need to be wrapped like this. -async def stop_socks5_proxy(): -await socks5.proxy.stop() - -portage.process.atexit_register(stop_socks5_proxy) - await portage.process.run_coroutine_exitfuncs() portage.process.run_exitfuncs() diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index f8fcdf9fca..c32ba77674 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -6,15 +6,8 @@ import asyncio import errno import os import socket -from typing import Union import portage - -portage.proxy.lazyimport.lazyimport( -globals(), -"portage.util._eventloop.global_event_loop:global_event_loop", -) - import portage.data from portage import _python_interpreter from portage.data import portage_gid, portage_uid, userpriv_groups @@ -65,41 +58,19 @@ class ProxyManager: **spawn_kwargs, ) -def stop(self) -> Union[None, asyncio.Future]: +async def stop(self): """ -Stop the SOCKSv5 server. - -If there is a running asyncio event loop then asyncio.Future is -returned which should be used to wait for the server process -to exit. +Stop the SOCKSv5 server. This method is a coroutine. """ -future = None -try: -loop = asyncio.get_running_loop() -except RuntimeError: -loop = None if self._proc is not None: self._proc.terminate() -if loop is None: -# In this case spawn internals would have used -# portage's global loop when attaching a waiter to -# self._proc, so we are obligated to use that. -global_event_loop().run_until_complete(self._proc.wait()) -else: -if self._proc_waiter is None: -self._proc_waiter = asyncio.ensure_future( -self._proc.wait(), loop=loop -) -future = asyncio.shield(self._proc_waiter) - -if loop is not None and future is None: -future = loop.create_future() -future.set_result(None) +if self._proc_waiter is None: +self._proc_waiter
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/, ...
commit: 6ce2be8d454f95c508d9f547d13487f9de863bdd
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 3 19:53:11 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 3 19:53:11 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6ce2be8d
_validate_deps: Discard configdict["pkg"]["USE"]
Since configdict["pkg"]["USE"] may contain package.use settings
from config.setcpv, it is inappropriate to use here (bug 675748),
so discard it. This is only an issue because configdict["pkg"] is
a sub-optimal place to extract metadata from. This issue does not
necessarily indicate a flaw in the Package constructor, since
passing in precalculated USE can be valid for things like
autounmask USE changes.
Bug: https://bugs.gentoo.org/675748
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/doebuild.py | 12
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 19 ++-
lib/portage/tests/resolver/ResolverPlayground.py | 1 +
3 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index 942fa90101..6691db4e97 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -1813,6 +1813,14 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi):
invalid_dep_exempt_phases = {"clean", "cleanrm", "help", "prerm", "postrm"}
all_keys = set(Package.metadata_keys)
all_keys.add("SRC_URI")
+# Since configdict["pkg"]["USE"] may contain package.use settings
+# from config.setcpv, it is inappropriate to use here (bug 675748),
+# so discard it. This is only an issue because configdict["pkg"] is
+# a sub-optimal place to extract metadata from. This issue does not
+# necessarily indicate a flaw in the Package constructor, since
+# passing in precalculated USE can be valid for things like
+# autounmask USE changes.
+all_keys.discard("USE")
all_keys = tuple(all_keys)
metadata = mysettings.configdict["pkg"]
if all(k in metadata for k in ("PORTAGE_REPO_NAME", "SRC_URI")):
@@ -1838,6 +1846,10 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi):
root_config = RootConfig(mysettings, {"porttree": FakeTree(mydbapi)}, None)
+# A USE calculation from setcpv should always be available here because
+# mysettings.mycpv is not None, so use it to prevent redundant setcpv
calls.
+metadata["USE"] = mysettings["PORTAGE_USE"]
+
pkg = Package(
built=False,
cpv=mysettings.mycpv,
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index b38605bb90..445fcf6c4e 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -51,10 +51,23 @@ class DoebuildFdPipesTestCase(TestCase):
ebuilds = {
"app-misct/foo-1": {
"EAPI": "8",
+"IUSE": "+foo +bar",
+"REQUIRED_USE": "|| ( foo bar )",
"MISC_CONTENT": ebuild_body,
}
}
+# Populate configdict["pkg"]["USE"] with something arbitrary in order
+# to try and trigger bug 675748 in doebuild _validate_deps.
+arbitrary_package_use = "baz"
+
+user_config = {
+# In order to trigger bug 675748, package.env must be non-empty,
+# but the referenced env file can be empty.
+"package.env": (f"app-misct/foo {os.devnull}",),
+"package.use": (f"app-misct/foo {arbitrary_package_use}",),
+}
+
# Override things that may be unavailable, or may have portability
# issues when running tests in exotic environments.
# prepstrip - bug #447810 (bash read builtin EINTR problem)
@@ -63,7 +76,7 @@ class DoebuildFdPipesTestCase(TestCase):
self.assertEqual(true_binary is None, False, "true command not found")
dev_null = open(os.devnull, "wb")
-playground = ResolverPlayground(ebuilds=ebuilds)
+playground = ResolverPlayground(ebuilds=ebuilds,
user_config=user_config)
try:
QueryCommand._db = playground.trees
root_config = playground.trees[playground.eroot]["root_config"]
@@ -106,6 +119,10 @@ class DoebuildFdPipesTestCase(TestCase):
)
settings.setcpv(pkg)
+# Demonstrate that settings.configdict["pkg"]["USE"] contains our
arbitrary
+# package.use setting in order to trigger bug 675748.
+self.assertEqual(settings.configdict["pkg"]["USE"],
arbitrary_package_use)
+
# Try to trigger the config.environ() split_LC_ALL assertion for
bug 925863.
settings["LC_ALL"] = "C"
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
b/lib/portage/tests/resolver/ResolverPlayground.py
index c0455415a1..f52a98f8db
[gentoo-commits] proj/portage:master commit in: lib/portage/elog/, lib/portage/, lib/portage/util/_eventloop/
commit: c3ebdbb42e72335ca65335c855a82b99537c7606
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 3 06:30:50 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 3 06:30:50 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=c3ebdbb4
elog/mod_custom: Spawn processes in background
Since elog_process is typically called while the event loop is
running, hold references to spawned processes and wait for them
asynchronously, ultimately waiting for them if necessary when
the AsyncioEventLoop _close_main method calls _async_finalize
via portage.process.run_coroutine_exitfuncs().
ConfigProtectTestCase is useful for exercising this code, and
this little make.globals patch can be used to test failure
during finalize with this error message:
!!! PORTAGE_ELOG_COMMAND failed with exitcode 1
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -144 +144,2 @@ PORTAGE_ELOG_CLASSES="log warn error"
-PORTAGE_ELOG_SYSTEM="save_summary:log,warn,error,qa echo"
+PORTAGE_ELOG_SYSTEM="save_summary:log,warn,error,qa echo custom"
+PORTAGE_ELOG_COMMAND="/bin/false"
Bug: https://bugs.gentoo.org/925907
Signed-off-by: Zac Medico gentoo.org>
lib/portage/elog/mod_custom.py| 73 +--
lib/portage/process.py| 29 +
lib/portage/util/_eventloop/asyncio_event_loop.py | 8 ++-
3 files changed, 105 insertions(+), 5 deletions(-)
diff --git a/lib/portage/elog/mod_custom.py b/lib/portage/elog/mod_custom.py
index e0ae77e100..a3e199bcb7 100644
--- a/lib/portage/elog/mod_custom.py
+++ b/lib/portage/elog/mod_custom.py
@@ -1,10 +1,33 @@
# elog/mod_custom.py - elog dispatch module
-# Copyright 2006-2020 Gentoo Authors
+# Copyright 2006-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+import types
+
+import portage
import portage.elog.mod_save
import portage.exception
import portage.process
+from portage.util.futures import asyncio
+
+# Since elog_process is typically called while the event loop is
+# running, hold references to spawned processes and wait for them
+# asynchronously, ultimately waiting for them if necessary when
+# the AsyncioEventLoop _close_main method calls _async_finalize
+# via portage.process.run_coroutine_exitfuncs().
+_proc_refs = None
+
+
+def _get_procs() -> list[tuple[portage.process.MultiprocessingProcess,
asyncio.Future]]:
+"""
+Return list of (proc, asyncio.ensure_future(proc.wait())) which is not
+inherited from the parent after fork.
+"""
+global _proc_refs
+if _proc_refs is None or _proc_refs.pid != portage.getpid():
+_proc_refs = types.SimpleNamespace(pid=portage.getpid(), procs=[])
+portage.process.atexit_register(_async_finalize)
+return _proc_refs.procs
def process(mysettings, key, logentries, fulltext):
@@ -18,8 +41,50 @@ def process(mysettings, key, logentries, fulltext):
mylogcmd = mysettings["PORTAGE_ELOG_COMMAND"]
mylogcmd = mylogcmd.replace("${LOGFILE}", elogfilename)
mylogcmd = mylogcmd.replace("${PACKAGE}", key)
-retval = portage.process.spawn_bash(mylogcmd)
-if retval != 0:
+loop = asyncio.get_event_loop()
+proc = portage.process.spawn_bash(mylogcmd, returnproc=True)
+procs = _get_procs()
+procs.append((proc, asyncio.ensure_future(proc.wait(), loop=loop)))
+for index, (proc, waiter) in reversed(list(enumerate(procs))):
+if not waiter.done():
+continue
+del procs[index]
+if waiter.result() != 0:
+raise portage.exception.PortageException(
+f"!!! PORTAGE_ELOG_COMMAND failed with exitcode
{waiter.result()}"
+)
+
+
+async def _async_finalize():
+"""
+Async finalize is preferred, since we can wait for process exit status.
+"""
+procs = _get_procs()
+while procs:
+proc, waiter = procs.pop()
+if (await waiter) != 0:
+raise portage.exception.PortageException(
+f"!!! PORTAGE_ELOG_COMMAND failed with exitcode
{waiter.result()}"
+)
+
+
+def finalize():
+"""
+NOTE: This raises PortageException if there are any processes
+still running, so it's better to use _async_finalize instead
+(invoked via portage.process.run_coroutine_exitfuncs() in
+the AsyncioEventLoop _close_main method).
+"""
+procs = _get_procs()
+while procs:
+proc, waiter = procs.pop()
+if not waiter.done():
+waiter.cancel()
+proc.terminate()
+raise portage.exception.PortageException(
+f"!!! PORTAGE_ELOG_COMMAND was killed after it was found
running in the background (pid {proc.pid})"
+)
+elif waiter.result() != 0:
raise portage.exception.PortageException(
-"!!! PORTAGE_ELOG_COMMAND failed with exitcode %d" % retval
+
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 62ee9bf8c680b2a18713da5bd453e3a771257409
Author: Zac Medico gentoo org>
AuthorDate: Sat Mar 2 22:48:54 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sat Mar 2 22:50:54 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=62ee9bf8
binarytree._populate_remote: Fix UnboundLocalError for binpkg-request-signature
If an InvalidBinaryPackageFormat exception was raised from
get_binpkg_format for binpkg-request-signature then it
triggered an UnboundLocalError here.
Fixes: 445f10f4214c ("Use binpkg extensions and header to get format")
Bug: https://bugs.gentoo.org/926048
Signed-off-by: Zac Medico gentoo.org>
lib/portage/dbapi/bintree.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index f4251b47d6..4ba1407cda 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1624,7 +1624,7 @@ class binarytree:
binpkg_format = get_binpkg_format(
d.get("PATH"), remote=True
)
-except InvalidBinaryPackageFormat:
+except InvalidBinaryPackageFormat as e:
writemsg(
colorize(
"WARN",
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/
commit: fe510e099bc9a8055c3ee50fced47fc3dc7ba166
Author: Zac Medico gentoo org>
AuthorDate: Fri Mar 1 17:09:56 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Mar 1 18:08:51 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fe510e09
doebuild: Call _setup_locale
Call _setup_locale in order to prevent an AssertionError from
config.environ() for the config phase (or any other phase for
that matter).
For returnproc or returnpid assume that the event loop is running
so we can't run the event loop to call _setup_locale in this case
and we have to assume the caller took care of it (otherwise
config.environ() will raise AssertionError).
Update DoebuildFdPipesTestCase to use EAPI 8 and test the
pkg_config function with an ebuild located in /var/db/pkg just
like emerge --config does. Set LC_ALL=C just before doebuild
calls in order to try and trigger the config.environ()
split_LC_ALL assertion.
Bug: https://bugs.gentoo.org/925863
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/doebuild.py | 8 +++
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 77 --
2 files changed, 52 insertions(+), 33 deletions(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index bc51fdff2d..942fa90101 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -43,6 +43,7 @@ portage.proxy.lazyimport.lazyimport(
"portage.util._async.SchedulerInterface:SchedulerInterface",
"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.util.ExtractKernelVersion:ExtractKernelVersion",
+"_emerge.EbuildPhase:_setup_locale",
)
from portage import (
@@ -1034,6 +1035,13 @@ def doebuild(
myebuild, mydo, myroot, mysettings, debug, use_cache, mydbapi
)
+# For returnproc or returnpid assume that the event loop is running
+# so we can't run the event loop to call _setup_locale in this case
+# and we have to assume the caller took care of it (otherwise
+# config.environ() will raise AssertionError).
+if not (returnproc or returnpid):
+asyncio.run(_setup_locale(mysettings))
+
if mydo in clean_phases:
builddir_lock = None
if not returnpid and "PORTAGE_BUILDDIR_LOCKED" not in mysettings:
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index 678486ed16..b38605bb90 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Gentoo Authors
+# Copyright 2013-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import multiprocessing
@@ -27,20 +27,22 @@ class DoebuildFdPipesTestCase(TestCase):
output_fd = self.output_fd
ebuild_body = ["S=${WORKDIR}"]
-for phase_func in (
-"pkg_info",
-"pkg_nofetch",
-"pkg_pretend",
-"pkg_setup",
-"src_unpack",
-"src_prepare",
-"src_configure",
-"src_compile",
-"src_test",
-"src_install",
+for phase_func, default in (
+("pkg_info", False),
+("pkg_nofetch", False),
+("pkg_pretend", False),
+("pkg_setup", False),
+("pkg_config", False),
+("src_unpack", False),
+("src_prepare", True),
+("src_configure", False),
+("src_compile", False),
+("src_test", False),
+("src_install", False),
):
ebuild_body.append(
-("%s() { echo ${EBUILD_PHASE}" " 1>&%s; }") % (phase_func,
output_fd)
+("%s() { %secho ${EBUILD_PHASE}" " 1>&%s; }")
+% (phase_func, "default; " if default else "", output_fd)
)
ebuild_body.append("")
@@ -48,7 +50,7 @@ class DoebuildFdPipesTestCase(TestCase):
ebuilds = {
"app-misct/foo-1": {
-"EAPI": "5",
+"EAPI": "8",
"MISC_CONTENT": ebuild_body,
}
}
@@ -103,24 +105,33 @@ class DoebuildFdPipesTestCase(TestCase):
type_name="ebuild",
)
settings.setcpv(pkg)
-ebuildpath = portdb.findname(cpv)
-self.assertNotEqual(ebuildpath, None)
-
-for phase in (
-"info",
-"nofetch",
-"pretend",
-"setup",
-"unpack",
-"prepare",
-"configure",
-"compile",
-"test",
-"install",
-"qmerge",
-"clean",
-"merge",
+
+# Try to trigger the
[gentoo-commits] proj/portage:master commit in: lib/portage/util/
commit: d8089d1af39c80e1edfb1669ae92fef7ab30e08a
Author: Mike Gilbert gentoo org>
AuthorDate: Fri Mar 1 15:49:26 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Fri Mar 1 16:19:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=d8089d1a
Improve whitespace handling when parsing /proc/self/mountinfo
Only break lines on "\n" (line feed).
Only split lines on " " (space).
Bug: https://bugs.gentoo.org/925888
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/writeable_check.py | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/lib/portage/util/writeable_check.py
b/lib/portage/util/writeable_check.py
index 3427315cf7..ad1d9edff0 100644
--- a/lib/portage/util/writeable_check.py
+++ b/lib/portage/util/writeable_check.py
@@ -47,6 +47,7 @@ def linux_ro_checker(dir_list):
"/proc/self/mountinfo",
encoding=_encodings["content"],
errors="replace",
+newline="\n",
) as f:
for line in f:
# we're interested in dir and both attr fields which always
@@ -58,7 +59,7 @@ def linux_ro_checker(dir_list):
# to the left of the ' - ', after the attr's, so split it there
mount = line.split(" - ", 1)
try:
-_dir, attr1 = mount[0].split()[4:6]
+_dir, attr1 = mount[0].split(" ")[4:6]
except ValueError:
# If it raises ValueError we can simply ignore the line.
invalids.append(line)
@@ -68,10 +69,10 @@ def linux_ro_checker(dir_list):
# for example: 16 1 0:16 / /root rw,noatime - lxfs rw
if len(mount) > 1:
try:
-attr2 = mount[1].split()[2]
+attr2 = mount[1].split(" ")[2]
except IndexError:
try:
-attr2 = mount[1].split()[1]
+attr2 = mount[1].split(" ")[1]
except IndexError:
invalids.append(line)
continue
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, src/
commit: a860484b6e8c8d107255f2105796ae7f58812e9f
Author: Mike Gilbert gentoo org>
AuthorDate: Thu Feb 29 19:38:20 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Thu Feb 29 19:38:20 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a860484b
Drop portage.util.libc extension module
This was originally added as a workaround for musl, where
ctypes.util.find_library fails.
Instead, we now try to load "libc.so" as a fallback and can just rely on
ctypes to call tolower() and toupper().
Signed-off-by: Mike Gilbert gentoo.org>
lib/portage/util/locale.py | 9 +++
src/meson.build| 9 ---
src/portage_util_libc.c| 67 --
3 files changed, 3 insertions(+), 82 deletions(-)
diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py
index d0edeb4afe..f45d761760 100644
--- a/lib/portage/util/locale.py
+++ b/lib/portage/util/locale.py
@@ -43,12 +43,9 @@ def _check_locale(silent):
"""
The inner locale check function.
"""
-try:
-from portage.util import libc
-except ImportError:
-(libc, _) = load_libc()
-if libc is None:
-return None
+(libc, _) = load_libc()
+if libc is None:
+return None
lc = list(range(ord("a"), ord("z") + 1))
uc = list(range(ord("A"), ord("Z") + 1))
diff --git a/src/meson.build b/src/meson.build
index cbc7aa611c..6a36724ceb 100644
--- a/src/meson.build
+++ b/src/meson.build
@@ -2,14 +2,6 @@
# and for development. Meson does not allow you to build in-place and Python
# cannot create a single namespace from two identically-named paths.
-libc_ext = py.extension_module(
-'libc',
-'portage_util_libc.c',
-dependencies : py.dependency(),
-subdir : 'portage' / 'util',
-install : true
-)
-
whirlpool_ext = py.extension_module(
'_whirlpool',
'portage_util__whirlpool.c',
@@ -21,7 +13,6 @@ whirlpool_ext = py.extension_module(
run_command(
[
'ln', '-srnf',
-libc_ext.full_path(),
whirlpool_ext.full_path(),
meson.project_source_root() / 'lib' / 'portage' / 'util/'
],
diff --git a/src/portage_util_libc.c b/src/portage_util_libc.c
deleted file mode 100644
index 12b2440c72..00
--- a/src/portage_util_libc.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/* Copyright 2005-2020 Gentoo Authors
- * Distributed under the terms of the GNU General Public License v2
- */
-
-#include
-#include
-#include
-
-static PyObject * _libc_tolower(PyObject *, PyObject *);
-static PyObject * _libc_toupper(PyObject *, PyObject *);
-
-static PyMethodDef LibcMethods[] = {
- {
- .ml_name = "tolower",
- .ml_meth = _libc_tolower,
- .ml_flags = METH_VARARGS,
- .ml_doc = "Convert to lower case using system locale."
-
- },
- {
- .ml_name = "toupper",
- .ml_meth = _libc_toupper,
- .ml_flags = METH_VARARGS,
- .ml_doc = "Convert to upper case using system locale."
- },
- {NULL, NULL, 0, NULL}
-};
-
-static struct PyModuleDef moduledef = {
- PyModuleDef_HEAD_INIT,
- .m_name = "libc",
- .m_doc = "Module for converting case using the system locale",
- .m_size = -1,
- .m_methods = LibcMethods,
-};
-
-PyMODINIT_FUNC
-PyInit_libc(void)
-{
- PyObject *m;
- m = PyModule_Create();
- return m;
-}
-
-
-static PyObject *
-_libc_tolower(PyObject *self, PyObject *args)
-{
- int c;
-
- if (!PyArg_ParseTuple(args, "i", ))
- return NULL;
-
- return Py_BuildValue("i", tolower(c));
-}
-
-
-static PyObject *
-_libc_toupper(PyObject *self, PyObject *args)
-{
- int c;
-
- if (!PyArg_ParseTuple(args, "i", ))
- return NULL;
-
- return Py_BuildValue("i", toupper(c));
-}
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/util/futures/, lib/portage/util/futures/, ...
commit: b2d8226af4589d95f44d20d441056f645a523039
Author: Zac Medico gentoo org>
AuthorDate: Thu Feb 29 04:26:02 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Thu Feb 29 04:37:21 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b2d8226a
Delete compat_coroutine module
The compat_coroutine module has been unused since the migration
to PEP 492 async and await syntax in 2021, which began in commit
b3b9acc13c43 and was completed in commit bcda30d0a6fa.
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/util/futures/meson.build | 1 -
.../tests/util/futures/test_compat_coroutine.py| 210 -
lib/portage/util/futures/_asyncio/__init__.py | 15 +-
lib/portage/util/futures/compat_coroutine.py | 142 --
lib/portage/util/futures/meson.build | 1 -
5 files changed, 1 insertion(+), 368 deletions(-)
diff --git a/lib/portage/tests/util/futures/meson.build
b/lib/portage/tests/util/futures/meson.build
index 877acc27cd..cb78314844 100644
--- a/lib/portage/tests/util/futures/meson.build
+++ b/lib/portage/tests/util/futures/meson.build
@@ -1,6 +1,5 @@
py.install_sources(
[
-'test_compat_coroutine.py',
'test_done_callback.py',
'test_done_callback_after_exit.py',
'test_iter_completed.py',
diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py
b/lib/portage/tests/util/futures/test_compat_coroutine.py
deleted file mode 100644
index b25708886c..00
--- a/lib/portage/tests/util/futures/test_compat_coroutine.py
+++ /dev/null
@@ -1,210 +0,0 @@
-# Copyright 2018 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-
-from portage.util.futures import asyncio
-from portage.util.futures.compat_coroutine import (
-coroutine,
-coroutine_return,
-)
-from portage.util.futures._sync_decorator import _sync_decorator, _sync_methods
-from portage.tests import TestCase
-
-
-class CompatCoroutineTestCase(TestCase):
-def test_returning_coroutine(self):
-@coroutine
-def returning_coroutine(loop=None):
-yield asyncio.sleep(0, loop=loop)
-coroutine_return("success")
-
-loop = asyncio.get_event_loop()
-self.assertEqual(
-"success",
-
asyncio.get_event_loop().run_until_complete(returning_coroutine(loop=loop)),
-)
-
-def test_raising_coroutine(self):
-class TestException(Exception):
-pass
-
-@coroutine
-def raising_coroutine(loop=None):
-yield asyncio.sleep(0, loop=loop)
-raise TestException("exception")
-
-loop = asyncio.get_event_loop()
-self.assertRaises(
-TestException, loop.run_until_complete,
raising_coroutine(loop=loop)
-)
-
-def test_catching_coroutine(self):
-class TestException(Exception):
-pass
-
-@coroutine
-def catching_coroutine(loop=None):
-loop = asyncio._wrap_loop(loop)
-future = loop.create_future()
-loop.call_soon(future.set_exception, TestException("exception"))
-try:
-yield future
-except TestException:
-self.assertTrue(True)
-else:
-self.assertTrue(False)
-coroutine_return("success")
-
-loop = asyncio.get_event_loop()
-self.assertEqual(
-"success", loop.run_until_complete(catching_coroutine(loop=loop))
-)
-
-def test_cancelled_coroutine(self):
-"""
-Verify that a coroutine can handle (and reraise) asyncio.CancelledError
-in order to perform any necessary cleanup. Note that the
-asyncio.CancelledError will only be thrown in the coroutine if there's
-an opportunity (yield) before the generator raises StopIteration.
-"""
-loop = asyncio.get_event_loop()
-ready_for_exception = loop.create_future()
-exception_in_coroutine = loop.create_future()
-
-@coroutine
-def cancelled_coroutine(loop=None):
-loop = asyncio._wrap_loop(loop)
-while True:
-task = loop.create_future()
-try:
-ready_for_exception.set_result(None)
-yield task
-except BaseException as e:
-# Since python3.8, asyncio.CancelledError inherits
-# from BaseException.
-task.done() or task.cancel()
-exception_in_coroutine.set_exception(e)
-raise
-else:
-exception_in_coroutine.set_result(None)
-
-future = cancelled_coroutine(loop=loop)
-loop.run_until_complete(ready_for_exception)
-future.cancel()
-
-self.assertRaises(asyncio.CancelledError, loop.run_until_complete,
future)
-
-
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: fa7fb4c5119aa077a0731e1f8892ea0791a4968b
Author: Sam James gentoo org>
AuthorDate: Wed Feb 28 16:00:57 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 28 16:01:04 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fa7fb4c5
gpkg: placate black
Fixes: 957902f84edece635210689f46e20741e76d9dba
Signed-off-by: Sam James gentoo.org>
lib/portage/gpkg.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index 5cd1f2394e..2b957d58c4 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -656,7 +656,9 @@ class tar_safe_extract:
raise ValueError("Path traversal detected.")
if member.isdev():
writemsg(
-colorize("BAD", f"Danger: device file detected:
{member.name}\n")
+colorize(
+"BAD", f"Danger: device file detected:
{member.name}\n"
+)
)
raise ValueError("Device file detected.")
if member.islnk() and (member.linkname not in self.file_list):
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 957902f84edece635210689f46e20741e76d9dba
Author: Sam James gentoo org>
AuthorDate: Wed Feb 28 15:51:15 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 28 15:51:15 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=957902f8
gpkg: add missing new lines to error messages
Signed-off-by: Sam James gentoo.org>
lib/portage/gpkg.py | 18 +-
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
index edb0e43fbf..5cd1f2394e 100644
--- a/lib/portage/gpkg.py
+++ b/lib/portage/gpkg.py
@@ -382,7 +382,7 @@ class tar_stream_reader:
try:
if self.proc.wait() != os.EX_OK:
if not self.killed:
-writemsg(colorize("BAD", f"GPKG external program
failed."))
+writemsg(colorize("BAD", f"GPKG external program
failed.\n"))
raise CompressorOperationFailed("decompression failed")
finally:
self.proc.stdout.close()
@@ -418,7 +418,7 @@ class checksum_helper:
else:
self.uid = pwd.getpwnam(drop_user).pw_uid
except KeyError:
-writemsg(colorize("BAD", f"!!! Failed to find user
{drop_user}."))
+writemsg(colorize("BAD", f"!!! Failed to find user
{drop_user}.\n"))
raise
try:
@@ -428,7 +428,7 @@ class checksum_helper:
else:
self.gid = grp.getgrnam(drop_group).gr_gid
except KeyError:
-writemsg(colorize("BAD", f"!!! Failed to find group
{drop_group}."))
+writemsg(colorize("BAD", f"!!! Failed to find group
{drop_group}.\n"))
raise
else:
self.uid = None
@@ -636,33 +636,33 @@ class tar_safe_extract:
):
writemsg(
colorize(
-"BAD", f"Danger: duplicate files detected:
{member.name}"
+"BAD", f"Danger: duplicate files detected:
{member.name}\n"
)
)
raise ValueError("Duplicate files detected.")
if member.name.startswith("/"):
writemsg(
colorize(
-"BAD", f"Danger: absolute path detected:
{member.name}"
+"BAD", f"Danger: absolute path detected:
{member.name}\n"
)
)
raise ValueError("Absolute path detected.")
if member.name.startswith("../") or ("/../" in member.name):
writemsg(
colorize(
-"BAD", f"Danger: path traversal detected:
{member.name}"
+"BAD", f"Danger: path traversal detected:
{member.name}\n"
)
)
raise ValueError("Path traversal detected.")
if member.isdev():
writemsg(
-colorize("BAD", f"Danger: device file detected:
{member.name}")
+colorize("BAD", f"Danger: device file detected:
{member.name}\n")
)
raise ValueError("Device file detected.")
if member.islnk() and (member.linkname not in self.file_list):
writemsg(
colorize(
-"BAD", f"Danger: hardlink escape detected:
{member.name}"
+"BAD", f"Danger: hardlink escape detected:
{member.name}\n"
)
)
raise ValueError("Hardlink escape detected.")
@@ -995,7 +995,7 @@ class gpkg:
image_safe.extractall(decompress_dir)
image_tar.close()
except Exception as ex:
-writemsg(colorize("BAD", "!!!Extract failed."))
+writemsg(colorize("BAD", "!!!Extract failed.\n"))
raise
finally:
if not image_tar.closed:
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: f070275fe05d5053c3756ebb5d0a602db8ba515d
Author: Sam James gentoo org>
AuthorDate: Wed Feb 28 15:48:48 2024 +
Commit: Sam James gentoo org>
CommitDate: Wed Feb 28 15:49:16 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=f070275f
binpkg: add another missing newline to error message
```
Error reading binpkg
'/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar': [Errno 22]
Invalid argument!!! Invalid binary package:
'/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar', Error reading
binpkg '/var/cache/binpkgs/dev-perl/SGMLSpm/SGMLSpm-1.1-r2-7.gpkg.tar': [Errno
22] Invalid argument
```
Bug: https://bugs.gentoo.org/925714
Signed-off-by: Sam James gentoo.org>
lib/portage/binpkg.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py
index 9ecd52cf3c..a48e09bdb8 100644
--- a/lib/portage/binpkg.py
+++ b/lib/portage/binpkg.py
@@ -54,7 +54,7 @@ def get_binpkg_format(binpkg_path, check_file=False,
remote=False):
# We got many different exceptions here, so have to catch all of them.
file_format = None
writemsg(
-colorize("ERR", f"Error reading binpkg '{binpkg_path}': {err}"),
+colorize("ERR", f"Error reading binpkg '{binpkg_path}': {err}\n"),
)
raise InvalidBinaryPackageFormat(f"Error reading binpkg
'{binpkg_path}': {err}")
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/process/
commit: e882b1e956d50808a0143875a8ca35f2fc21f368
Author: Zac Medico gentoo org>
AuthorDate: Wed Feb 28 06:25:45 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Wed Feb 28 06:25:45 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e882b1e9
UnshareNetTestCase: Initialize ABILITY_TO_UNSHARE in setUp
Initialize ABILITY_TO_UNSHARE in setUp so that _unshare_validate
uses the correct PORTAGE_MULTIPROCESSING_START_METHOD setup
from super().setUp(), eliminating messages like this from CI
runs for the multiprocessing start method spawn:
/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/popen_fork.py:66:
DeprecationWarning: This process (pid=2886) is multi-threaded, use of
fork() may lead to deadlocks in the child.
The cause of these messages can be traced by patching python's
multiprocessing popen_fork.py like this:
--- /usr/lib/python3.12/multiprocessing/popen_fork.py
+++ /usr/lib/python3.12/multiprocessing/popen_fork.py
@@ -2,2 +2,3 @@
import signal
+import traceback
@@ -62,2 +63,3 @@
def _launch(self, process_obj):
+traceback.print_stack()
code = 1
Fixes: 3110ec376cbc ("actions: Fix interaction between start-method and
pytest-xdist")
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/process/test_unshare_net.py | 18 +-
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/lib/portage/tests/process/test_unshare_net.py
b/lib/portage/tests/process/test_unshare_net.py
index dabf15585f..ad3b288ef4 100644
--- a/lib/portage/tests/process/test_unshare_net.py
+++ b/lib/portage/tests/process/test_unshare_net.py
@@ -20,19 +20,27 @@ ping -c 1 -W 1 10.0.0.1 || exit 1
ping -c 1 -W 1 ::1 || exit 1
ping -c 1 -W 1 fd::1 || exit 1
"""
-ABILITY_TO_UNSHARE = portage.process._unshare_validate(CLONE_NEWNET)
class UnshareNetTestCase(TestCase):
-@pytest.mark.skipif(
-ABILITY_TO_UNSHARE != 0,
-reason=f"Unable to unshare: {errno.errorcode.get(ABILITY_TO_UNSHARE,
'?')}",
-)
+def setUp(self):
+"""
+Initialize ABILITY_TO_UNSHARE in setUp so that _unshare_validate
+uses the correct PORTAGE_MULTIPROCESSING_START_METHOD setup
+from super().setUp().
+"""
+super().setUp()
+self.ABILITY_TO_UNSHARE =
portage.process._unshare_validate(CLONE_NEWNET)
+
@pytest.mark.skipif(
portage.process.find_binary("ping") is None, reason="ping not found"
)
@pytest.mark.skipif(platform.system() != "Linux", reason="not Linux")
def testUnshareNet(self):
+if self.ABILITY_TO_UNSHARE != 0:
+pytest.skip(
+f"Unable to unshare:
{errno.errorcode.get(self.ABILITY_TO_UNSHARE, '?')}"
+)
env = os.environ.copy()
env["IPV6"] = "1" if portage.process._has_ipv6() else ""
self.assertEqual(
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/dbapi/
commit: e9bdb7342b3048ab3236bff9d94ce733bb877d8e
Author: Zac Medico gentoo org>
AuthorDate: Tue Feb 27 04:02:28 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Feb 27 04:03:07 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=e9bdb734
BinarytreeTestCase: Use temporary TMPDIR
Create a temporary TMPDIR which prevents test methods of this
class from leaving behind an empty /tmp/Packages file if TMPDIR
is initially unset.
Signed-off-by: Zac Medico gentoo.org>
lib/portage/tests/dbapi/test_bintree.py | 22 +-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/lib/portage/tests/dbapi/test_bintree.py
b/lib/portage/tests/dbapi/test_bintree.py
index 018f1cf9bd..91ac338a05 100644
--- a/lib/portage/tests/dbapi/test_bintree.py
+++ b/lib/portage/tests/dbapi/test_bintree.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Gentoo Authors
+# Copyright 2022-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from unittest.mock import MagicMock, patch, call
@@ -13,6 +13,26 @@ from portage.const import BINREPOS_CONF_FILE
class BinarytreeTestCase(TestCase):
+@classmethod
+def setUpClass(cls):
+"""
+Create a temporary TMPDIR which prevents test
+methods of this class from leaving behind an empty
+/tmp/Packages file if TMPDIR is initially unset.
+"""
+cls._orig_tmpdir = os.environ.get("TMPDIR")
+cls._tmpdir = tempfile.TemporaryDirectory()
+os.environ["TMPDIR"] = cls._tmpdir.name
+
+@classmethod
+def tearDownClass(cls):
+cls._tmpdir.cleanup()
+if cls._orig_tmpdir is None:
+os.environ.pop("TMPDIR", None)
+else:
+os.environ["TMPDIR"] = cls._orig_tmpdir
+del cls._orig_tmpdir, cls._tmpdir
+
def test_required_init_params(self):
with self.assertRaises(TypeError) as cm:
binarytree()
[gentoo-commits] proj/portage:master commit in: man/, lib/_emerge/
commit: 0896ede9663d1ffb10434ee163205e7d9a909667
Author: Zac Medico gentoo org>
AuthorDate: Mon Feb 26 00:12:13 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Tue Feb 27 02:52:19 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0896ede9
Scheduler: Support parallel-install with merge-wait
For system packages, always serialize install regardless of
parallel-install, in order to mitigate failures triggered
by fragile states as in bug 256616. For other packages,
continue to populate self._task_queues.merge, which will
serialize install unless parallel-install is enabled.
Fixes: 825db01b91a3 ("Add merge-wait FEATURES setting enabled by default")
Bug: https://bugs.gentoo.org/256616
Bug: https://bugs.gentoo.org/925213
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/PackageMerge.py | 4 ++--
lib/_emerge/Scheduler.py| 44 +---
man/make.conf.5 | 9 ++---
3 files changed, 41 insertions(+), 16 deletions(-)
diff --git a/lib/_emerge/PackageMerge.py b/lib/_emerge/PackageMerge.py
index 82725c66a5..11d0ff2f37 100644
--- a/lib/_emerge/PackageMerge.py
+++ b/lib/_emerge/PackageMerge.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.CompositeTask import CompositeTask
@@ -7,7 +7,7 @@ from portage.output import colorize
class PackageMerge(CompositeTask):
-__slots__ = ("merge", "postinst_failure")
+__slots__ = ("is_system_pkg", "merge", "postinst_failure")
def _should_show_status(self):
return (
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index 9950792dc9..5c318f89b9 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -1519,17 +1519,20 @@ class Scheduler(PollScheduler):
self._deallocate_config(build.settings)
elif build.returncode == os.EX_OK:
self.curval += 1
-merge = PackageMerge(merge=build, scheduler=self._sched_iface)
+merge = PackageMerge(
+is_system_pkg=(build.pkg in self._deep_system_deps),
+merge=build,
+scheduler=self._sched_iface,
+)
self._running_tasks[id(merge)] = merge
# By default, merge-wait only allows merge when no builds are
executing.
# As a special exception, dependencies on system packages are
frequently
# unspecified and will therefore force merge-wait.
-is_system_pkg = build.pkg in self._deep_system_deps
if not build.build_opts.buildpkgonly and (
-"merge-wait" in build.settings.features or is_system_pkg
+"merge-wait" in build.settings.features or merge.is_system_pkg
):
self._merge_wait_queue.append(merge)
-if is_system_pkg:
+if merge.is_system_pkg:
merge.addStartListener(self._system_merge_started)
else:
self._task_queues.merge.add(merge)
@@ -1804,13 +1807,32 @@ class Scheduler(PollScheduler):
and not self._jobs
and not self._task_queues.merge
):
-task = self._merge_wait_queue.popleft()
-task.scheduler = self._sched_iface
-self._merge_wait_scheduled.append(task)
-self._task_queues.merge.add(task)
-task.addExitListener(self._merge_wait_exit_handler)
-self._status_display.merges = len(self._task_queues.merge)
-state_change += 1
+while self._merge_wait_queue:
+# If we added non-system packages to the merge queue in a
+# previous iteration of this loop, then for system
packages we
+# need to come back later when the merge queue is empty.
+# TODO: Maybe promote non-system packages to the front of
the
+# queue and process them within the current loop, though
that
+# causes merge order to differ from the order builds
finish.
+if (
+self._task_queues.merge
+and self._merge_wait_queue[0].is_system_pkg
+):
+break
+task = self._merge_wait_queue.popleft()
+task.scheduler = self._sched_iface
+self._merge_wait_scheduled.append(task)
+self._task_queues.merge.add(task)
+task.addExitListener(self._merge_wait_exit_handler)
+self._status_display.merges = len(self._task_queues.merge)
+state_change += 1
+# For system packages, always serialize install regardless
of
+# parallel-install, in
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/resolver/, cnf/sets/, lib/portage/tests/sets/base/
commit: 1d3e3843f2a51c581d344540c5c6ee266afa30d2
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 25 22:57:43 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 26 23:17:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=1d3e3843
cnf: sets: Migrate @golang-rebuild to dev-lang/go
Bug: https://bugs.gentoo.org/919751
Signed-off-by: Zac Medico gentoo.org>
cnf/sets/portage.conf| 6 --
lib/portage/tests/resolver/ResolverPlayground.py | 10 +-
lib/portage/tests/sets/base/test_variable_set.py | 8 ++--
3 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf
index 2e02f91f97..c272f98db1 100644
--- a/cnf/sets/portage.conf
+++ b/cnf/sets/portage.conf
@@ -110,12 +110,6 @@ class = portage.sets.dbapi.UnavailableBinaries
[changed-deps]
class = portage.sets.dbapi.ChangedDepsSet
-# Installed packages for which vdb *DEPEND includes dev-lang/go.
-[golang-rebuild]
-class = portage.sets.dbapi.VariableSet
-variable = BDEPEND
-includes = dev-lang/go
-
# Installed packages for which vdb *DEPEND includes virtual/rust
[rust-rebuild]
class = portage.sets.dbapi.VariableSet
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
b/lib/portage/tests/resolver/ResolverPlayground.py
index 75c86b615c..c0455415a1 100644
--- a/lib/portage/tests/resolver/ResolverPlayground.py
+++ b/lib/portage/tests/resolver/ResolverPlayground.py
@@ -28,7 +28,7 @@ from portage.exception import InvalidBinaryPackageFormat
from portage.gpg import GPG
import _emerge
-from _emerge.actions import _calc_depclean
+from _emerge.actions import _calc_depclean, expand_set_arguments
from _emerge.Blocker import Blocker
from _emerge.create_depgraph_params import create_depgraph_params
from _emerge.DependencyArg import DependencyArg
@@ -747,6 +747,14 @@ class ResolverPlayground:
self.settings, self.trees, options, params, None
)
+atoms, retval = expand_set_arguments(
+atoms, action, self.trees[self.eroot]["root_config"]
+)
+if retval != os.EX_OK:
+raise AssertionError(
+f"expand_set_arguments failed with retval {retval}"
+)
+
if params_action == "remove":
depclean_result = _calc_depclean(
self.settings,
diff --git a/lib/portage/tests/sets/base/test_variable_set.py
b/lib/portage/tests/sets/base/test_variable_set.py
index 9e90ee6dd7..60c43a5b83 100644
--- a/lib/portage/tests/sets/base/test_variable_set.py
+++ b/lib/portage/tests/sets/base/test_variable_set.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Gentoo Authors
+# Copyright 2022-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
@@ -10,6 +10,10 @@ from portage.tests.resolver.ResolverPlayground import (
class VariableSetTestCase(TestCase):
def testVariableSetEmerge(self):
+
+# Using local set definition because @golang-rebuild migrated to
dev-lang/go since bug 919751.
+golang_rebuild =
"{class=portage.sets.dbapi.VariableSet,variable=BDEPEND,includes=dev-lang/go}"
+
ebuilds = {
"dev-go/go-pkg-1": {"BDEPEND": "dev-lang/go"},
"www-client/firefox-1": {
@@ -21,7 +25,7 @@ class VariableSetTestCase(TestCase):
test_cases = (
ResolverPlaygroundTestCase(
-["@golang-rebuild"],
+[f"@golang-rebuild{golang_rebuild}"],
mergelist=["dev-go/go-pkg-1"],
success=True,
),
[gentoo-commits] proj/portage:master commit in: lib/portage/util/, lib/portage/, lib/_emerge/
commit: 0ff7a3b28e0ec63d68d32e01145db8962d53774d
Author: Zac Medico gentoo org>
AuthorDate: Mon Feb 26 05:09:21 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 26 05:09:21 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=0ff7a3b2
SpawnProcess: Wait for async set_term_size
Use the SpawnProcess _unregister method to handle async
set_term_size results, avoiding possible messages like
bug 925456 triggered:
[ERROR] Task was destroyed but it is pending!
Also update _BinpkgFetcherProcess and _EbuildFetcherProcess
which inherit the _pty_ready attribute from SpawnProcess.
Fixes: f97e414ce980 ("set_term_size: Wait asynchronously if event loop is
running")
Bug: https://bugs.gentoo.org/923750
Bug: https://bugs.gentoo.org/925456
Signed-off-by: Zac Medico gentoo.org>
lib/_emerge/BinpkgFetcher.py | 6 --
lib/_emerge/EbuildFetcher.py | 6 --
lib/_emerge/SpawnProcess.py | 14 +-
lib/portage/output.py| 13 +
lib/portage/util/_pty.py | 23 ---
5 files changed, 46 insertions(+), 16 deletions(-)
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index a1524dc009..587e4a57a3 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.AsynchronousLock import AsynchronousLock
@@ -233,7 +233,9 @@ class _BinpkgFetcherProcess(SpawnProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def sync_timestamp(self):
diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py
index 7a45d95172..81d4b1054b 100644
--- a/lib/_emerge/EbuildFetcher.py
+++ b/lib/_emerge/EbuildFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -394,7 +394,9 @@ class _EbuildFetcherProcess(ForkProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def _eerror(self, lines):
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index b63afae01c..9fc12c42e5 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -46,6 +46,7 @@ class SpawnProcess(SubProcess):
+ (
"_main_task",
"_main_task_cancel",
+"_pty_ready",
"_selinux_type",
)
)
@@ -193,6 +194,9 @@ class SpawnProcess(SubProcess):
self._main_task.add_done_callback(self._main_exit)
async def _main(self, build_logger, pipe_logger, loop=None):
+if isinstance(self._pty_ready, asyncio.Future):
+await self._pty_ready
+self._pty_ready = None
try:
if pipe_logger.poll() is None:
await pipe_logger.async_wait()
@@ -238,7 +242,9 @@ class SpawnProcess(SubProcess):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
-got_pty, master_fd, slave_fd =
_create_pty_or_pipe(copy_term_size=stdout_pipe)
+self._pty_ready, master_fd, slave_fd = _create_pty_or_pipe(
+copy_term_size=stdout_pipe
+)
return (master_fd, slave_fd)
def _spawn(
@@ -258,6 +264,12 @@ class SpawnProcess(SubProcess):
SubProcess._unregister(self)
if self._main_task is not None:
self._main_task.done() or self._main_task.cancel()
+if isinstance(self._pty_ready, asyncio.Future):
+(
+self._pty_ready.done()
+and (self._pty_ready.cancelled() or self._pty_ready.result()
or True)
+) or self._pty_ready.cancel()
+self._pty_ready = None
def _cancel(self):
if self._main_task is not None:
diff --git a/lib/portage/output.py b/lib/portage/output.py
index 7d3a6278f3..4408705c45 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -8,6 +8,7 @@ import itertools
import re
import subprocess
import sys
+from typing import Optional
import portage
@@ -554,10 +555,16 @@ def get_term_size(fd=None):
return (0, 0)
-def set_term_size(lines, columns, fd):
+def set_term_size(lines: int, columns: int, fd: int) ->
Optional[asyncio.Future]:
[gentoo-commits] proj/portage:master commit in: /
commit: 50b1f276d3473d21dea2925a4e4de7d24edeade2 Author: Mike Gilbert gentoo org> AuthorDate: Mon Feb 26 04:12:23 2024 + Commit: Mike Gilbert gentoo org> CommitDate: Mon Feb 26 04:12:23 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=50b1f276 Fix NEWS Signed-off-by: Mike Gilbert gentoo.org> NEWS | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 258d800373..5da71f776e 100644 --- a/NEWS +++ b/NEWS @@ -10,11 +10,10 @@ portage-3.0.64 (UNRELEASED) -- Bug fixes: - +* ctypes: Add workaround for loading libc on musl. portage-3.0.63 (2024-02-25) -- -* ctypes: Add workaround for loading libc on musl Bug fixes: * emerge: Skip installed packages with emptytree in depgraph selection (bug #651018).
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/, lib/portage/util/, /, lib/portage/
commit: 19e27e0415fd321c39104f7d687bcdc4f4132e24
Author: Mike Gilbert gentoo org>
AuthorDate: Sun Feb 25 18:10:15 2024 +
Commit: Mike Gilbert gentoo org>
CommitDate: Mon Feb 26 04:10:32 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=19e27e04
Add workaround for loading libc on musl
musl libc has no soname, which causes ctypes.util.find_library to fail.
As a fallback, try to load "libc.so".
Signed-off-by: Mike Gilbert gentoo.org>
NEWS| 7 +++
lib/portage/dbapi/_MergeProcess.py | 4 ++--
lib/portage/dbapi/_SyncfsProcess.py | 14 --
lib/portage/process.py | 16 +---
lib/portage/util/_ctypes.py | 15 +++
lib/portage/util/locale.py | 7 ++-
6 files changed, 35 insertions(+), 28 deletions(-)
diff --git a/NEWS b/NEWS
index eb84651b53..258d800373 100644
--- a/NEWS
+++ b/NEWS
@@ -6,8 +6,15 @@ Release notes take the form of the following optional
categories:
* Bug fixes
* Cleanups
+portage-3.0.64 (UNRELEASED)
+--
+
+Bug fixes:
+
+
portage-3.0.63 (2024-02-25)
--
+* ctypes: Add workaround for loading libc on musl
Bug fixes:
* emerge: Skip installed packages with emptytree in depgraph selection (bug
#651018).
diff --git a/lib/portage/dbapi/_MergeProcess.py
b/lib/portage/dbapi/_MergeProcess.py
index dd5ad71cf8..d9ab2b47aa 100644
--- a/lib/portage/dbapi/_MergeProcess.py
+++ b/lib/portage/dbapi/_MergeProcess.py
@@ -10,7 +10,7 @@ import fcntl
import portage
from portage import os, _unicode_decode
from portage.package.ebuild._ipc.QueryCommand import QueryCommand
-from portage.util._ctypes import find_library
+from portage.util._ctypes import load_libc
import portage.elog.messages
from portage.util._async.ForkProcess import ForkProcess
from portage.util import no_color
@@ -64,7 +64,7 @@ class MergeProcess(ForkProcess):
# process, so that it's only done once rather than
# for each child process.
if platform.system() == "Linux" and "merge-sync" in settings.features:
-find_library("c")
+load_libc()
# Inherit stdin by default, so that the pdb SIGUSR1
# handler is usable for the subprocess.
diff --git a/lib/portage/dbapi/_SyncfsProcess.py
b/lib/portage/dbapi/_SyncfsProcess.py
index ddc2240071..300ae53985 100644
--- a/lib/portage/dbapi/_SyncfsProcess.py
+++ b/lib/portage/dbapi/_SyncfsProcess.py
@@ -4,7 +4,7 @@
import functools
from portage import os
-from portage.util._ctypes import find_library, LoadLibrary
+from portage.util._ctypes import load_libc
from portage.util._async.ForkProcess import ForkProcess
@@ -24,15 +24,9 @@ class SyncfsProcess(ForkProcess):
@staticmethod
def _get_syncfs():
-filename = find_library("c")
-if filename is not None:
-library = LoadLibrary(filename)
-if library is not None:
-try:
-return library.syncfs
-except AttributeError:
-pass
-
+(libc, _) = load_libc()
+if libc is not None:
+return getattr(libc, "syncfs", None)
return None
@staticmethod
diff --git a/lib/portage/process.py b/lib/portage/process.py
index cc9ed7bf78..b652e32942 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -37,7 +37,7 @@ portage.proxy.lazyimport.lazyimport(
from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY
from portage.exception import CommandNotFound
from portage.proxy.objectproxy import ObjectProxy
-from portage.util._ctypes import find_library, LoadLibrary, ctypes
+from portage.util._ctypes import load_libc, LoadLibrary, ctypes
try:
from portage.util.netlink import RtNetlink
@@ -960,11 +960,9 @@ def _exec(
have_unshare = False
libc = None
if unshare_net or unshare_ipc or unshare_mount or unshare_pid:
-filename = find_library("c")
-if filename is not None:
-libc = LoadLibrary(filename)
-if libc is not None:
-have_unshare = hasattr(libc, "unshare")
+(libc, _) = load_libc()
+if libc is not None:
+have_unshare = hasattr(libc, "unshare")
if not have_unshare:
# unshare() may not be supported by libc
@@ -1212,11 +1210,7 @@ class _unshare_validator:
"""
# This ctypes library lookup caches the result for use in the
# subprocess when the multiprocessing start method is fork.
-filename = find_library("c")
-if filename is None:
-return errno.ENOTSUP
-
-libc = LoadLibrary(filename)
+(libc, filename) = load_libc()
if libc is None:
return errno.ENOTSUP
diff --git a/lib/portage/util/_ctypes.py b/lib/portage/util/_ctypes.py
index e6d1e327cb..04e965ba92 100644
--- a/lib/portage/util/_ctypes.py
+++ b/lib/portage/util/_ctypes.py
@@
[gentoo-commits] proj/portage:master commit in: /
commit: 8832264b4e5bff768346ba5a6d2f828dcf468c60 Author: Sam James gentoo org> AuthorDate: Sun Feb 25 08:29:05 2024 + Commit: Sam James gentoo org> CommitDate: Sun Feb 25 08:29:43 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8832264b NEWS, meson.build: prepare for portage-3.0.63 Signed-off-by: Sam James gentoo.org> NEWS| 30 +- meson.build | 2 +- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 83be372043..eb84651b53 100644 --- a/NEWS +++ b/NEWS @@ -6,7 +6,7 @@ Release notes take the form of the following optional categories: * Bug fixes * Cleanups -portage-3.0.63 (UNRELEASED) +portage-3.0.63 (2024-02-25) -- Bug fixes: @@ -14,6 +14,34 @@ Bug fixes: * bin/install-qa-check.d: 90gcc-warnings: drop warnings with too many FPs (bug #925460). +* AsyncioEventLoop: Call process.run_exitfuncs() before close (bug #925240). + + Fixes hang with FEATURES="network-sandbox-proxy" or FEATURES="distcc". + +* SchedulerInterface/PollScheduler: Add _loop property (bug #925333). + + Fixes erroneous assert. + +* _dynamic_deps_preload: Fix settings reference (bug #925350). + + Fix KeyError with --dynamic-deps. + +* process.spawn: Fix logic for missing libc.unshare on musl (bug #925311). + + Fix handling when we can't unshare where we might e.g. modify the hostname + of the real system on musl. + +* dbapi: Fix TypeError when passing Exception to warnings.warn (bug #922935). + + Fix testUpdateDbentryDbapiTestCase. + +* _start_proc: Prevent premature ForkProcess garbage collection (bug #925456). + + Fix warnings/errors like "[ERROR] Task was destroyed but it is pending!". + +Cleanups: +* More asyncio porting. + portage-3.0.62 (2024-02-22) -- diff --git a/meson.build b/meson.build index aeb3f93e68..e826f6ffad 100644 --- a/meson.build +++ b/meson.build @@ -1,7 +1,7 @@ project( 'portage', 'c', -version : '3.0.62', +version : '3.0.63', license : 'GPL-2.0-or-later', meson_version : '>=0.58.0' )
[gentoo-commits] proj/portage:master commit in: lib/portage/dbapi/
commit: 0855821572f32e81b031a250f7491f34a2fd4078
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 24 23:29:29 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Feb 25 08:24:55 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=08558215
dbapi: Fix TypeError when passing Exception to warnings.warn
If an Exception is passed as a message to warnings.warn then
it triggers this TypeError:
if metadata_updates:
try:
aux_update(cpv, metadata_updates)
except (InvalidBinaryPackageFormat, CorruptionKeyError) as e:
> warnings.warn(e)
E TypeError: expected string or bytes-like object, got
'CorruptionKeyError'
Fixes: fb1d0a22f657 ("dbapi: KeyError tolerance during package moves")
Bug: https://bugs.gentoo.org/922935
Signed-off-by: Zac Medico gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1282
Signed-off-by: Sam James gentoo.org>
lib/portage/dbapi/__init__.py | 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 6f95b93a21..9105227c77 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -1,11 +1,12 @@
-# Copyright 1998-2023 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ["dbapi"]
import functools
+import logging
import re
-import warnings
+import sys
from typing import Any, Dict, List, Optional, Tuple
from collections.abc import Sequence
@@ -429,7 +430,9 @@ class dbapi:
try:
aux_update(cpv, metadata_updates)
except (InvalidBinaryPackageFormat, CorruptionKeyError) as e:
-warnings.warn(e)
+logging.warning(
+f"{e.__class__.__name__}: {e}", exc_info=sys.exc_info()
+)
if onUpdate:
onUpdate(maxval, i + 1)
if onProgress:
@@ -477,6 +480,6 @@ class dbapi:
try:
self.aux_update(mycpv, mydata)
except CorruptionKeyError as e:
-warnings.warn(e)
+logging.warning(f"{e.__class__.__name__}: {e}",
exc_info=sys.exc_info())
continue
return moves
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/emerge/
commit: 8a2f1d14788d107ec54dc53c9ef1cf00ee310d51 Author: Gábor Oszkár Dénes protonmail com> AuthorDate: Sat Feb 24 20:48:05 2024 + Commit: Sam James gentoo org> CommitDate: Sun Feb 25 08:25:06 2024 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=8a2f1d14 test_baseline: Improve robustness with cleanup The baseline tests need to cleanup the ResolverPlayground after each testcase, with each different parametrization. This is ensured by making the scope of the playground fixture the function instead of the module. With module the cleanup only happens before/after the switch from/to xpak and gpkg. Signed-off-by: Gábor Oszkár Dénes protonmail.com> Closes: https://github.com/gentoo/portage/pull/1281 Signed-off-by: Sam James gentoo.org> lib/portage/tests/emerge/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/tests/emerge/conftest.py b/lib/portage/tests/emerge/conftest.py index 580d1e09ab..356e09879c 100644 --- a/lib/portage/tests/emerge/conftest.py +++ b/lib/portage/tests/emerge/conftest.py @@ -406,7 +406,7 @@ def async_loop(): yield asyncio._wrap_loop() -@pytest.fixture(params=SUPPORTED_GENTOO_BINPKG_FORMATS, scope="module") +@pytest.fixture(params=SUPPORTED_GENTOO_BINPKG_FORMATS, scope="function") def playground(request, tmp_path_factory): """Fixture that provides instances of ``ResolverPlayground`` each one with one supported value for ``BINPKG_FORMAT``."""
[gentoo-commits] proj/portage:master commit in: lib/portage/
commit: 92615cd37d7a1efce923c474e455f59fe61a589c
Author: Zac Medico gentoo org>
AuthorDate: Sun Feb 25 05:09:48 2024 +
Commit: Sam James gentoo org>
CommitDate: Sun Feb 25 08:24:59 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=92615cd3
_start_proc: Prevent premature ForkProcess garbage collection
In order to prevent the constructed ForkProcess instance from
being prematurely garbage collected, return a reference to the
caller inside of a _GCProtector object, preventing messages
reported in bug 925456 like this:
[ERROR] Task was destroyed but it is pending!
Fixes: a69c1b853a47 ("process.spawn: Use multiprocessing.Process for
returnproc")
Bug: https://bugs.gentoo.org/925456
Signed-off-by: Zac Medico gentoo.org>
Closes: https://github.com/gentoo/portage/pull/1284
Signed-off-by: Sam James gentoo.org>
lib/portage/process.py | 41 +++--
1 file changed, 39 insertions(+), 2 deletions(-)
diff --git a/lib/portage/process.py b/lib/portage/process.py
index d16262e75a..cc9ed7bf78 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -36,6 +36,7 @@ portage.proxy.lazyimport.lazyimport(
from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY
from portage.exception import CommandNotFound
+from portage.proxy.objectproxy import ObjectProxy
from portage.util._ctypes import find_library, LoadLibrary, ctypes
try:
@@ -1493,8 +1494,44 @@ def _start_proc(
proc.start()
# ForkProcess conveniently holds a MultiprocessingProcess
-# instance that is suitable to return here.
-return proc._proc
+# instance that is suitable to return here, but use _GCProtector
+# to protect the ForkProcess instance from being garbage collected
+# and triggering messages like this (bug 925456):
+# [ERROR] Task was destroyed but it is pending!
+return _GCProtector(proc._proc, proc.async_wait)
+
+
+class _GCProtector(ObjectProxy):
+"""
+Proxy a target object, and also hold a reference to something
+extra in order to protect it from garbage collection. Override
+the wait method to first call target's wait method and then
+wait for extra (a coroutine function) before returning the result.
+"""
+
+__slots__ = ("_extra", "_target")
+
+def __init__(self, target, extra):
+super().__init__()
+object.__setattr__(self, "_target", target)
+object.__setattr__(self, "_extra", extra)
+
+def _get_target(self):
+return object.__getattribute__(self, "_target")
+
+def __getattribute__(self, attr):
+if attr == "wait":
+return object.__getattribute__(self, attr)
+return getattr(object.__getattribute__(self, "_target"), attr)
+
+async def wait(self):
+"""
+Wrap the target's wait method to also wait for an extra
+coroutine function.
+"""
+result = await object.__getattribute__(self, "_target").wait()
+await object.__getattribute__(self, "_extra")()
+return result
def find_binary(binary):
