[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/, ...
commit: 6ce2be8d454f95c508d9f547d13487f9de863bdd
Author: Zac Medico gentoo org>
AuthorDate: Sun Mar 3 19:53:11 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Mar 3 19:53:11 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=6ce2be8d
_validate_deps: Discard configdict["pkg"]["USE"]
Since configdict["pkg"]["USE"] may contain package.use settings
from config.setcpv, it is inappropriate to use here (bug 675748),
so discard it. This is only an issue because configdict["pkg"] is
a sub-optimal place to extract metadata from. This issue does not
necessarily indicate a flaw in the Package constructor, since
passing in precalculated USE can be valid for things like
autounmask USE changes.
Bug: https://bugs.gentoo.org/675748
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/doebuild.py | 12
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 19 ++-
lib/portage/tests/resolver/ResolverPlayground.py | 1 +
3 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index 942fa90101..6691db4e97 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -1813,6 +1813,14 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi):
invalid_dep_exempt_phases = {"clean", "cleanrm", "help", "prerm", "postrm"}
all_keys = set(Package.metadata_keys)
all_keys.add("SRC_URI")
+# Since configdict["pkg"]["USE"] may contain package.use settings
+# from config.setcpv, it is inappropriate to use here (bug 675748),
+# so discard it. This is only an issue because configdict["pkg"] is
+# a sub-optimal place to extract metadata from. This issue does not
+# necessarily indicate a flaw in the Package constructor, since
+# passing in precalculated USE can be valid for things like
+# autounmask USE changes.
+all_keys.discard("USE")
all_keys = tuple(all_keys)
metadata = mysettings.configdict["pkg"]
if all(k in metadata for k in ("PORTAGE_REPO_NAME", "SRC_URI")):
@@ -1838,6 +1846,10 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi):
root_config = RootConfig(mysettings, {"porttree": FakeTree(mydbapi)}, None)
+# A USE calculation from setcpv should always be available here because
+# mysettings.mycpv is not None, so use it to prevent redundant setcpv
calls.
+metadata["USE"] = mysettings["PORTAGE_USE"]
+
pkg = Package(
built=False,
cpv=mysettings.mycpv,
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index b38605bb90..445fcf6c4e 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -51,10 +51,23 @@ class DoebuildFdPipesTestCase(TestCase):
ebuilds = {
"app-misct/foo-1": {
"EAPI": "8",
+"IUSE": "+foo +bar",
+"REQUIRED_USE": "|| ( foo bar )",
"MISC_CONTENT": ebuild_body,
}
}
+# Populate configdict["pkg"]["USE"] with something arbitrary in order
+# to try and trigger bug 675748 in doebuild _validate_deps.
+arbitrary_package_use = "baz"
+
+user_config = {
+# In order to trigger bug 675748, package.env must be non-empty,
+# but the referenced env file can be empty.
+"package.env": (f"app-misct/foo {os.devnull}",),
+"package.use": (f"app-misct/foo {arbitrary_package_use}",),
+}
+
# Override things that may be unavailable, or may have portability
# issues when running tests in exotic environments.
# prepstrip - bug #447810 (bash read builtin EINTR problem)
@@ -63,7 +76,7 @@ class DoebuildFdPipesTestCase(TestCase):
self.assertEqual(true_binary is None, False, "true command not found")
dev_null = open(os.devnull, "wb")
-playground = ResolverPlayground(ebuilds=ebuilds)
+playground = ResolverPlayground(ebuilds=ebuilds,
user_config=user_config)
try:
QueryCommand._db = playground.trees
root_config = playground.trees[playground.eroot]["root_config"]
@@ -106,6 +119,10 @@ class DoebuildFdPipesTestCase(TestCase):
)
settings.setcpv(pkg)
+# Demonstrate that settings.configdict["pkg"]["USE"] contains our
arbitrary
+# package.use setting in order to trigger bug 675748.
+self.assertEqual(settings.configdict["pkg"]["USE"],
arbitrary_package_use)
+
# Try to trigger the config.environ() split_LC_ALL assertion for
bug 925863.
settings["LC_ALL"] = "C"
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py
b/lib/portage/tests/resolver/ResolverPlayground.py
index c0455415a1..f52a98f8db
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/
commit: fe510e099bc9a8055c3ee50fced47fc3dc7ba166
Author: Zac Medico gentoo org>
AuthorDate: Fri Mar 1 17:09:56 2024 +
Commit: Zac Medico gentoo org>
CommitDate: Fri Mar 1 18:08:51 2024 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=fe510e09
doebuild: Call _setup_locale
Call _setup_locale in order to prevent an AssertionError from
config.environ() for the config phase (or any other phase for
that matter).
For returnproc or returnpid assume that the event loop is running
so we can't run the event loop to call _setup_locale in this case
and we have to assume the caller took care of it (otherwise
config.environ() will raise AssertionError).
Update DoebuildFdPipesTestCase to use EAPI 8 and test the
pkg_config function with an ebuild located in /var/db/pkg just
like emerge --config does. Set LC_ALL=C just before doebuild
calls in order to try and trigger the config.environ()
split_LC_ALL assertion.
Bug: https://bugs.gentoo.org/925863
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/doebuild.py | 8 +++
lib/portage/tests/ebuild/test_doebuild_fd_pipes.py | 77 --
2 files changed, 52 insertions(+), 33 deletions(-)
diff --git a/lib/portage/package/ebuild/doebuild.py
b/lib/portage/package/ebuild/doebuild.py
index bc51fdff2d..942fa90101 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -43,6 +43,7 @@ portage.proxy.lazyimport.lazyimport(
"portage.util._async.SchedulerInterface:SchedulerInterface",
"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.util.ExtractKernelVersion:ExtractKernelVersion",
+"_emerge.EbuildPhase:_setup_locale",
)
from portage import (
@@ -1034,6 +1035,13 @@ def doebuild(
myebuild, mydo, myroot, mysettings, debug, use_cache, mydbapi
)
+# For returnproc or returnpid assume that the event loop is running
+# so we can't run the event loop to call _setup_locale in this case
+# and we have to assume the caller took care of it (otherwise
+# config.environ() will raise AssertionError).
+if not (returnproc or returnpid):
+asyncio.run(_setup_locale(mysettings))
+
if mydo in clean_phases:
builddir_lock = None
if not returnpid and "PORTAGE_BUILDDIR_LOCKED" not in mysettings:
diff --git a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
index 678486ed16..b38605bb90 100644
--- a/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
+++ b/lib/portage/tests/ebuild/test_doebuild_fd_pipes.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2023 Gentoo Authors
+# Copyright 2013-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import multiprocessing
@@ -27,20 +27,22 @@ class DoebuildFdPipesTestCase(TestCase):
output_fd = self.output_fd
ebuild_body = ["S=${WORKDIR}"]
-for phase_func in (
-"pkg_info",
-"pkg_nofetch",
-"pkg_pretend",
-"pkg_setup",
-"src_unpack",
-"src_prepare",
-"src_configure",
-"src_compile",
-"src_test",
-"src_install",
+for phase_func, default in (
+("pkg_info", False),
+("pkg_nofetch", False),
+("pkg_pretend", False),
+("pkg_setup", False),
+("pkg_config", False),
+("src_unpack", False),
+("src_prepare", True),
+("src_configure", False),
+("src_compile", False),
+("src_test", False),
+("src_install", False),
):
ebuild_body.append(
-("%s() { echo ${EBUILD_PHASE}" " 1>&%s; }") % (phase_func,
output_fd)
+("%s() { %secho ${EBUILD_PHASE}" " 1>&%s; }")
+% (phase_func, "default; " if default else "", output_fd)
)
ebuild_body.append("")
@@ -48,7 +50,7 @@ class DoebuildFdPipesTestCase(TestCase):
ebuilds = {
"app-misct/foo-1": {
-"EAPI": "5",
+"EAPI": "8",
"MISC_CONTENT": ebuild_body,
}
}
@@ -103,24 +105,33 @@ class DoebuildFdPipesTestCase(TestCase):
type_name="ebuild",
)
settings.setcpv(pkg)
-ebuildpath = portdb.findname(cpv)
-self.assertNotEqual(ebuildpath, None)
-
-for phase in (
-"info",
-"nofetch",
-"pretend",
-"setup",
-"unpack",
-"prepare",
-"configure",
-"compile",
-"test",
-"install",
-"qmerge",
-"clean",
-"merge",
+
+# Try to trigger the
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/
commit: b9ef191c74982b0e8d837aa7dd256dc3c52f7d2c
Author: Zac Medico gentoo org>
AuthorDate: Sat Feb 20 23:11:46 2021 +
Commit: Zac Medico gentoo org>
CommitDate: Mon Feb 22 11:48:41 2021 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=b9ef191c
MirrorLayoutConfig: content digest support (bug 756778)
In order to support mirror layouts that use content
digests, extend MirrorLayoutConfig validate_structure and
get_best_supported_layout methods to support an optional
filename parameter of type DistfileName which includes a digests
attribute. Use the new parameter to account for availablility
of specific distfile content digests when validating and selecting
mirror layouts which require those digests.
The DistfileName type represents a distfile name and associated
content digests, used by MirrorLayoutConfig and related layout
implementations.
The path of a distfile within a layout must be dependent on
nothing more than the distfile name and its associated content
digests. For filename-hash layout, path is dependent on distfile
name alone, and the get_filenames implementation yields strings
corresponding to distfile names. For content-hash layout, path is
dependent on content digest alone, and the get_filenames
implementation yields DistfileName instances whose names are equal
to content digest values. The content-hash layout simply lacks
the filename-hash layout's innate ability to translate a distfile
path to a distfile name, and instead caries an innate ability
to translate a distfile path to a content digest.
In order to prepare for a migration from filename-hash to
content-hash layout, all consumers of the layout get_filenames
method need to be updated to work with content digests as a
substitute for distfile names. For example, in order to prepare
emirrordist for content-hash, a key-value store needs to be
added as a means to associate distfile names with content
digest values yielded by the content-hash get_filenames
implementation.
Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/fetch.py| 98 ++
lib/portage/tests/ebuild/test_fetch.py | 33 +---
2 files changed, 114 insertions(+), 17 deletions(-)
diff --git a/lib/portage/package/ebuild/fetch.py
b/lib/portage/package/ebuild/fetch.py
index e0fecaf23..af9edd91e 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
__all__ = ['fetch']
@@ -344,6 +344,57 @@ _size_suffix_map = {
}
+class DistfileName(str):
+ """
+ The DistfileName type represents a distfile name and associated
+ content digests, used by MirrorLayoutConfig and related layout
+ implementations.
+
+ The path of a distfile within a layout must be dependent on
+ nothing more than the distfile name and its associated content
+ digests. For filename-hash layout, path is dependent on distfile
+ name alone, and the get_filenames implementation yields strings
+ corresponding to distfile names. For content-hash layout, path is
+ dependent on content digest alone, and the get_filenames
+ implementation yields DistfileName instances whose names are equal
+ to content digest values. The content-hash layout simply lacks
+ the filename-hash layout's innate ability to translate a distfile
+ path to a distfile name, and instead caries an innate ability
+ to translate a distfile path to a content digest.
+
+ In order to prepare for a migration from filename-hash to
+ content-hash layout, all consumers of the layout get_filenames
+ method need to be updated to work with content digests as a
+ substitute for distfile names. For example, in order to prepare
+ emirrordist for content-hash, a key-value store needs to be
+ added as a means to associate distfile names with content
+ digest values yielded by the content-hash get_filenames
+ implementation.
+ """
+ def __new__(cls, s, digests=None):
+ return str.__new__(cls, s)
+
+ def __init__(self, s, digests=None):
+ super().__init__()
+ self.digests = {} if digests is None else digests
+
+ def digests_equal(self, other):
+ """
+ Test if digests compare equal to those of another instance.
+ """
+ if not isinstance(other, DistfileName):
+ return False
+ matches = []
+ for algo, digest in self.digests.items():
+ other_digest = other.digests.get(algo)
+ if other_digest is not None:
+ if other_digest == digest:
+
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/
commit: a4f06ab3cf7339100b2af2146ae90cbba8bac371 Author: Daniel Robbins funtoo org> AuthorDate: Sat Feb 20 23:11:46 2021 + Commit: Zac Medico gentoo org> CommitDate: Mon Feb 22 11:48:41 2021 + URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=a4f06ab3 Add content-hash distfiles layout (bug 756778) The content-hash layout is identical to the filename-hash layout, except for these three differences: 1) A content digest is used instead of a filename digest. 2) The final element of the path returned from the get_path method corresponds to the complete content digest. The path is a function of the content digest alone. 3) Because the path is a function of content digest alone, the get_filenames implementation cannot derive distfiles names from paths, so it instead yields DistfileName instances whose names are equal to content digest values. The DistfileName documentation discusses resulting implications. Motivations to use the content-hash layout instead of the filename-hash layout may include: 1) Since the file path is independent of the file name, file name collisions cannot occur. This makes the content-hash layout suitable for storage of multiple types of files (not only gentoo distfiles). For example, it can be used to store distfiles for multiple linux distros within the same tree, with automatic deduplication based on content digest. This layout can be used to store and distribute practically anything (including binary packages for example). 2) Allows multiple revisions for the same distfiles name. An existing distfile can be updated, and if a user still has an older copy of an ebuild repository (or an overlay), then a user can successfully fetch a desired revision of the distfile as long as it has not been purged from the mirror. 3) File integrity data is integrated into the layout itself, making it very simple to verify the integrity of any file that it contains. The only tool required is an implementation of the chosen hash algorithm. Bug: https://bugs.gentoo.org/756778 Signed-off-by: Zac Medico gentoo.org> lib/portage/package/ebuild/fetch.py| 97 ++ lib/portage/tests/ebuild/test_fetch.py | 36 + 2 files changed, 133 insertions(+) diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py index af9edd91e..f0ae864ad 100644 --- a/lib/portage/package/ebuild/fetch.py +++ b/lib/portage/package/ebuild/fetch.py @@ -464,6 +464,97 @@ class FilenameHashLayout: return False +class ContentHashLayout(FilenameHashLayout): + """ + The content-hash layout is identical to the filename-hash layout, + except for these three differences: + + 1) A content digest is used instead of a filename digest. + + 2) The final element of the path returned from the get_path method + corresponds to the complete content digest. The path is a function + of the content digest alone. + + 3) Because the path is a function of content digest alone, the + get_filenames implementation cannot derive distfiles names from + paths, so it instead yields DistfileName instances whose names are + equal to content digest values. The DistfileName documentation + discusses resulting implications. + + Motivations to use the content-hash layout instead of the + filename-hash layout may include: + + 1) Since the file path is independent of the file name, file + name collisions cannot occur. This makes the content-hash + layout suitable for storage of multiple types of files (not + only gentoo distfiles). For example, it can be used to store + distfiles for multiple linux distros within the same tree, + with automatic deduplication based on content digest. This + layout can be used to store and distribute practically anything + (including binary packages for example). + + 2) Allows multiple revisions for the same distfiles name. An + existing distfile can be updated, and if a user still has an + older copy of an ebuild repository (or an overlay), then a user + can successfully fetch a desired revision of the distfile as + long as it has not been purged from the mirror. + + 3) File integrity data is integrated into the layout itself, + making it very simple to verify the integrity of any file that + it contains. The only tool required is an implementation of + the chosen hash algorithm. + """ + + def get_path(self, filename): + """ + For content-hash, the path is a function of the content digest alone. + The final element of the path returned from the get_path method + corresponds to the complete content digest. + """ + fnhash = remaining = filename.digests[self.algo] + ret = "" + for c in self.cutoffs: +
[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/
commit: 52bc75a60b84d709712e91c68782f2f207bfce4e
Author: Zac Medico gentoo org>
AuthorDate: Sun Oct 20 00:55:09 2019 +
Commit: Zac Medico gentoo org>
CommitDate: Sun Oct 20 08:33:09 2019 +
URL:https://gitweb.gentoo.org/proj/portage.git/commit/?id=52bc75a6
fetch: add force parameter (bug 697566)
Add a force parameter which forces download even when a file already
exists in DISTDIR (and no digests are available to verify it). This
avoids the need to remove the existing file in advance, which makes
it possible to atomically replace the file and avoid interference
with concurrent processes. This is useful when using FETCHCOMMAND to
fetch a mirror's layout.conf file, for the purposes of bug 697566.
Bug: https://bugs.gentoo.org/697566
Reviewed-by: Michał Górny gentoo.org>
Signed-off-by: Zac Medico gentoo.org>
lib/portage/package/ebuild/fetch.py| 22 +++
lib/portage/tests/ebuild/test_fetch.py | 40 +++---
2 files changed, 55 insertions(+), 7 deletions(-)
diff --git a/lib/portage/package/ebuild/fetch.py
b/lib/portage/package/ebuild/fetch.py
index 76e4636c2..05de12740 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -432,7 +432,7 @@ def get_mirror_url(mirror_url, filename, cache_path=None):
def fetch(myuris, mysettings, listonly=0, fetchonly=0,
locks_in_subdir=".locks", use_locks=1, try_mirrors=1, digests=None,
- allow_missing_digests=True):
+ allow_missing_digests=True, force=False):
"""
Fetch files to DISTDIR and also verify digests if they are available.
@@ -455,10 +455,23 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
@param allow_missing_digests: Enable fetch even if there are no digests
available for verification.
@type allow_missing_digests: bool
+ @param force: Force download, even when a file already exists in
+ DISTDIR. This is most useful when there are no digests
available,
+ since otherwise download will be automatically forced if the
+ existing file does not match the available digests. Also, this
+ avoids the need to remove the existing file in advance, which
+ makes it possible to atomically replace the file and avoid
+ interference with concurrent processes.
+ @type force: bool
@rtype: int
@return: 1 if successful, 0 otherwise.
"""
+ if force and digests:
+ # Since the force parameter can trigger unnecessary fetch when
the
+ # digests match, do not allow force=True when digests are
provided.
+ raise PortageException(_('fetch: force=True is not allowed when
digests are provided'))
+
if not myuris:
return 1
@@ -878,7 +891,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
eout.quiet = mysettings.get("PORTAGE_QUIET") ==
"1"
match, mystat = _check_distfile(
myfile_path, pruned_digests, eout,
hash_filter=hash_filter)
- if match:
+ if match and not force:
# Skip permission adjustment for
symlinks, since we don't
# want to modify anything outside of
the primary DISTDIR,
# and symlinks typically point to
PORTAGE_RO_DISTDIRS.
@@ -1042,10 +1055,11 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
os.unlink(download_path)
except EnvironmentError:
pass
- elif myfile not in mydigests:
+ elif not orig_digests:
# We don't have a digest, but
the file exists. We must
# assume that it is fully
downloaded.
- continue
+ if not force:
+ continue
else:
if
(mydigests[myfile].get("size") is not None
and
mystat.st_size < mydigests[myfile]["size"]
diff --git a/lib/portage/tests/ebuild/test_fetch.py
b/lib/portage/tests/ebuild/test_fetch.py
index f50fea0dd..538fb1754 100644
--- a/lib/portage/tests/ebuild/test_fetch.py
+++ b/lib/portage/tests/ebuild/test_fetch.py
@@ -119,10 +119,44 @@ class EbuildFetchTestCase(TestCase):
with
