Add FEATURES=binpkg-docompress that can be used whether docompress compression is performed before or after creating binary packages. With the feature enabled (the default), the current behavior of storing compressed files in binpkg is preserved. With it disabled, uncompressed files are stored inside binary package and are compressed when installing.
Storing uncompressed files in binary packages has two advantages: 1. Avoids the double-compression penalty, effectively improving binary package compression speed and compression ratio. 2. Allows the same packages to be reused on systems with different docompress configurations. The option is roughly backwards compatible. Old Portage versions will install packages created with FEATURES=-binpkg-docompress correctly, albeit without compression. Portage with FEATURES=binpkg-docompress should install old binpackages semi-correctly, potentially recompressing them (and throwing already-compressed warnings on format mismatch). The new behavior is left off by default to avoid those problems. Signed-off-by: Michał Górny <mgo...@gentoo.org> --- bin/misc-functions.sh | 43 +++++++++++++++++++++++--- bin/phase-functions.sh | 2 +- cnf/make.globals | 2 +- lib/portage/const.py | 1 + lib/portage/dbapi/vartree.py | 12 +++++++ lib/portage/package/ebuild/doebuild.py | 4 ++- man/ebuild.1 | 5 +++ man/make.conf.5 | 6 ++++ 8 files changed, 68 insertions(+), 7 deletions(-) Changes in v2: * added .instprepped file logic, * added instprep -> install dependency, * documented in ebuild.1. I've tested this thoroughly via ebuild(1) and emerge(1). diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index db7aaed5a..3b3a5e09c 100755 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -109,10 +109,13 @@ install_qa_check() { [[ -d ${ED%/}/usr/share/info ]] && prepinfo - # Apply compression. - "${PORTAGE_BIN_PATH}"/ecompress --queue "${PORTAGE_DOCOMPRESS[@]}" - "${PORTAGE_BIN_PATH}"/ecompress --ignore "${PORTAGE_DOCOMPRESS_SKIP[@]}" - "${PORTAGE_BIN_PATH}"/ecompress --dequeue + # If binpkg-docompress is enabled, apply compression before creating + # the binary package. + if has binpkg-docompress ${FEATURES}; then + "${PORTAGE_BIN_PATH}"/ecompress --queue "${PORTAGE_DOCOMPRESS[@]}" + "${PORTAGE_BIN_PATH}"/ecompress --ignore "${PORTAGE_DOCOMPRESS_SKIP[@]}" + "${PORTAGE_BIN_PATH}"/ecompress --dequeue + fi export STRIP_MASK if ___eapi_has_dostrip; then @@ -160,6 +163,38 @@ install_qa_check() { rm -f "${ED%/}"/usr/share/info/dir{,.gz,.bz2} || die "rm failed!" } +__dyn_instprep() { + if [[ -e ${PORTAGE_BUILDDIR}/.instprepped ]] ; then + __vecho ">>> It appears that '$PF' is already instprepped; skipping." + __vecho ">>> Remove '${PORTAGE_BUILDDIR}/.instprepped' to force instprep." + return 0 + fi + + if has chflags ${FEATURES}; then + # Save all the file flags for restoration afterwards. + mtree -c -p "${ED}" -k flags > "${T}/bsdflags.mtree" + # Remove all the file flags so that we can do anything necessary. + chflags -R noschg,nouchg,nosappnd,nouappnd "${ED}" + chflags -R nosunlnk,nouunlnk "${ED}" 2>/dev/null + fi + + # If binpkg-docompress is disabled, we need to apply compression + # before installing. + if ! has binpkg-docompress ${FEATURES}; then + "${PORTAGE_BIN_PATH}"/ecompress --queue "${PORTAGE_DOCOMPRESS[@]}" + "${PORTAGE_BIN_PATH}"/ecompress --ignore "${PORTAGE_DOCOMPRESS_SKIP[@]}" + "${PORTAGE_BIN_PATH}"/ecompress --dequeue + fi + + if has chflags ${FEATURES}; then + # Restore all the file flags that were saved earlier on. + mtree -U -e -p "${ED}" -k flags < "${T}/bsdflags.mtree" &> /dev/null + fi + + >> "${PORTAGE_BUILDDIR}/.instprepped" || \ + die "Failed to create ${PORTAGE_BUILDDIR}/.instprepped" +} + preinst_qa_check() { postinst_qa_check preinst } diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh index 51b480bfb..d8ebf3d3e 100644 --- a/bin/phase-functions.sh +++ b/bin/phase-functions.sh @@ -288,7 +288,7 @@ __dyn_clean() { if [[ $EMERGE_FROM = binary ]] || ! has keepwork $FEATURES; then rm -f "$PORTAGE_BUILDDIR"/.{ebuild_changed,logid,pretended,setuped,unpacked,prepared} \ - "$PORTAGE_BUILDDIR"/.{configured,compiled,tested,packaged} \ + "$PORTAGE_BUILDDIR"/.{configured,compiled,tested,packaged,instprepped} \ "$PORTAGE_BUILDDIR"/.die_hooks \ "$PORTAGE_BUILDDIR"/.ipc_{in,out,lock} \ "$PORTAGE_BUILDDIR"/.exit_status diff --git a/cnf/make.globals b/cnf/make.globals index 04a708af8..72b567e98 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -50,7 +50,7 @@ RESUMECOMMAND_SSH=${FETCHCOMMAND_SSH} FETCHCOMMAND_SFTP="bash -c \"x=\\\${2#sftp://} ; host=\\\${x%%/*} ; port=\\\${host##*:} ; host=\\\${host%:*} ; [[ \\\${host} = \\\${port} ]] && port= ; eval \\\"declare -a ssh_opts=(\\\${3})\\\" ; exec sftp \\\${port:+-P \\\${port}} \\\"\\\${ssh_opts[@]}\\\" \\\"\\\${host}:/\\\${x#*/}\\\" \\\"\\\$1\\\"\" sftp \"\${DISTDIR}/\${FILE}\" \"\${URI}\" \"\${PORTAGE_SSH_OPTS}\"" # Default user options -FEATURES="assume-digests binpkg-logs +FEATURES="assume-digests binpkg-docompress binpkg-logs config-protect-if-modified distlocks ebuild-locks fixlafiles merge-sync multilib-strict news parallel-fetch preserve-libs protect-owned diff --git a/lib/portage/const.py b/lib/portage/const.py index 7f84bf0e9..a343fc040 100644 --- a/lib/portage/const.py +++ b/lib/portage/const.py @@ -122,6 +122,7 @@ EBUILD_PHASES = ( ) SUPPORTED_FEATURES = frozenset([ "assume-digests", + "binpkg-docompress", "binpkg-logs", "binpkg-multi-instance", "buildpkg", diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index c16fdfe88..fd8aaeb8e 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -3719,6 +3719,7 @@ class dblink(object): This function does the following: + calls doebuild(mydo=instprep) calls get_ro_checker to retrieve a function for checking whether Portage will write to a read-only filesystem, then runs it against the directory list calls self._preserve_libs if FEATURES=preserve-libs @@ -3768,6 +3769,17 @@ class dblink(object): level=logging.ERROR, noiselevel=-1) return 1 + # run instprep internal phase + doebuild_environment(myebuild, "instprep", + settings=self.settings, db=mydbapi) + phase = EbuildPhase(background=False, phase="instprep", + scheduler=self._scheduler, settings=self.settings) + phase.start() + if phase.wait() != os.EX_OK: + showMessage(_("!!! instprep failed\n"), + level=logging.ERROR, noiselevel=-1) + return 1 + is_binpkg = self.settings.get("EMERGE_FROM") == "binary" slot = '' for var_name in ('CHOST', 'SLOT'): diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py index 9706de422..d0e96f34c 100644 --- a/lib/portage/package/ebuild/doebuild.py +++ b/lib/portage/package/ebuild/doebuild.py @@ -656,6 +656,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, "compile":["configure"], "test": ["compile"], "install":["test"], + "instprep":["install"], "rpm": ["install"], "package":["install"], "merge" :["install"], @@ -674,7 +675,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, "config", "info", "setup", "depend", "pretend", "fetch", "fetchall", "digest", "unpack", "prepare", "configure", "compile", "test", - "install", "rpm", "qmerge", "merge", + "install", "instprep", "rpm", "qmerge", "merge", "package", "unmerge", "manifest", "nofetch"] if mydo not in validcommands: @@ -1402,6 +1403,7 @@ def _spawn_actionmap(settings): "compile": {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}}, "test": {"cmd":ebuild_sh, "args":{"droppriv":droppriv, "free":nosandbox, "sesandbox":sesandbox, "fakeroot":0}}, "install": {"cmd":ebuild_sh, "args":{"droppriv":0, "free":0, "sesandbox":sesandbox, "fakeroot":fakeroot}}, +"instprep": {"cmd":misc_sh, "args":{"droppriv":0, "free":0, "sesandbox":sesandbox, "fakeroot":fakeroot}}, "rpm": {"cmd":misc_sh, "args":{"droppriv":0, "free":0, "sesandbox":0, "fakeroot":fakeroot}}, "package": {"cmd":misc_sh, "args":{"droppriv":0, "free":0, "sesandbox":0, "fakeroot":fakeroot}}, } diff --git a/man/ebuild.1 b/man/ebuild.1 index 1d48844da..5893c3df6 100644 --- a/man/ebuild.1 +++ b/man/ebuild.1 @@ -130,6 +130,11 @@ the \fIsrc_install()\fR function. When completed, the will contain all the files that should either be merged to the local filesystem or included in a binary package. .TP +.BR instprep +Performs the additional post-install/pre-merge preparations inside +the temporary \fIinstall directory\fR. This is intended to be called +\fBafter\fR building binary package but before executing \fBpreinst\fR. +.TP .BR postinst Runs package-specific actions that need to be done after the package is installed into the live filesystem. Usually helpful messages are diff --git a/man/make.conf.5 b/man/make.conf.5 index a33929143..ec03c93ca 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -275,6 +275,12 @@ existing digest, the digest will be regenerated regardless of whether or not \fIassume\-digests\fR is enabled. The \fBebuild\fR(1) \fBdigest\fR command has a \fB\-\-force\fR option that can be used to force regeneration of digests. .TP +.B binpkg\-docompress +Perform \fBdocompress\fR (controllable file compression) before creating binary +package. When this option is enabled (the default), documentation files are +already compressed inside binary packages. When it is disabled, binary packages +contain uncompressed documentation and Portage compresses it before installing. +.TP .B binpkg\-logs Keep logs from successful binary package merges. This is relevant only when \fBPORT_LOGDIR\fR is set. -- 2.19.1