On Sun, Aug 21, 2011 at 3:42 PM, Rafael Goncalves Martins
<rafaelmart...@gentoo.org> wrote:
> Patch reworked, after Zac's feedback on #-portage.
>
> Attached.
>
Re-sending. I forgot a print statement in the code. :(
Sorry.
--
Rafael Goncalves Martins
Gentoo Linux developer
http://rafaelmartins.eng.br/
From 3fe23a65638160c19abe991d9db6f023818c5395 Mon Sep 17 00:00:00 2001
From: "Rafael G. Martins" <rafaelmart...@gentoo.org>
Date: Sun, 21 Aug 2011 20:32:02 -0300
Subject: [PATCH] distpatch: basic implementation of distfile patching
support.
This patch allows Portage to call the tools from app-portage/distpatch
to reconstruct distfiles from binary deltas and validate them.
---
cnf/make.globals | 6 ++
man/make.conf.5 | 15 ++++
pym/portage/const.py | 6 +-
pym/portage/dbapi/porttree.py | 24 ++++++-
pym/portage/package/ebuild/doebuild.py | 3 +
pym/portage/package/ebuild/fetch.py | 116 +++++++++++++++++++++++++++-----
6 files changed, 149 insertions(+), 21 deletions(-)
diff --git a/cnf/make.globals b/cnf/make.globals
index 2892d50..33b93fc 100644
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -126,6 +126,12 @@ PORTAGE_ELOG_MAILFROM="portage@localhost"
# Signing command used by repoman
PORTAGE_GPG_SIGNING_COMMAND="gpg --sign --clearsign --yes --default-key \"\${PORTAGE_GPG_KEY}\" --homedir \"\${PORTAGE_GPG_DIR}\" \"\${FILE}\""
+# URL for distpatch deltas root url
+PORTAGE_DELTAS_ROOT_URL="http://soc.dev.gentoo.org/~rafaelmartins/";
+
+# distpatch command, to fetch deltas (depends on app-portage/distpatch)
+PORTAGE_DISTPATCH_COMMAND="distpatcher --db \"\${DELTADB}\" --root-url \"\${ROOT_URL}\" --output \"\${OUTPUT_DIR}\" --input \"\${INPUT_DIR}\" --distfile \${VERBOSE} \"\${FILE}\""
+
# *****************************
# ** DO NOT EDIT THIS FILE **
# ***************************************************
diff --git a/man/make.conf.5 b/man/make.conf.5
index d83258c..b19b38c 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -272,6 +272,11 @@ strangely configured Samba server (oplocks off, NFS re\-export). A tool
/usr/lib/portage/bin/clean_locks exists to help handle lock issues
when a problem arises (normally due to a crash or disconnect).
.TP
+.B distpatch
+Enable experimental support to Distfile Patching. It depends on
+\fBapp-portage/distpatch\fR. Partially based on GLEP 25.
+See: \fIhttp://www.gentoo.org/proj/en/infrastructure/distpatch/\fR
+.TP
.B ebuild\-locks
Use locks to ensure that unsandboxed ebuild phases never execute
concurrently. Also see \fIparallel\-install\fR.
@@ -652,6 +657,16 @@ matching files are excluded when the \fBPORTAGE_COMPRESS\fR command is
called. Regular expressions are supported and the match is performed only
against the portion of the file name which follows the last period character.
.TP
+\fBPORTAGE_DELTAS_ROOT_URL\fR = \fI"URL"\fR
+This variable contains the URL used to fetch deltas to be used by the
+\fBdistpatch\fR feature.
+.TP
+.B PORTAGE_DISTPATCH_COMMAND
+This variable contains the command used for reconstruct distfiles from old
+distfiles and deltas. It must contain the executable as well as the
+place\-holders \\${DELTADB}, \\${ROOT_URL}, \\${OUTPUT_DIR}, \\${INPUT_DIR},
+\\${VERBOSE} and \\${FILE}.
+.TP
.B PORTAGE_ELOG_CLASSES
.TP
.B PORTAGE_ELOG_SYSTEM
diff --git a/pym/portage/const.py b/pym/portage/const.py
index ecaa8f1..6b211f8 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -71,6 +71,8 @@ PRELINK_BINARY = "/usr/sbin/prelink"
INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env"
REPO_NAME_FILE = "repo_name"
REPO_NAME_LOC = "profiles" + "/" + REPO_NAME_FILE
+DELTADB_FILE = "delta.db"
+DELTAS_DIR = "deltas"
PORTAGE_PACKAGE_ATOM = "sys-apps/portage"
LIBC_PACKAGE_ATOM = "virtual/libc"
@@ -89,8 +91,8 @@ SUPPORTED_FEATURES = frozenset([
"allow-missing-manifests",
"assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy",
"ccache", "chflags", "collision-protect", "compress-build-logs",
- "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot",
- "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
+ "digest", "distcc", "distcc-pump", "distlocks", "distpatch", "ebuild-locks",
+ "fakeroot", "fail-clean", "fixpackages", "force-mirror", "getbinpkg",
"installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror",
"metadata-transfer", "mirror", "multilib-strict", "news",
"noauto", "noclean", "nodoc", "noinfo", "noman",
diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py
index bf8ecd9..75a5c8f 100644
--- a/pym/portage/dbapi/porttree.py
+++ b/pym/portage/dbapi/porttree.py
@@ -15,11 +15,13 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
'portage.util.listdir:listdir',
'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp',
+ 'subprocess',
)
from portage.cache import metadata_overlay, volatile
from portage.cache.cache_errors import CacheError
from portage.cache.mappings import Mapping
+from portage.const import DELTADB_FILE, DELTAS_DIR
from portage.dbapi import dbapi
from portage.exception import PortageException, \
FileNotFound, InvalidAtom, InvalidDependString, InvalidPackageName
@@ -588,7 +590,27 @@ class portdbapi(dbapi):
# into account? check checksums?
for myfile in myfiles:
try:
- fetch_size = int(checksums[myfile]["size"])
+ if "distpatch" in self.settings.features:
+ try:
+ remaining_size = subprocess.check_output([
+ "distpatchq",
+ "delta_fetch_size",
+ os.path.join(self.settings["DISTDIR"], DELTADB_FILE),
+ myfile,
+ self.settings["DISTDIR"],
+ os.path.join(self.settings["DISTDIR"], DELTAS_DIR),
+ ], env=self.settings.environ())
+ except subprocess.CalledProcessError:
+ fetch_size = int(checksums[myfile]["size"])
+ else:
+ try:
+ filesdict[myfile] = int(remaining_size)
+ except ValueError:
+ fetch_size = int(checksums[myfile]["size"])
+ else:
+ continue
+ else:
+ fetch_size = int(checksums[myfile]["size"])
except (KeyError, ValueError):
if debug:
writemsg(_("[bad digest]: missing %(file)s for %(pkg)s\n") % {"file":myfile, "pkg":mypkg})
diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py
index a710e09..19963bb 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -1037,6 +1037,9 @@ def _prepare_fake_distdir(settings, alist):
for x in alist:
symlink_path = os.path.join(edpath, x)
target = os.path.join(orig_distdir, x)
+ reconstructed_target = os.path.join(orig_distdir, "delta-reconstructed", x)
+ if "distpatch" in settings.features and os.path.exists(reconstructed_target):
+ target = reconstructed_target
try:
link_target = os.readlink(symlink_path)
except OSError:
diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py
index 5cbbf87..1948926 100644
--- a/pym/portage/package/ebuild/fetch.py
+++ b/pym/portage/package/ebuild/fetch.py
@@ -22,13 +22,14 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.package.ebuild.doebuild:doebuild_environment,' + \
'_doebuild_spawn',
'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
+ 'subprocess'
)
from portage import OrderedDict, os, selinux, _encodings, \
_shell_quote, _unicode_encode
from portage.checksum import hashfunc_map, perform_md5, verify_all
from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
- GLOBAL_CONFIG_PATH
+ GLOBAL_CONFIG_PATH, DELTADB_FILE, DELTAS_DIR
from portage.data import portage_gid, portage_uid, secpass, userpriv_groups
from portage.exception import FileNotFound, OperationNotPermitted, \
PortageException, TryAgain
@@ -96,6 +97,33 @@ def _spawn_fetch(settings, args, **kwargs):
return rval
+def _spawn_distpatch(settings, myfile, **kwargs):
+ phase_backup = settings.get('EBUILD_PHASE')
+ settings['EBUILD_PHASE'] = 'fetch'
+
+ # mangle distpatcher args
+ variables = {
+ "DELTADB": os.path.join(settings['DISTDIR'], DELTADB_FILE),
+ "ROOT_URL": settings.get('PORTAGE_DELTAS_ROOT_URL', 'mirror://gentoo/'),
+ "OUTPUT_DIR": settings['DISTDIR'],
+ "INPUT_DIR": os.path.join(settings['DISTDIR'], DELTAS_DIR),
+ "VERBOSE": "" if settings.get('PORTAGE_QUIET', False) else "--verbose",
+ "FILE": myfile,
+ }
+ args = shlex_split(settings['PORTAGE_DISTPATCH_COMMAND'])
+ args = [varexpand(x, mydict=variables) for x in args]
+
+ # run distpatcher
+ try:
+ rval = spawn(args, env=settings.environ(), **kwargs)
+ finally:
+ if phase_backup is None:
+ settings.pop('EBUILD_PHASE', None)
+ else:
+ settings['EBUILD_PHASE'] = phase_backup
+
+ return rval
+
_userpriv_test_write_file_cache = {}
_userpriv_test_write_cmd_script = ">> %(file_path)s 2>/dev/null ; rval=$? ; " + \
"rm -f %(file_path)s ; exit $rval"
@@ -572,6 +600,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
pruned_digests["size"] = size
myfile_path = os.path.join(mysettings["DISTDIR"], myfile)
+ myfile_reconstructed_path = os.path.join(mysettings["DISTDIR"], "delta-reconstructed", myfile)
has_space = True
has_space_superuser = True
file_lock = None
@@ -739,13 +768,30 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
raise
del e
+ have_myfile = False
+ have_distpatch = False
try:
mystat = os.stat(myfile_path)
except OSError as e:
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
+ else:
+ if 'distpatch' in mysettings.features:
+ try:
+ mystat = os.stat(myfile_reconstructed_path)
+ except OSError as _e:
+ if e.errno not in (errno.ENOENT, errno.ESTALE):
+ raise
+ del _e
+ else:
+ myfile_path = myfile_reconstructed_path
+ have_myfile = True
+ have_distpatch = True
del e
else:
+ have_myfile = True
+
+ if have_myfile:
try:
apply_secpass_permissions(
myfile_path, gid=portage_gid, mode=0o664, mask=0o2,
@@ -784,23 +830,38 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
verified_ok, reason = verify_all(
myfile_path, mydigests[myfile])
if not verified_ok:
- writemsg(_("!!! Previously fetched"
- " file: '%s'\n") % myfile, noiselevel=-1)
- writemsg(_("!!! Reason: %s\n") % reason[0],
- noiselevel=-1)
- writemsg(_("!!! Got: %s\n"
- "!!! Expected: %s\n") % \
- (reason[1], reason[2]), noiselevel=-1)
- if reason[0] == _("Insufficient data for checksum verification"):
- return 0
- if distdir_writable:
- temp_filename = \
- _checksum_failure_temp_file(
- mysettings["DISTDIR"], myfile)
- writemsg_stdout(_("Refetching... "
- "File renamed to '%s'\n\n") % \
- temp_filename, noiselevel=-1)
- else:
+ if not have_distpatch:
+ writemsg(_("!!! Previously fetched"
+ " file: '%s'\n") % myfile, noiselevel=-1)
+ writemsg(_("!!! Reason: %s\n") % reason[0],
+ noiselevel=-1)
+ writemsg(_("!!! Got: %s\n"
+ "!!! Expected: %s\n") % \
+ (reason[1], reason[2]), noiselevel=-1)
+ if reason[0] == _("Insufficient data for checksum verification"):
+ return 0
+ if distdir_writable:
+ temp_filename = \
+ _checksum_failure_temp_file(
+ mysettings["DISTDIR"], myfile)
+ writemsg_stdout(_("Refetching... "
+ "File renamed to '%s'\n\n") % \
+ temp_filename, noiselevel=-1)
+ else:
+ eout = EOutput()
+ eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
+ eout.ebegin("%s reconstructed file, verifying checksums using distpatch" % myfile)
+ myret = subprocess.call([
+ "distpatchq",
+ "delta_verify_checksums",
+ os.path.join(mysettings["DISTDIR"], DELTADB_FILE),
+ myfile,
+ mysettings["DISTDIR"],
+ ], env=mysettings.environ())
+ eout.eend(myret)
+ if myret == 0:
+ continue
+ if verified_ok:
eout = EOutput()
eout.quiet = \
mysettings.get("PORTAGE_QUIET", None) == "1"
@@ -937,6 +998,25 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
command_var = resumecommand_var
else:
#normal mode:
+ if "distpatch" in mysettings.features:
+ myret = _spawn_distpatch(mysettings, myfile)
+ if myret == 0:
+ reconstructed_dir = os.path.join(mysettings["DISTDIR"], "delta-reconstructed")
+ if os.path.isdir(reconstructed_dir) and myfile in os.listdir(reconstructed_dir):
+ eout = EOutput()
+ eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
+ eout.ebegin("%s reconstructed file, verifying checksums using distpatch" % myfile)
+ myret = subprocess.call([
+ "distpatchq",
+ "delta_verify_checksums",
+ os.path.join(mysettings["DISTDIR"], DELTADB_FILE),
+ myfile,
+ mysettings["DISTDIR"],
+ ], env=mysettings.environ())
+ eout.eend(myret)
+ if myret == 0:
+ fetched = 2
+ break
locfetch=fetchcommand
command_var = fetchcommand_var
writemsg_stdout(_(">>> Downloading '%s'\n") % \
--
1.7.6
