On Wed, Jan 11, 2017 at 12:55 PM, Andreas Beckmann <a...@debian.org> wrote:
> On 2017-01-11 12:44, Michael Stapelberg wrote:
>> Thanks for the suggestion. I considered this, but I’m worried this
>> might blow up the logfiles quite a bit for some packages.
>
> How much "logfile" data would that be? Installing texlive-full in sid
> with --install-recommends produces 460 kb, distupgrading it from stable
> (without recommends) 700 kb, and there are some packages producing more
> output than that.
I don’t expect the output to be more than a few kilobytes, so size is
probably indeed not a concern. Making the files available under a
clean URL that is independent of the test result remains a concern of
mine, though.
Attached you can find a first stab at implementing this feature. I
introduced a new protocol message to transfer base64-encoded arbitrary
binary data. This can easily be used to transfer other files in the
same spirit, should that become necessary in the future (it also seems
like the clean thing to do, even if we’re just talking about a single
file). The files are then made available at
/<section>/aux/<package>_<version>/<filename>, e.g.
/sid/aux/libva1_1.7.3-2/alternatives.tar.gz.
Looking forward to your feedback,
--
Best regards,
Michael
From db76ac889047d26d103caf2b3380417134964b67 Mon Sep 17 00:00:00 2001
From: Michael Stapelberg <stapelb...@debian.org>
Date: Wed, 11 Jan 2017 23:24:00 +0100
Subject: [PATCH] Export /var/lib/dpkg/alternatives tarball as aux file
---
README_server.txt | 11 +++++++++++
piuparts-master-backend.py | 13 +++++++++++++
piuparts-slave.py | 20 +++++++++++++++++++-
piuparts.py | 21 +++++++++++++++++++++
4 files changed, 64 insertions(+), 1 deletion(-)
diff --git a/README_server.txt b/README_server.txt
index 506d5149..f82450cb 100644
--- a/README_server.txt
+++ b/README_server.txt
@@ -242,6 +242,17 @@ Success: ok
Slave informs master it cannot test the desired version of a
package (perhaps it went away from the mirror?).
+----
+Command: aux <packagename> <packageversion> <filename>
+ base64-encoded file contents
+ .
+Success: ok
+----
+
+Slave transfers an auxiliary file (e.g. the /var/lib/dpkg/alternatives
+tarball) to the master.
+
+
----
Command: status
Success: ok <package-state>=<count> <package-state>=<count>...
diff --git a/piuparts-master-backend.py b/piuparts-master-backend.py
index 3e87c412..69b86786 100644
--- a/piuparts-master-backend.py
+++ b/piuparts-master-backend.py
@@ -32,6 +32,7 @@ import os
import fcntl
import time
import random
+import base64
from urllib2 import URLError
import piupartslib
@@ -142,6 +143,7 @@ class Master(Protocol):
"pass": self._pass,
"fail": self._fail,
"untestable": self._untestable,
+ "aux": self._aux,
# debug commands, unstable and undocumented interface
"_state": self._state,
@@ -367,6 +369,17 @@ class Master(Protocol):
% ("untestable", args[0], args[1]))
self._short_response("ok")
+ def _aux(self, command, args):
+ self._check_args(3, command, args)
+ encoded = self._read_long_part()
+ decoded = base64.b64decode(encoded)
+ destdir = os.path.join(self._section, "aux", args[0] + "_" + args[1])
+ if not os.path.isdir(destdir):
+ os.makedirs(destdir)
+ with open(os.path.join(destdir, args[2]), "wb") as f:
+ f.write(decoded)
+ self._short_response("ok")
+
# debug command
def _state(self, command, args):
self._check_args(1, command, args)
diff --git a/piuparts-slave.py b/piuparts-slave.py
index 9b147529..642ff30b 100644
--- a/piuparts-slave.py
+++ b/piuparts-slave.py
@@ -36,6 +36,7 @@ import fcntl
import random
import ConfigParser
import apt_pkg
+import base64
import piupartslib.conf
import piupartslib.packagesdb
@@ -270,6 +271,20 @@ class Slave:
if line != "ok\n":
raise MasterNotOK()
+ def send_aux(self, section, filename):
+ logging.info("Sending aux data for %s/%s" % (section, filename))
+ basename = os.path.basename(filename)
+ package, rest = basename.split("_", 1)
+ version = rest[:-len(".log")]
+ for filename in os.listdir("aux"):
+ self._writeline("aux", package, version, filename)
+ with open(os.path.join("aux", filename), "r") as f:
+ self._writeline(" " + base64.b64encode(f.self()))
+ writeline._read(".")
+ line = self._readline()
+ if line != "ok\n":
+ raise MasterNotOK()
+
def get_status(self, section):
self._writeline("status")
line = self._readline()
@@ -576,6 +591,9 @@ class Section:
self._slave.send_log(self._config.section, logdir, fullname)
os.remove(fullname)
+ if os.path.exists("aux"):
+ self._slave.send_aux(self._config.section, fullname)
+
if unreserve:
for logdir in ["new", "reserved"]:
for basename in os.listdir(logdir):
@@ -905,7 +923,7 @@ def create_file(filename, contents):
def main():
- setup_logging(logging.INFO, None)
+ setup_logging(logging.DEBUG, None)
signal(SIGHUP, sighup_handler)
# For supporting multiple architectures and suites, we take command-line
diff --git a/piuparts.py b/piuparts.py
index dbf2b18d..0e2bbe87 100644
--- a/piuparts.py
+++ b/piuparts.py
@@ -1157,6 +1157,22 @@ class Chroot:
else:
self.install_packages_by_name(packages, with_scripts=with_scripts, reinstall=reinstall)
+ if not settings.auxdir:
+ return
+
+ target_name = os.path.join(settings.auxdir, "alternatives.tar.gz")
+ print 'Exporting /var/lib/dpkg/alternatives for manpages.debian.org'
+ self.run(["tar", "czf", "/tmp/export.tar.gz", "-C", "/var/lib/dpkg/alternatives", "."])
+ source_name = self.relative("tmp/export.tar.gz")
+ try:
+ if not os.path.isdir(settings.auxdir):
+ os.makedirs(settings.auxdir)
+ shutil.copy(source_name, target_name)
+ except IOError as detail:
+ logging.error("Error copying %s to %s: %s" %
+ (source_name, target_name, detail))
+ panic()
+
def install_package_files(self, package_files, packages=None, with_scripts=False):
if packages and settings.testdebs_repo:
self.install_packages_by_name(packages, with_scripts=with_scripts)
@@ -2807,6 +2823,10 @@ def parse_command_line():
help="Write log file to FILENAME in addition to " +
"the standard output.")
+ parser.add_option("--auxdir", metavar="AUXDIR",
+ help="Write auxiliary output files (e.g. the tarball "
+ "containing /var/lib/dpkg/alternatives) to AUXDIR.")
+
parser.add_option("--list-installed-files",
action="store_true", default=False,
help="List files added to the chroot after the " +
@@ -2984,6 +3004,7 @@ def parse_command_line():
settings.defaults = opts.defaults
defaults = DefaultsFactory().new_defaults()
+ settings.auxdir = opts.auxdir
settings.tmpdir = opts.tmpdir
settings.keep_tmpdir = opts.keep_tmpdir
settings.single_changes_list = opts.single_changes_list
--
2.11.0
