Control: tags -1 + patch Hi!
Here are four patches based on the current master (1e059955) that will
write files in deterministic order in the control and data archives.
File names are sorted by forking `sort` before being piped to `tar`.
They have been tested with the test case previously sent and on bigger
packages.
--
Lunar .''`.
lu...@debian.org : :Ⓐ : # apt-get install anarchism
`. `'`
`-
From fd525d04c230d5bc85dcf544467dc5de8cab053c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lu...@debian.org>
Date: Tue, 27 Aug 2013 18:10:15 +0200
Subject: [PATCH 1/4] Ensure deterministic file order in data.tar.* files
---
dpkg-deb/build.c | 42 ++++++++++++++++++++++++++++++------------
lib/dpkg/dpkg.h | 1 +
2 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c
index 2871234..01b625b 100644
--- a/dpkg-deb/build.c
+++ b/dpkg-deb/build.c
@@ -170,19 +170,36 @@ file_info_list_free(struct file_info *fi)
static void
file_treewalk_feed(const char *dir, int fd_out)
{
- int pipefd[2];
- pid_t pid;
+ int sort_pipefd[2], find_pipefd[2];
+ pid_t sort_pid, find_pid;
struct file_info *fi;
struct file_info *symlist = NULL;
struct file_info *symlist_end = NULL;
- m_pipe(pipefd);
+ m_pipe(find_pipefd);
+ m_pipe(sort_pipefd);
+
+ sort_pid = subproc_fork();
+ if (sort_pid == 0) {
+ m_dup2(find_pipefd[0], 0);
+ close(find_pipefd[0]);
+ close(find_pipefd[1]);
+ m_dup2(sort_pipefd[1], 1);
+ close(sort_pipefd[0]);
+ close(sort_pipefd[1]);
+ if (setenv("LC_ALL", "C", 1 /* overwrite */))
+ ohshite(_("unable to setenv"));
+ execlp(SORT, "sort", "--zero-terminated", NULL);
+ ohshite(_("unable to execute %s (%s)"), "sort", SORT);
+ }
+ close(find_pipefd[0]);
+ close(sort_pipefd[1]);
- pid = subproc_fork();
- if (pid == 0) {
- m_dup2(pipefd[1], 1);
- close(pipefd[0]);
- close(pipefd[1]);
+ find_pid = subproc_fork();
+ if (find_pid == 0) {
+ m_dup2(find_pipefd[1], 1);
+ close(find_pipefd[0]);
+ close(find_pipefd[1]);
if (chdir(dir))
ohshite(_("failed to chdir to `%.255s'"), dir);
@@ -191,11 +208,11 @@ file_treewalk_feed(const char *dir, int fd_out)
"-print0", NULL);
ohshite(_("unable to execute %s (%s)"), "find", FIND);
}
- close(pipefd[1]);
+ close(find_pipefd[1]);
/* We need to reorder the files so we can make sure that symlinks
* will not appear before their target. */
- while ((fi = file_info_get(dir, pipefd[0])) != NULL) {
+ while ((fi = file_info_get(dir, sort_pipefd[0])) != NULL) {
if (S_ISLNK(fi->st.st_mode)) {
file_info_list_append(&symlist, &symlist_end, fi);
} else {
@@ -206,8 +223,9 @@ file_treewalk_feed(const char *dir, int fd_out)
}
}
- close(pipefd[0]);
- subproc_wait_check(pid, "find", 0);
+ close(sort_pipefd[0]);
+ subproc_wait_check(find_pid, "find", 0);
+ subproc_wait_check(sort_pid, "sort", 0);
for (fi = symlist; fi; fi = fi->next)
if (fd_write(fd_out, fi->fn, strlen(fi->fn) + 1) < 0)
diff --git a/lib/dpkg/dpkg.h b/lib/dpkg/dpkg.h
index fbdc73e..f816aa2 100644
--- a/lib/dpkg/dpkg.h
+++ b/lib/dpkg/dpkg.h
@@ -113,6 +113,7 @@ DPKG_BEGIN_DECLS
#define CAT "cat"
#define FIND "find"
#define DIFF "diff"
+#define SORT "sort"
#define FIND_EXPRSTARTCHARS "-(),!"
--
1.7.10.4
From 6190a30b1d8fef99801f1da8263a937884ce2fba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lu...@debian.org>
Date: Fri, 17 Jan 2014 12:56:13 +0100
Subject: [PATCH 2/4] Extract the creation of the control tarball to a
dedicated function
---
dpkg-deb/build.c | 73 ++++++++++++++++++++++++++++++++----------------------
1 file changed, 43 insertions(+), 30 deletions(-)
diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c
index 01b625b..ebc0572 100644
--- a/dpkg-deb/build.c
+++ b/dpkg-deb/build.c
@@ -234,6 +234,40 @@ file_treewalk_feed(const char *dir, int fd_out)
file_info_list_free(symlist);
}
+static void
+create_control_tar(const char *dir, struct compress_params *control_compress_params,
+ int gzfd)
+{
+ int p1[2];
+ pid_t c1, c2;
+
+ /* Fork a tar to package the control-section of the package. */
+ unsetenv("TAR_OPTIONS");
+ m_pipe(p1);
+ c1 = subproc_fork();
+ if (!c1) {
+ m_dup2(p1[1],1); close(p1[0]); close(p1[1]);
+ if (chdir(dir))
+ ohshite(_("failed to chdir to `%.255s'"), dir);
+ if (chdir(BUILDCONTROLDIR))
+ ohshite(_("failed to chdir to `%.255s'"), ".../DEBIAN");
+ execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL);
+ ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR);
+ }
+ close(p1[1]);
+
+ /* And run the compressor on our control archive. */
+
+ c2 = subproc_fork();
+ if (!c2) {
+ compress_filter(control_compress_params, p1[0], gzfd, _("compressing control member"));
+ exit(0);
+ }
+ close(p1[0]);
+ subproc_wait_check(c2, "gzip -9c", 0);
+ subproc_wait_check(c1, "tar -cf", 0);
+}
+
static const char *const maintainerscripts[] = {
PREINSTFILE,
POSTINSTFILE,
@@ -509,20 +543,15 @@ do_build(const char *const *argv)
arfd = creat(debar, 0644);
if (arfd < 0)
ohshite(_("unable to create `%.255s'"), debar);
- /* Fork a tar to package the control-section of the package. */
- unsetenv("TAR_OPTIONS");
- m_pipe(p1);
- c1 = subproc_fork();
- if (!c1) {
- m_dup2(p1[1],1); close(p1[0]); close(p1[1]);
- if (chdir(dir))
- ohshite(_("failed to chdir to `%.255s'"), dir);
- if (chdir(BUILDCONTROLDIR))
- ohshite(_("failed to chdir to `%.255s'"), ".../DEBIAN");
- execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL);
- ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR);
+
+ if (opt_uniform_compression) {
+ control_compress_params = compress_params;
+ } else {
+ control_compress_params.type = compressor_type_gzip;
+ control_compress_params.strategy = compressor_strategy_none;
+ control_compress_params.level = -1;
}
- close(p1[1]);
+
/* Create a temporary file to store the control data in. Immediately
* unlink our temporary file so others can't mess with it. */
tfbuf = path_make_temp_template("dpkg-deb");
@@ -535,23 +564,7 @@ do_build(const char *const *argv)
tfbuf);
free(tfbuf);
- /* And run the compressor on our control archive. */
- if (opt_uniform_compression) {
- control_compress_params = compress_params;
- } else {
- control_compress_params.type = compressor_type_gzip;
- control_compress_params.strategy = compressor_strategy_none;
- control_compress_params.level = -1;
- }
-
- c2 = subproc_fork();
- if (!c2) {
- compress_filter(&control_compress_params, p1[0], gzfd, _("compressing control member"));
- exit(0);
- }
- close(p1[0]);
- subproc_wait_check(c2, "gzip -9c", 0);
- subproc_wait_check(c1, "tar -cf", 0);
+ create_control_tar(dir, &control_compress_params, gzfd);
if (lseek(gzfd, 0, SEEK_SET))
ohshite(_("failed to rewind temporary file (%s)"), _("control member"));
--
1.7.10.4
From 2356ce6a9a9761ca9c62fb8dd8d39527a29cff68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lu...@debian.org>
Date: Fri, 17 Jan 2014 12:58:15 +0100
Subject: [PATCH 3/4] Rename create_control_tar() variables to more meaningful
names
---
dpkg-deb/build.c | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c
index ebc0572..fdaf253 100644
--- a/dpkg-deb/build.c
+++ b/dpkg-deb/build.c
@@ -238,15 +238,15 @@ static void
create_control_tar(const char *dir, struct compress_params *control_compress_params,
int gzfd)
{
- int p1[2];
- pid_t c1, c2;
+ int tar_pipefd[2];
+ pid_t tar_pid, gzip_pid;
/* Fork a tar to package the control-section of the package. */
unsetenv("TAR_OPTIONS");
- m_pipe(p1);
- c1 = subproc_fork();
- if (!c1) {
- m_dup2(p1[1],1); close(p1[0]); close(p1[1]);
+ m_pipe(tar_pipefd);
+ tar_pid = subproc_fork();
+ if (!tar_pid) {
+ m_dup2(tar_pipefd[1],1); close(tar_pipefd[0]); close(tar_pipefd[1]);
if (chdir(dir))
ohshite(_("failed to chdir to `%.255s'"), dir);
if (chdir(BUILDCONTROLDIR))
@@ -254,18 +254,18 @@ create_control_tar(const char *dir, struct compress_params *control_compress_par
execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL);
ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR);
}
- close(p1[1]);
+ close(tar_pipefd[1]);
/* And run the compressor on our control archive. */
- c2 = subproc_fork();
- if (!c2) {
- compress_filter(control_compress_params, p1[0], gzfd, _("compressing control member"));
+ gzip_pid = subproc_fork();
+ if (!gzip_pid) {
+ compress_filter(control_compress_params, tar_pipefd[0], gzfd, _("compressing control member"));
exit(0);
}
- close(p1[0]);
- subproc_wait_check(c2, "gzip -9c", 0);
- subproc_wait_check(c1, "tar -cf", 0);
+ close(tar_pipefd[0]);
+ subproc_wait_check(gzip_pid, "gzip -9c", 0);
+ subproc_wait_check(tar_pid, "tar -cf", 0);
}
static const char *const maintainerscripts[] = {
--
1.7.10.4
From 973bcc24ac6bfa3636d15a15fa2881dc13e6eba4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lu...@debian.org>
Date: Tue, 27 Aug 2013 23:03:49 +0200
Subject: [PATCH 4/4] Also write control.tar.gz in deterministic order
---
dpkg-deb/build.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++-------
1 file changed, 54 insertions(+), 7 deletions(-)
diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c
index fdaf253..1afe4ed 100644
--- a/dpkg-deb/build.c
+++ b/dpkg-deb/build.c
@@ -238,32 +238,79 @@ static void
create_control_tar(const char *dir, struct compress_params *control_compress_params,
int gzfd)
{
- int tar_pipefd[2];
- pid_t tar_pid, gzip_pid;
+ int tar_pipefd[2], gzip_pipefd[2], sort_pipefd[2];
+ pid_t tar_pid, gzip_pid, sort_pid, find_pid;
/* Fork a tar to package the control-section of the package. */
unsetenv("TAR_OPTIONS");
m_pipe(tar_pipefd);
+ m_pipe(gzip_pipefd);
tar_pid = subproc_fork();
if (!tar_pid) {
- m_dup2(tar_pipefd[1],1); close(tar_pipefd[0]); close(tar_pipefd[1]);
+ m_dup2(tar_pipefd[0], 0);
+ close(tar_pipefd[0]);
+ close(tar_pipefd[1]);
+ m_dup2(gzip_pipefd[1], 1);
+ close(gzip_pipefd[0]);
+ close(gzip_pipefd[1]);
if (chdir(dir))
ohshite(_("failed to chdir to `%.255s'"), dir);
if (chdir(BUILDCONTROLDIR))
ohshite(_("failed to chdir to `%.255s'"), ".../DEBIAN");
- execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL);
+ execlp(TAR, "tar", "-cf", "-", "--format=gnu", "--null", "-T", "-", "--no-recursion", NULL);
ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR);
}
- close(tar_pipefd[1]);
+ close(tar_pipefd[0]);
+ close(gzip_pipefd[1]);
/* And run the compressor on our control archive. */
gzip_pid = subproc_fork();
if (!gzip_pid) {
- compress_filter(control_compress_params, tar_pipefd[0], gzfd, _("compressing control member"));
+ close(tar_pipefd[1]);
+ compress_filter(control_compress_params, gzip_pipefd[0], gzfd, _("compressing control member"));
exit(0);
}
- close(tar_pipefd[0]);
+ close(gzip_pipefd[0]);
+
+ /* We pipe the filename to sort between find and tar to get deterministic
+ * builds. */
+ m_pipe(sort_pipefd);
+ sort_pid = subproc_fork();
+ if (!sort_pid) {
+ m_dup2(sort_pipefd[0], 0);
+ close(sort_pipefd[0]);
+ close(sort_pipefd[1]);
+ m_dup2(tar_pipefd[1], 1);
+ close(tar_pipefd[0]);
+ close(tar_pipefd[1]);
+ if (setenv("LC_ALL", "C", 1 /* overwrite */))
+ ohshite(_("unable to setenv"));
+ execlp(SORT, "sort", "--zero-terminated", NULL);
+ ohshite(_("unable to execute %s (%s)"), "sort", SORT);
+ }
+ close(sort_pipefd[0]);
+ close(tar_pipefd[1]);
+
+ /* All the pipes are set, now lets run find, and start feeding
+ * filenames to tar. */
+ find_pid = subproc_fork();
+ if (!find_pid) {
+ m_dup2(sort_pipefd[1], 1);
+ close(sort_pipefd[0]);
+ close(sort_pipefd[1]);
+ if (chdir(dir))
+ ohshite(_("failed to chdir to `%.255s'"), dir);
+ if (chdir(BUILDCONTROLDIR))
+ ohshite(_("failed to chdir to `%.255s'"), ".../DEBIAN");
+ execlp(FIND, "find", ".", "-path", "./" BUILDCONTROLDIR, "-prune", "-o",
+ "-print0", NULL);
+ ohshite(_("unable to execute %s (%s)"), "find", FIND);
+ }
+ close(sort_pipefd[1]);
+
+ subproc_wait_check(find_pid, "find", 0);
+ subproc_wait_check(sort_pid, "sort", 0);
subproc_wait_check(gzip_pid, "gzip -9c", 0);
subproc_wait_check(tar_pid, "tar -cf", 0);
}
--
1.7.10.4
signature.asc
Description: Digital signature
