Hi! Jérémy Bobbio: > Here are four patches based on the current master (1e059955) that will > write files in deterministic order in the control and data archives. > File names are sorted by forking `sort` before being piped to `tar`.
Attached are the same patches rebased on the current master (29422bfd).
--
Lunar .''`.
lu...@debian.org : :Ⓐ : # apt-get install anarchism
`. `'`
`-
From 5d80504af3d8855f4fb584cd839ff9153ca76453 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lu...@debian.org>
Date: Tue, 27 Aug 2013 18:10:15 +0200
Subject: [PATCH 1/4] Ensure deterministic file order in data.tar.* files
---
dpkg-deb/build.c | 42 ++++++++++++++++++++++++++++++------------
lib/dpkg/dpkg.h | 1 +
2 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c
index 0b9cfb6..bbd0c79 100644
--- a/dpkg-deb/build.c
+++ b/dpkg-deb/build.c
@@ -170,19 +170,36 @@ file_info_list_free(struct file_info *fi)
static void
file_treewalk_feed(const char *dir, int fd_out)
{
- int pipefd[2];
- pid_t pid;
+ int sort_pipefd[2], find_pipefd[2];
+ pid_t sort_pid, find_pid;
struct file_info *fi;
struct file_info *symlist = NULL;
struct file_info *symlist_end = NULL;
- m_pipe(pipefd);
+ m_pipe(find_pipefd);
+ m_pipe(sort_pipefd);
+
+ sort_pid = subproc_fork();
+ if (sort_pid == 0) {
+ m_dup2(find_pipefd[0], 0);
+ close(find_pipefd[0]);
+ close(find_pipefd[1]);
+ m_dup2(sort_pipefd[1], 1);
+ close(sort_pipefd[0]);
+ close(sort_pipefd[1]);
+ if (setenv("LC_ALL", "C", 1 /* overwrite */))
+ ohshite(_("unable to setenv"));
+ execlp(SORT, "sort", "--zero-terminated", NULL);
+ ohshite(_("unable to execute %s (%s)"), "sort", SORT);
+ }
+ close(find_pipefd[0]);
+ close(sort_pipefd[1]);
- pid = subproc_fork();
- if (pid == 0) {
- m_dup2(pipefd[1], 1);
- close(pipefd[0]);
- close(pipefd[1]);
+ find_pid = subproc_fork();
+ if (find_pid == 0) {
+ m_dup2(find_pipefd[1], 1);
+ close(find_pipefd[0]);
+ close(find_pipefd[1]);
if (chdir(dir))
ohshite(_("failed to chdir to `%.255s'"), dir);
@@ -191,11 +208,11 @@ file_treewalk_feed(const char *dir, int fd_out)
"-print0", NULL);
ohshite(_("unable to execute %s (%s)"), "find", FIND);
}
- close(pipefd[1]);
+ close(find_pipefd[1]);
/* We need to reorder the files so we can make sure that symlinks
* will not appear before their target. */
- while ((fi = file_info_get(dir, pipefd[0])) != NULL) {
+ while ((fi = file_info_get(dir, sort_pipefd[0])) != NULL) {
if (S_ISLNK(fi->st.st_mode)) {
file_info_list_append(&symlist, &symlist_end, fi);
} else {
@@ -206,8 +223,9 @@ file_treewalk_feed(const char *dir, int fd_out)
}
}
- close(pipefd[0]);
- subproc_wait_check(pid, "find", 0);
+ close(sort_pipefd[0]);
+ subproc_wait_check(find_pid, "find", 0);
+ subproc_wait_check(sort_pid, "sort", 0);
for (fi = symlist; fi; fi = fi->next)
if (fd_write(fd_out, fi->fn, strlen(fi->fn) + 1) < 0)
diff --git a/lib/dpkg/dpkg.h b/lib/dpkg/dpkg.h
index 1dfef96..8bbad67 100644
--- a/lib/dpkg/dpkg.h
+++ b/lib/dpkg/dpkg.h
@@ -112,6 +112,7 @@ DPKG_BEGIN_DECLS
#define CAT "cat"
#define FIND "find"
#define DIFF "diff"
+#define SORT "sort"
#define FIND_EXPRSTARTCHARS "-(),!"
--
1.7.10.4
From 753454b4062e3d4c1f7d87057b8e9b1d80eb32f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lu...@debian.org>
Date: Fri, 17 Jan 2014 12:56:13 +0100
Subject: [PATCH 2/4] Extract the creation of the control tarball to a
dedicated function
---
dpkg-deb/build.c | 73 ++++++++++++++++++++++++++++++++----------------------
1 file changed, 43 insertions(+), 30 deletions(-)
diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c
index bbd0c79..9dff37f 100644
--- a/dpkg-deb/build.c
+++ b/dpkg-deb/build.c
@@ -234,6 +234,40 @@ file_treewalk_feed(const char *dir, int fd_out)
file_info_list_free(symlist);
}
+static void
+create_control_tar(const char *dir, struct compress_params *control_compress_params,
+ int gzfd)
+{
+ int p1[2];
+ pid_t c1, c2;
+
+ /* Fork a tar to package the control-section of the package. */
+ unsetenv("TAR_OPTIONS");
+ m_pipe(p1);
+ c1 = subproc_fork();
+ if (!c1) {
+ m_dup2(p1[1],1); close(p1[0]); close(p1[1]);
+ if (chdir(dir))
+ ohshite(_("failed to chdir to `%.255s'"), dir);
+ if (chdir(BUILDCONTROLDIR))
+ ohshite(_("failed to chdir to `%.255s'"), ".../DEBIAN");
+ execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL);
+ ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR);
+ }
+ close(p1[1]);
+
+ /* And run the compressor on our control archive. */
+
+ c2 = subproc_fork();
+ if (!c2) {
+ compress_filter(control_compress_params, p1[0], gzfd, _("compressing control member"));
+ exit(0);
+ }
+ close(p1[0]);
+ subproc_wait_check(c2, "gzip -9c", 0);
+ subproc_wait_check(c1, "tar -cf", 0);
+}
+
static const char *const maintainerscripts[] = {
PREINSTFILE,
POSTINSTFILE,
@@ -509,20 +543,15 @@ do_build(const char *const *argv)
arfd = creat(debar, 0644);
if (arfd < 0)
ohshite(_("unable to create `%.255s'"), debar);
- /* Fork a tar to package the control-section of the package. */
- unsetenv("TAR_OPTIONS");
- m_pipe(p1);
- c1 = subproc_fork();
- if (!c1) {
- m_dup2(p1[1],1); close(p1[0]); close(p1[1]);
- if (chdir(dir))
- ohshite(_("failed to chdir to `%.255s'"), dir);
- if (chdir(BUILDCONTROLDIR))
- ohshite(_("failed to chdir to `%.255s'"), ".../DEBIAN");
- execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL);
- ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR);
+
+ if (opt_uniform_compression) {
+ control_compress_params = compress_params;
+ } else {
+ control_compress_params.type = COMPRESSOR_TYPE_GZIP;
+ control_compress_params.strategy = COMPRESSOR_STRATEGY_NONE;
+ control_compress_params.level = -1;
}
- close(p1[1]);
+
/* Create a temporary file to store the control data in. Immediately
* unlink our temporary file so others can't mess with it. */
tfbuf = path_make_temp_template("dpkg-deb");
@@ -535,23 +564,7 @@ do_build(const char *const *argv)
tfbuf);
free(tfbuf);
- /* And run the compressor on our control archive. */
- if (opt_uniform_compression) {
- control_compress_params = compress_params;
- } else {
- control_compress_params.type = COMPRESSOR_TYPE_GZIP;
- control_compress_params.strategy = COMPRESSOR_STRATEGY_NONE;
- control_compress_params.level = -1;
- }
-
- c2 = subproc_fork();
- if (!c2) {
- compress_filter(&control_compress_params, p1[0], gzfd, _("compressing control member"));
- exit(0);
- }
- close(p1[0]);
- subproc_wait_check(c2, "gzip -9c", 0);
- subproc_wait_check(c1, "tar -cf", 0);
+ create_control_tar(dir, &control_compress_params, gzfd);
if (lseek(gzfd, 0, SEEK_SET))
ohshite(_("failed to rewind temporary file (%s)"), _("control member"));
--
1.7.10.4
From f4d44792a081b660fbd298289841c8ab47e212d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lu...@debian.org>
Date: Fri, 17 Jan 2014 12:58:15 +0100
Subject: [PATCH 3/4] Rename create_control_tar() variables to more meaningful
names
---
dpkg-deb/build.c | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c
index 9dff37f..de6230a 100644
--- a/dpkg-deb/build.c
+++ b/dpkg-deb/build.c
@@ -238,15 +238,15 @@ static void
create_control_tar(const char *dir, struct compress_params *control_compress_params,
int gzfd)
{
- int p1[2];
- pid_t c1, c2;
+ int tar_pipefd[2];
+ pid_t tar_pid, gzip_pid;
/* Fork a tar to package the control-section of the package. */
unsetenv("TAR_OPTIONS");
- m_pipe(p1);
- c1 = subproc_fork();
- if (!c1) {
- m_dup2(p1[1],1); close(p1[0]); close(p1[1]);
+ m_pipe(tar_pipefd);
+ tar_pid = subproc_fork();
+ if (!tar_pid) {
+ m_dup2(tar_pipefd[1],1); close(tar_pipefd[0]); close(tar_pipefd[1]);
if (chdir(dir))
ohshite(_("failed to chdir to `%.255s'"), dir);
if (chdir(BUILDCONTROLDIR))
@@ -254,18 +254,18 @@ create_control_tar(const char *dir, struct compress_params *control_compress_par
execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL);
ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR);
}
- close(p1[1]);
+ close(tar_pipefd[1]);
/* And run the compressor on our control archive. */
- c2 = subproc_fork();
- if (!c2) {
- compress_filter(control_compress_params, p1[0], gzfd, _("compressing control member"));
+ gzip_pid = subproc_fork();
+ if (!gzip_pid) {
+ compress_filter(control_compress_params, tar_pipefd[0], gzfd, _("compressing control member"));
exit(0);
}
- close(p1[0]);
- subproc_wait_check(c2, "gzip -9c", 0);
- subproc_wait_check(c1, "tar -cf", 0);
+ close(tar_pipefd[0]);
+ subproc_wait_check(gzip_pid, "gzip -9c", 0);
+ subproc_wait_check(tar_pid, "tar -cf", 0);
}
static const char *const maintainerscripts[] = {
--
1.7.10.4
From e771147ac08429a1229e78a4ef615ddcb150ab1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <lu...@debian.org>
Date: Tue, 27 Aug 2013 23:03:49 +0200
Subject: [PATCH 4/4] Also write control.tar.gz in deterministic order
---
dpkg-deb/build.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 55 insertions(+), 7 deletions(-)
diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c
index de6230a..821335a 100644
--- a/dpkg-deb/build.c
+++ b/dpkg-deb/build.c
@@ -238,32 +238,80 @@ static void
create_control_tar(const char *dir, struct compress_params *control_compress_params,
int gzfd)
{
- int tar_pipefd[2];
- pid_t tar_pid, gzip_pid;
+ int tar_pipefd[2], gzip_pipefd[2], sort_pipefd[2];
+ pid_t tar_pid, gzip_pid, sort_pid, find_pid;
/* Fork a tar to package the control-section of the package. */
unsetenv("TAR_OPTIONS");
m_pipe(tar_pipefd);
+ m_pipe(gzip_pipefd);
tar_pid = subproc_fork();
if (!tar_pid) {
- m_dup2(tar_pipefd[1],1); close(tar_pipefd[0]); close(tar_pipefd[1]);
+ m_dup2(tar_pipefd[0], 0);
+ close(tar_pipefd[0]);
+ close(tar_pipefd[1]);
+ m_dup2(gzip_pipefd[1], 1);
+ close(gzip_pipefd[0]);
+ close(gzip_pipefd[1]);
if (chdir(dir))
ohshite(_("failed to chdir to `%.255s'"), dir);
if (chdir(BUILDCONTROLDIR))
ohshite(_("failed to chdir to `%.255s'"), ".../DEBIAN");
- execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL);
+ execlp(TAR, "tar", "-cf", "-", "--format=gnu", "--null", "--no-unquote",
+ "-T", "-", "--no-recursion", NULL);
ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR);
}
- close(tar_pipefd[1]);
+ close(tar_pipefd[0]);
+ close(gzip_pipefd[1]);
/* And run the compressor on our control archive. */
gzip_pid = subproc_fork();
if (!gzip_pid) {
- compress_filter(control_compress_params, tar_pipefd[0], gzfd, _("compressing control member"));
+ close(tar_pipefd[1]);
+ compress_filter(control_compress_params, gzip_pipefd[0], gzfd, _("compressing control member"));
exit(0);
}
- close(tar_pipefd[0]);
+ close(gzip_pipefd[0]);
+
+ /* We pipe the filename to sort between find and tar to get deterministic
+ * builds. */
+ m_pipe(sort_pipefd);
+ sort_pid = subproc_fork();
+ if (!sort_pid) {
+ m_dup2(sort_pipefd[0], 0);
+ close(sort_pipefd[0]);
+ close(sort_pipefd[1]);
+ m_dup2(tar_pipefd[1], 1);
+ close(tar_pipefd[0]);
+ close(tar_pipefd[1]);
+ if (setenv("LC_ALL", "C", 1 /* overwrite */))
+ ohshite(_("unable to setenv"));
+ execlp(SORT, "sort", "--zero-terminated", NULL);
+ ohshite(_("unable to execute %s (%s)"), "sort", SORT);
+ }
+ close(sort_pipefd[0]);
+ close(tar_pipefd[1]);
+
+ /* All the pipes are set, now lets run find, and start feeding
+ * filenames to tar. */
+ find_pid = subproc_fork();
+ if (!find_pid) {
+ m_dup2(sort_pipefd[1], 1);
+ close(sort_pipefd[0]);
+ close(sort_pipefd[1]);
+ if (chdir(dir))
+ ohshite(_("failed to chdir to `%.255s'"), dir);
+ if (chdir(BUILDCONTROLDIR))
+ ohshite(_("failed to chdir to `%.255s'"), ".../DEBIAN");
+ execlp(FIND, "find", ".", "-path", "./" BUILDCONTROLDIR, "-prune", "-o",
+ "-print0", NULL);
+ ohshite(_("unable to execute %s (%s)"), "find", FIND);
+ }
+ close(sort_pipefd[1]);
+
+ subproc_wait_check(find_pid, "find", 0);
+ subproc_wait_check(sort_pid, "sort", 0);
subproc_wait_check(gzip_pid, "gzip -9c", 0);
subproc_wait_check(tar_pid, "tar -cf", 0);
}
--
1.7.10.4
signature.asc
Description: Digital signature
