Here is an updated version of this patch set.
I have removed the "dirty check" stuff. It didn't really work well/was
buggy under meson, and it failed mysteriously on the Linux CI tasks. So
let's just drop that functionality for now.
I have also added a more complete commit message and some more code
comments.
I have extracted the freebsd CI script fix into a separate patch (0002).
I think this is useful even if we don't take the full CI patch (0003).
About the 0003 patch: It seems useful in principle to test these things
continuously. The dist script runs about 10 seconds in each task, and
takes a bit of disk space for the artifacts. I'm not sure to what
degree this might bother someone.
From d7c11e243d46e7e4ca0c4d0397680c3cce003687 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Sun, 11 Feb 2024 23:58:42 +0100
Subject: [PATCH v4 1/3] make dist uses git archive
This changes "make dist" to directly use "git archive", rather than
the custom shell script it currently runs.
This is to make the creation of the distribution tarball more directly
traceable to the git repository. That is why we removed the "make
distprep" step.
"make dist" continues to produce a .gz and a .bz2 tarball as before.
The archives produced this way are deterministic and reproducible,
meaning for a given commit the result file should always be
bit-for-bit identical. The exception is that if you use a git version
older than 2.38.0, gzip records the platform in the archive, so you'd
get a different output on Windows vs. macOS vs. "UNIX" (everything
else). In git 2.38.0, this was changed so that everything is recorded
as "UNIX" now. This is just something to keep in mind. This issue is
specific to the gzip format, it does not affect other compression
formats.
Meson has its own distribution building command (meson dist), but we
are not using that at this point. The main problem is that the way
they have implemented it, it is not deterministic in the above sense.
Also, we want a "make" version for the time being. But the target
name "dist" in meson is reserved for that reason, so we call the
custom target "pgdist" (so call something like "meson compile -C build
pgdist").
Discussion:
https://www.postgresql.org/message-id/flat/40e80f77-a294-4f29-a16f-e21bc7bc75fc%40eisentraut.org
---
GNUmakefile.in | 35 +++++++++++++--------------------
meson.build | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 65 insertions(+), 22 deletions(-)
diff --git a/GNUmakefile.in b/GNUmakefile.in
index 4d8fc794bbb..4dc13a3e2ea 100644
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -87,29 +87,20 @@ update-unicode: | submake-generated-headers
submake-libpgport
distdir = postgresql-$(VERSION)
dummy = =install=
+GIT = git
+
dist: $(distdir).tar.gz $(distdir).tar.bz2
- rm -rf $(distdir)
-
-$(distdir).tar: distdir
- $(TAR) chf $@ $(distdir)
-
-.INTERMEDIATE: $(distdir).tar
-
-distdir-location:
- @echo $(distdir)
-
-distdir:
- rm -rf $(distdir)* $(dummy)
- for x in `cd $(top_srcdir) && find . \( -name CVS -prune \) -o \( -name
.git -prune \) -o -print`; do \
- file=`expr X$$x : 'X\./\(.*\)'`; \
- if test -d "$(top_srcdir)/$$file" ; then \
- mkdir "$(distdir)/$$file" && chmod 777 "$(distdir)/$$file"; \
- else \
- ln "$(top_srcdir)/$$file" "$(distdir)/$$file" >/dev/null 2>&1 \
- || cp "$(top_srcdir)/$$file" "$(distdir)/$$file"; \
- fi || exit; \
- done
- $(MAKE) -C $(distdir) distclean
+
+# Note: core.autocrlf=false is needed to avoid line-ending conversion
+# in case the environment has a different setting. Without this, a
+# tarball created on Windows might be different than on, and unusable
+# on, Unix machines.
+
+$(distdir).tar.gz:
+ $(GIT) -C $(srcdir) -c core.autocrlf=false archive --format tar.gz
--prefix $(distdir)/ HEAD -o $(abs_top_builddir)/$@
+
+$(distdir).tar.bz2:
+ $(GIT) -C $(srcdir) -c core.autocrlf=false -c
tar.tar.bz2.command='$(BZIP2) -c' archive --format tar.bz2 --prefix $(distdir)/
HEAD -o $(abs_top_builddir)/$@
distcheck: dist
rm -rf $(dummy)
diff --git a/meson.build b/meson.build
index c8fdfeb0ec3..7e233317075 100644
--- a/meson.build
+++ b/meson.build
@@ -3359,6 +3359,58 @@ run_target('help',
+###############################################################
+# Distribution archive
+###############################################################
+
+git = find_program('git', required: false, native: true)
+bzip2 = find_program('bzip2', required: false, native: true)
+
+distdir = meson.project_name() + '-' + meson.project_version()
+
+# Note: core.autocrlf=false is needed to avoid line-ending conversion
+# in case the environment has a different setting. Without this, a
+# tarball created on Windows might be different than on, and unusable
+# on, Unix machines.
+
+tar_gz = custom_target('tar.gz',
+ build_always_stale: true,
+ command: [git, '-C', '@SOURCE_ROOT@',
+ '-c', 'core.autocrlf=false',
+ 'archive',
+ '--format', 'tar.gz',
+ '--prefix', distdir + '/',
+ '-o', join_paths(meson.build_root(), '@OUTPUT@'),
+ 'HEAD', '.'],
+ install: false,
+ output: distdir + '.tar.gz',
+)
+
+if bzip2.found()
+ tar_bz2 = custom_target('tar.bz2',
+ build_always_stale: true,
+ command: [git, '-C', '@SOURCE_ROOT@',
+ '-c', 'core.autocrlf=false',
+ '-c', 'tar.tar.bz2.command="' + bzip2.path() + '" -c',
+ 'archive',
+ '--format', 'tar.bz2',
+ '--prefix', distdir + '/',
+ '-o', join_paths(meson.build_root(), '@OUTPUT@'),
+ 'HEAD', '.'],
+ install: false,
+ output: distdir + '.tar.bz2',
+ )
+else
+ tar_bz2 = custom_target('tar.bz2',
+ command: ['sh', '-c', 'exit 1'],
+ output: distdir + '.tar.bz2',
+ )
+endif
+
+alias_target('pgdist', [tar_gz, tar_bz2])
+
+
+
###############################################################
# The End, The End, My Friend
###############################################################
base-commit: a145f424d5248a09d766e8cb503b999290cb3b31
--
2.44.0
From 2ba3315c75ff5f2a0123e6e0bc258e8bb4ae2f3a Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Thu, 21 Mar 2024 09:19:50 +0100
Subject: [PATCH v4 2/3] ci: freebsd repartition script didn't copy .git
directory
We need a slightly different "cp" incantation to make sure top-level
"dot" files, such as ".git", are also copied.
This is relevant for example if a script wants to execute a git
command. This currently does not happen, but it has come up while
testing other patches.
Discussion:
https://www.postgresql.org/message-id/flat/40e80f77-a294-4f29-a16f-e21bc7bc75fc%40eisentraut.org
---
src/tools/ci/gcp_freebsd_repartition.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/tools/ci/gcp_freebsd_repartition.sh
b/src/tools/ci/gcp_freebsd_repartition.sh
index cc7f6bc35ef..3adb8fb88ec 100755
--- a/src/tools/ci/gcp_freebsd_repartition.sh
+++ b/src/tools/ci/gcp_freebsd_repartition.sh
@@ -23,4 +23,4 @@ du -hs $CIRRUS_WORKING_DIR
mv $CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR.orig
mkdir $CIRRUS_WORKING_DIR
mount -o noatime /dev/md1 $CIRRUS_WORKING_DIR
-cp -r $CIRRUS_WORKING_DIR.orig/* $CIRRUS_WORKING_DIR/
+cp -a $CIRRUS_WORKING_DIR.orig/ $CIRRUS_WORKING_DIR/
--
2.44.0
From 1e4aec24aceec0f1ef874c2a968ec9e9908e6405 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Thu, 21 Mar 2024 09:23:13 +0100
Subject: [PATCH v4 3/3] ci: Add dist building
XXX only for testing
---
.cirrus.tasks.yml | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index 1adfdfdd456..38c5cc54311 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -202,6 +202,11 @@ task:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
EOF
+ dist_script: |
+ su postgres -c 'meson compile -C build -v pgdist'
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
# if the server continues running, it often causes cirrus-ci to fail
# during upload, as it doesn't expect artifacts to change size
@@ -345,6 +350,10 @@ task:
make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
EOF
+ dist_script: su postgres -c "make dist -j${BUILD_JOBS}"
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_ac
@@ -403,6 +412,10 @@ task:
PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32
--num-processes ${TEST_JOBS}
EOF
+ dist_script: su postgres -c 'meson compile -C build-32 -v pgdist'
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
@@ -498,6 +511,10 @@ task:
ulimit -n 1024 # default is 256, pretty low
meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
+ dist_script: meson compile -C build -v pgdist
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"
@@ -569,6 +586,12 @@ task:
vcvarsall x64
meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
+ dist_script: |
+ vcvarsall x64
+ meson compile -C build -v pgdist
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
crashlog_artifacts:
@@ -629,6 +652,11 @@ task:
test_world_script: |
%BASH% -c "meson test %MTEST_ARGS% --num-processes %TEST_JOBS%"
+ dist_script: |
+ %BASH% -c "meson compile -C build -v pgdist"
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
crashlog_artifacts:
--
2.44.0
