On Wed, 29 Jun 2022 09:16:17 +0200 Daniel Sahlberg <daniel.l.sahlb...@gmail.com> wrote:
> Den lör 25 juni 2022 kl 00:41 skrev Sergei Trofimovich <sly...@gmail.com>: > > > On Fri, 24 Jun 2022 15:14:23 -0400 > > Nathan Hartman <hartman.nat...@gmail.com> wrote: > > > > > On Thu, Jun 23, 2022 at 5:58 AM Sergei Trofimovich <sly...@gmail.com> > > wrote: > > > > > > > > Hi subversion developers! > > > > > > > > I'm working on fixing parallel build (or install) faliures in upstream > > > > packages used in NixOS linux distribution. Mostly to speed package > > builds > > > > up on continuous build farm. > > > > > > > > That usually means that > > > > $ ./configure && make && make install > > > > is changed to > > > > $ ./configure && make -j$(nproc) && make -j$(nproc) install > > > > > > > > To make the error more obvioud GNU make recently added --shuffle option > > > > to better expose missing dependencies across Makefile targets: > > > > https://savannah.gnu.org/bugs/index.php?62100 > > > > > > > > 'subversion-1.14.2' came as one of the packages that sometimes fails on > > > > 'make install' > > > > > > > > There is a build log snippet that exposed missing install dependencies: > > > > > > > > $ ./configure --disable-static --prefix=/<<NIX>>/subversion-1.14.2 > > --bindir=/<<NIX>>/subversion-1.14.2/bin > > --sbindir=/<<NIX>>/subversion-1.14.2/sbin > > --includedir=/<<NIX>>/subversion-1.14.2-dev/include > > --oldincludedir=/<<NIX>>/subversion-1.14.2-dev/include > > --mandir=/<<NIX>>/subversion-1.14.2-man/share/man > > --infodir=/<<NIX>>/subversion-1.14.2/share/info > > --docdir=/<<NIX>>/subversion-1.14.2/share/doc/subversion > > --libdir=/<<NIX>>/subversion-1.14.2/lib > > --libexecdir=/<<NIX>>/subversion-1.14.2/libexec > > --localedir=/<<NIX>>/subversion-1.14.2/share/locale --with-berkeley-db > > --without-apxs --without-swig --without-sasl > > --with-serf=/<<NIX>>/serf-1.3.9 --with-zlib=/<<NIX>>/zlib-1.2.12-dev > > --with-sqlite=/<<NIX>>/sqlite-3.38.5-dev --build=x86_64-unknown-linux-gnu > > --host=x86_64-unknown-linux-gnu > > > > ... > > > > $ make -j16 -l16 SHELL=/<<NIX>>/bash-5.1-p16/bin/bash > > APACHE_LIBEXECDIR=/<<NIX>>/subversion-1.14.2/modules > > > > ... > > > > $ make --shuffle > > APACHE_LIBEXECDIR=/<<NIX>>/subversion-1.14.2/modules > > pkgconfigdir=/<<NIX>>/subversion-1.14.2-dev/lib/pkgconfig > > m4datadir=/<<NIX>>/subversion-1.14.2-dev/share/aclocal > > aclocaldir=/<<NIX>>/subversion-1.14.2-dev/share/aclocal install > > > > test -d /<<NIX>>/subversion-1.14.2-dev/include/subversion-1 || \ > > > > /<<NIX>>/coreutils-9.1/bin/install -c -d > > /<<NIX>>/subversion-1.14.2-dev/include/subversion-1 > > > > (subversion/svnversion/svnversion . 2> /dev/null || \ > > > > svnversion . 2> /dev/null || \ > > > > echo "unknown"; \ > > > > ) > > > /<<NIX>>/subversion-1.14.2-dev/include/subversion-1/svn-revision.txt > > > > /<<NIX>>/coreutils-9.1/bin/install -c -d > > /<<NIX>>/subversion-1.14.2/lib /<<NIX>>/subversion-1.14.2/share/pkgconfig > > > > cd subversion/libsvn_fs_base ; /<<NIX>>/bash-5.1-p16/bin/bash > > "/build/subversion/libtool" --mode=install > > /<<NIX>>/coreutils-9.1/bin/install -c libsvn_fs_base-1.la > > /<<NIX>>/subversion-1.14.2/lib/libsvn_fs_base-1.la > > > > libtool: warning: relinking 'libsvn_fs_base-1.la' > > > > libtool: install: (cd /build/subversion/subversion/libsvn_fs_base; > > /<<NIX>>/bash-5.1-p16/bin/bash "/build/subversion/libtool" --tag CC > > --silent --mode=relink gcc -shared -g -O2 -g -O2 > > -L/<<NIX>>/openssl-1.1.1o/lib -L/<<NIX>>/db-5.3.28/lib > > -L/<<NIX>>/expat-2.4.8/lib -L/<<NIX>>/serf-1.3.9/lib -rpath > > /<<NIX>>/subversion-1.14.2/lib -version-info 0 -Wl,--no-undefined -o > > libsvn_fs_base-1.la bdb/bdb-err.lo bdb/bdb_compat.lo bdb/changes-table.lo > > bdb/checksum-reps-table.lo bdb/copies-table.lo bdb/dbt.lo bdb/env.lo > > bdb/lock-tokens-table.lo bdb/locks-table.lo bdb/miscellaneous-table.lo > > bdb/node-origins-table.lo bdb/nodes-table.lo bdb/reps-table.lo > > bdb/rev-table.lo bdb/strings-table.lo bdb/txn-table.lo bdb/uuids-table.lo > > dag.lo err.lo fs.lo id.lo key-gen.lo lock.lo node-rev.lo reps-strings.lo > > revs-txns.lo trail.lo tree.lo util/fs_skels.lo uuid.lo > > ../../subversion/libsvn_delta/libsvn_delta-1.la > > ../../subversion/libsvn_subr/libsvn_subr-1.la > > -L/<<NIX>>/apr-util-1.6.1/lib -laprutil-1 -L/<<N > > IX>>/apr-1.7.0/lib -lapr-1 -ldb-5.3 ../../subversion/libsvn_fs_util/ > > libsvn_fs_util-1.la ) > > > > /<<NIX>>/binutils-2.38/bin/ld: cannot find -lsvn_delta-1: No such > > file or directory > > > > /<<NIX>>/binutils-2.38/bin/ld: cannot find -lsvn_fs_util-1: No > > such file or directory > > > > collect2: error: ld returned 1 exit status > > > > libtool: error: error: relink 'libsvn_fs_base-1.la' with the > > above command before installing it > > > > make: *** [build-outputs.mk:1422: install-bdb-lib] Error 1 > > shuffle=1656015674 > > > > > > > > The failure as I understand it: > > > > > > > > 'install-bdb-lib' install target assumes that '-lsvn_delta-1' and > > '-lsvn_fs_util-1' > > > > are already installed into 'DESTDIR'. But they are not. 'libtool' > > fails to relink > > > > 'libsvn_fs_base' against 'DESTDIR'. > > > > > > > > I think the dependencies usually happen to be present if > > 'install-fsmod-lib' target > > > > was lucky to be executed first. But nothing in 'Makefile.in' or ' > > build-outputs.mk' > > > > seems to have an equivalent of dependency like: > > > > > > > > # hypothetical fix > > > > install-bdb-lib : install-fsmod-lib > > > > > > > > It looks like 'build.conf' does contain some build dependency > > information. > > > > > > > > I also see special cases like fs-lib to handle some of install deps: > > > > > > > > # manual > > > > Makefile.in:SVN_FS_LIB_INSTALL_DEPS = @SVN_FS_LIB_INSTALL_DEPS@ > > > > > > > > # generated: > > > > build-outputs.mk:install-ramod-lib: $(SVN_FS_LIB_INSTALL_DEPS) > > > > > > > > Would it make sense to add 'install-bdb-lib : install-fsmod-lib' > > dependency explicitly? > > > > > > > > Thank you! > > > > > > > > -- > > > > > > > > Sergei > > > > > > > > > Hi Sergei, > > > > > > This is a very useful feature for GNU Make to have! This is the first > > > I've heard of it and it should allow many, many projects, not just > > > Subversion, to fix these kinds of pesky issues in parallel builds. > > > > > > It seems like this landed in Make literally a few days ago so isn't in > > > any released version. Is that correct? > > > > That is correct: there are no releases or snapshots of GNU make with a > > feature yet. I'm using locally built make from tip of the git tree. > > > > > If so, I don't currently have a (convenient) setup to experiment with > > > this at the moment; are you able to just hack your hypothetical fix > > > into your SVN makefile, i.e., `install-bdb-lib : install-fsmod-lib`, > > > and run make with shuffle=1656015674 and tell us if that appears to > > > fix it? Then I can go digging to see where to add it correctly. > > > > Before the hack my installs were failing 60% of the time (6 of 10 attmpts > > failed). After the change below 100 of 100 install succeeded. > > > > I had to add two depdendencies to get there: > > > > install-bdb-lib: install-fsmod-lib > > install-serf-lib: install-fsmod-lib > > > > The change itself (with linkage errors they fix): > > > > $ svn diff > > Index: Makefile.in > > =================================================================== > > --- Makefile.in (revision 1902174) > > +++ Makefile.in (working copy) > > @@ -431,6 +431,36 @@ > > > > @INCLUDE_OUTPUTS@ > > > > +# A hack to test https://issues.apache.org/jira/browse/SVN-4901: > > +# orkaround missing dependencies on installing libtool libraries > > +# that depend on other libraries: > > +# > > +## case 1: fs_base depend on fs_util and delta: > > +# > > +# libtool: install: (cd /build/subversion/subversion/libsvn_fs_base; bash > > "/build/subversion/libtool" \ > > +# --tag CC --silent --mode=relink gcc -shared -g -O2 -g -O2 > > -L...-openssl-1.1.1p/lib \ > > +# -L...-db-5.3.28/lib -L...-expat-2.4.8/lib -L...-serf-1.3.9/lib > > -rpath ...-subversion-1.14.2/lib \ > > +# -version-info 0 -Wl,--no-undefined \ > > +# -o libsvn_fs_base-1.la bdb/bdb-err.lo \ > > +# ... ../../subversion/libsvn_delta/libsvn_delta-1.la > > ../../subversion/libsvn_subr/libsvn_subr-1.la \ > > +# -L...-apr-util-1.6.1/lib -laprutil-1 -L...-apr-1.7.0/lib -lapr-1 > > -ldb-5.3 \ > > +# ../../subversion/libsvn_fs_util/libsvn_fs_util-1.la ) > > +# ld: cannot find -lsvn_delta-1: No such file or directory > > +# ld: cannot find -lsvn_fs_util-1: No such file or directory > > +install-bdb-lib: install-fsmod-lib > > +# > > +## case 2: ra-serf depends on delta: > > +# > > +# libtool: install: (cd /build/subversion/subversion/libsvn_ra_serf; bash > > "/build/subversion/libtool" \ > > +# --tag CC --silent --mode=relink gcc -shared -g -O2 -g -O2 > > -L...-openssl-1.1.1p/lib \ > > +# -L...-db-5.3.28/lib -L...-expat-2.4.8/lib -L...-serf-1.3.9/lib > > -rpath ...-subversion-1.14.2/lib \ > > +# -version-info 0 -Wl,--no-undefined > > +# -o libsvn_ra_serf-1.la blame.lo \ > > +# ... xml.lo ../../subversion/libsvn_delta/libsvn_delta-1.la > > ../../subversion/libsvn_subr/libsvn_subr-1.la \ > > +# -L...-apr-util-1.6.1/lib -laprutil-1 -L...-apr-1.7.0/lib -lapr-1 > > -lserf-1 -L...-zlib-1.2.12-dev/lib -lz ) > > +# ld: cannot find -lsvn_delta-1: No such file or directory > > +install-serf-lib: install-fsmod-lib > > + > > local-all: @BUILD_RULES@ @TRANSFORM_LIBTOOL_SCRIPTS@ > > > > transform-libtool-scripts: @BUILD_RULES@ > > > > I have committed a fix in r1902335 adding install-fsmod-lib as a depencency > for install-bdb-lib. > > It was tested by Nathan and I believe it is correct, but I must admit I > have almost no experience in the build system. > > Sergei, can you make some tests on /trunk? Sure! Ran a few builds against current trunk. I don't see bdb-related failures any more \o/. But I still do see libsvn_ra_serf related failures: /<<NIX>>/coreutils-9.1/bin/install -c -d /<<NIX>>/subversion-1.14.2/lib /<<NIX>>/subversion-1.14.2/share/pkgconfig cd subversion/libsvn_ra_serf ; /<<NIX>>/bash-5.1-p16/bin/bash "/build/subversion/libtool" --mode=install /<<NIX>>/coreutils-9.1/bin/install -c libsvn_ra_serf-1.la /<<NIX>>/subversion-1.14.2/lib/libsvn_ra_serf-1.la libtool: warning: relinking 'libsvn_ra_serf-1.la' libtool: install: (cd /build/subversion/subversion/libsvn_ra_serf; /<<NIX>>/bash-5.1-p16/bin/bash "/build/subversion/libtool" --tag CC --silent --mode=relink gcc -shared -g -O2 -g -O2 -L/<<NIX>>/openssl-1.1.1p/lib -L/<<NIX>>/db-5.3.28/lib -L/<<NIX>>/expat-2.4.8/lib -L/<<NIX>>/serf-1.3.9/lib -rpath /<<NIX>>/subversion-1.14.2/lib -version-info 0 -Wl,--no-undefined -o libsvn_ra_serf-1.la blame.lo blncache.lo commit.lo eagain_bucket.lo get_deleted_rev.lo get_file.lo get_lock.lo getdate.lo getlocations.lo getlocationsegments.lo getlocks.lo inherited_props.lo list.lo lock.lo log.lo merge.lo mergeinfo.lo multistatus.lo options.lo property.lo replay.lo request_body.lo sb_bucket.lo serf.lo stat.lo stream_bucket.lo update.lo util.lo util_error.lo xml.lo ../../subversion/libsvn_delta/libsvn_delta-1.la ../../subversion/libsvn_subr/libsvn_subr-1.la -L/<<NIX>>/apr-util-1.6.1/lib -laprutil-1 -L/<<NIX>>/apr-1.7.0/lib -lapr-1 -lserf-1 -L/<<NIX>>/zlib-1.2.12-dev/lib -lz ) /<<NIX>>/binutils-2.38/bin/ld: cannot find -lsvn_delta-1: No such file or directory collect2: error: ld returned 1 exit status libtool: error: error: relink 'libsvn_ra_serf-1.la' with the above command before installing it Adding 'install-serf-lib: install-fsmod-lib' seems to work it around. I also tried the fix below on top of trunk. It fixes install failures for me as well: --- build.conf (revision 1902359) +++ build.conf (working copy) @@ -340,6 +340,7 @@ path = subversion/libsvn_ra_serf install = serf-lib libs = libsvn_delta libsvn_subr aprutil apriconv apr serf zlib +add-install-deps = $(SVN_FS_LIB_INSTALL_DEPS) msvc-static = yes # Accessing repositories via SVN -- Sergei
