The final piece. A bit ugly because the changes in require_conf_file depend on nonlocal semantics in require_file_internal and in maybe_push_required_file. Oh well.
Cheers, Ralf Parallel automake --add-missing: serialized file installs. * automake.in (QUEUE_CONF_FILE, QUEUE_LOCATION, QUEUE_STRING): New serialization keys. ($required_conf_file_queue): New file global. (queue_required_conf_file, require_queued_conf_file): New functions, to queue and dequeue requirements for aux dir files. (require_conf_file): Enqueue if needed. (get_number_of_threads): Can do threads with --add-missing now. (handle_makefiles_threaded): Let worker threads enqueue, let master attend to queued requirements at the right time. * tests/parallel-am.test: Explain the purpose of the include chain used here. * tests/parallel-am2.test: Also cope with --add-missing. * tests/parallel-am3.test: New test, test absence of races with concurrent same-file installs stemming from --add-missing. * tests/Makefile.am: Adjust. diff --git a/automake.in b/automake.in index d7db627..0815773 100755 --- a/automake.in +++ b/automake.in @@ -280,6 +280,9 @@ use constant INTERNAL => new Automake::Location; # Serialization keys for message queues. use constant { QUEUE_MESSAGE => "msg", + QUEUE_CONF_FILE => "conf file", + QUEUE_LOCATION => "location", + QUEUE_STRING => "string" }; @@ -7475,13 +7478,80 @@ sub require_libsource_with_macro ($$$@) } } +# Queue to push require_conf_file requirements to. +my $required_conf_file_queue; + +# &queue_required_conf_file ($QUEUE, $KEY, $DIR, $WHERE, $MYSTRICT, @FILES) +# ------------------------------------------------------------------------- +sub queue_required_conf_file ($$$$@) +{ + my ($queue, $key, $dir, $where, $mystrict, @files) = @_; + my @serial_loc; + if (ref $where) + { + @serial_loc = (QUEUE_LOCATION, $where->serialize ()); + } + else + { + @serial_loc = (QUEUE_STRING, $where); + } + $queue->enqueue ($key, $dir, @serial_loc, $mystrict, 0 + @files, @files); +} + +# &require_queued_conf_file ($QUEUE) +# ---------------------------------- +sub require_queued_conf_file ($) +{ + my ($queue) = @_; + my $where; + my $dir = $queue->dequeue (); + my $loc_key = $queue->dequeue (); + if ($loc_key eq QUEUE_LOCATION) + { + $where = Automake::Location::deserialize ($queue); + } + elsif ($loc_key eq QUEUE_STRING) + { + $where = $queue->dequeue (); + } + else + { + prog_error "unexpected key $loc_key"; + } + my $mystrict = $queue->dequeue (); + my $nfiles = $queue->dequeue (); + my @files; + push @files, $queue->dequeue () + foreach (1 .. $nfiles); + + # Dequeuing happens outside of per-makefile context, so we have to + # set the variables used by require_file_internal and the functions + # it calls. Gross! + $relative_dir = $dir; + require_file_internal ($where, $mystrict, $config_aux_dir, @files); +} + # &require_conf_file ($WHERE, $MYSTRICT, @FILES) # ---------------------------------------------- -# Looks in configuration path, as specified by AC_CONFIG_AUX_DIR. +# Looks in configuration path, as specified by AC_CONFIG_AUX_DIR; +# worker threads may queue up the action to be serialized by the master. +# +# FIXME: this seriously relies on the semantics of require_file_internal +# and maybe_push_required_file, in that we exploit the fact that only the +# contents of the last handled output file may be impacted (which in turn +# is dealt with by the master thread). sub require_conf_file ($$@) { my ($where, $mystrict, @files) = @_; - require_file_internal ($where, $mystrict, $config_aux_dir, @files); + if (defined $required_conf_file_queue) + { + queue_required_conf_file ($required_conf_file_queue, QUEUE_CONF_FILE, + $relative_dir, $where, $mystrict, @files); + } + else + { + require_file_internal ($where, $mystrict, $config_aux_dir, @files); + } } @@ -8051,11 +8121,6 @@ sub get_number_of_threads { $nthreads = $max_threads; } - - # We cannot deal with --add-missing (yet). - $nthreads = 0 - if ($add_missing); - return $nthreads; } @@ -8066,6 +8131,7 @@ sub get_number_of_threads # worker threads push back everything that needs serialization: # * warning and (normal) error messages, for stable stderr output # order and content (avoiding duplicates, for example), +# * races when installing aux files (and respective messages), # * races when collecting aux files for distribution. # # The latter requires that the makefile that deals with the aux dir @@ -8101,9 +8167,11 @@ sub handle_makefiles_threaded ($) verb "handling $file"; my $queue = $msg_queues{$file}; setup_channel_queue ($queue, QUEUE_MESSAGE); + $required_conf_file_queue = $queue; handle_makefile ($file); $queue->enqueue (undef); setup_channel_queue (undef, undef); + $required_conf_file_queue = undef; } return $exit_code; }); @@ -8125,6 +8193,10 @@ sub handle_makefiles_threaded ($) { pop_channel_queue ($queue); } + elsif ($key eq QUEUE_CONF_FILE) + { + require_queued_conf_file ($queue); + } else { prog_error "unexpected key $key"; diff --git a/tests/Makefile.am b/tests/Makefile.am index 7c6d4a2..2526acf 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -451,6 +451,7 @@ output-order.test \ overrid.test \ parallel-am.test \ parallel-am2.test \ +parallel-am3.test \ parse.test \ percent.test \ percent2.test \ diff --git a/tests/parallel-am.test b/tests/parallel-am.test index d57e014..de86271 100755 --- a/tests/parallel-am.test +++ b/tests/parallel-am.test @@ -60,6 +60,8 @@ for i in $list; do mkdir sub$i echo > sub$i/Makefile.am done +# Use an include chain to cause a nontrivial location object to be +# serialized through a thread queue. echo 'include foo.am' >> sub7/Makefile.am echo 'include bar.am' > sub7/foo.am echo 'python_PYTHON = foo.py' > sub7/bar.am diff --git a/tests/parallel-am2.test b/tests/parallel-am2.test index b5a4ac7..b92b79e 100755 --- a/tests/parallel-am2.test +++ b/tests/parallel-am2.test @@ -62,15 +62,16 @@ $ACLOCAL # Generate expected output using non-threaded code. unset AUTOMAKE_JOBS +rm -f install-sh missing depcomp AUTOMAKE_fails --add-missing -AUTOMAKE_fails mv stderr expected AUTOMAKE_JOBS=5 export AUTOMAKE_JOBS for i in 1 2 3 4 5 6 7 8; do - AUTOMAKE_fails + rm -f install-sh missing depcomp + AUTOMAKE_fails --add-missing diff expected stderr done diff --git a/tests/parallel-am3.test b/tests/parallel-am3.test new file mode 100755 index 0000000..b3b0aac --- /dev/null +++ b/tests/parallel-am3.test @@ -0,0 +1,75 @@ +#! /bin/sh +# Copyright (C) 2008 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# Test parallel automake execution. + +# This tests: +# 3) normal automake output should be identical and ordered in the same way +# with --add-missing, even with concurrent file requirements, and the +# installation of aux files should be race-free, + +. ./defs || Exit 1 + +set -e + +cat > configure.in << 'END' +AC_INIT([parallel-am], [1.0]) +AC_CONFIG_AUX_DIR([build-aux]) +AM_INIT_AUTOMAKE +AC_PROG_CC +AM_PATH_LISPDIR +AM_PATH_PYTHON +AC_CONFIG_FILES([Makefile]) +END + +cat > Makefile.am << 'END' +SUBDIRS = +END + +list='1 2 3' +for i in $list; do + echo "AC_CONFIG_FILES([sub$i/Makefile])" >> configure.in + echo "SUBDIRS += sub$i" >> Makefile.am + mkdir sub$i + cat > sub$i/Makefile.am <<END +python_PYTHON = foo$i.py +lisp_LISP = foo$i.el +bin_PROGRAMS = p$i +END +done + +rm -f install-sh missing depcomp +mkdir build-aux + +$ACLOCAL + +# Generate expected output using the non-threaded code. +unset AUTOMAKE_JOBS +AUTOMAKE_run 0 --add-missing +mv stderr expected +mv Makefile.in Makefile.in.exp + +AUTOMAKE_JOBS=3 +export AUTOMAKE_JOBS + +for run in 1 2 3 4 5 6 7; do + rm -f build-aux/* sub*/Makefile.in + AUTOMAKE_run 0 --add-missing + diff stderr expected + diff Makefile.in Makefile.in.exp +done + +: