Greets,
My laptop has two cores, but the Lucy build process is single threaded and
doesn't take advantage of the second processor.
I hacked up the patch below for trunk/perl/buildlib/Lucy/Build.pm to try to
speed things up. It forks off a max of 4 child processes which compile up to
10 C files each. Here are before-and-after results for "time ./Build code":
BEFORE:
real 2m37.562s
user 2m18.278s
sys 0m19.448s
AFTER:
real 1m49.056s
user 2m16.673s
sys 0m20.221s
A nice gain... However, the patch isn't ready for primetime because it doesn't
handle compilation failure gracefully, check for number of CPU cores, or fall
back to single-threaded mode when fork() isn't available.
Any suggestions about the approach? If not, I'll pursue this further and fix
the problems when I find another round tuit.
Marvin Humphrey
Index: buildlib/Lucy/Build.pm
===================================================================
--- buildlib/Lucy/Build.pm (revision 1035418)
+++ buildlib/Lucy/Build.pm (working copy)
@@ -71,6 +71,8 @@
use Env qw( @PATH );
use Fcntl;
use Carp;
+use POSIX qw( WNOHANG );
+use Time::HiRes qw( sleep );
BEGIN { unshift @PATH, curdir() }
@@ -491,7 +493,7 @@
);
my @objects;
- # Compile C source files.
+ # Gather C source files, generate list of object files.
my $c_files = [];
push @$c_files, @{ $self->rscan_dir( $CORE_SOURCE_DIR, qr/\.c$/ ) };
push @$c_files, @{ $self->rscan_dir( $XS_SOURCE_DIR, qr/\.c$/ ) };
@@ -502,16 +504,50 @@
my $o_file = $c_file;
$o_file =~ s/\.c/$Config{_o}/;
push @objects, $o_file;
- next if $self->up_to_date( $c_file, $o_file );
$self->add_to_cleanup($o_file);
- $cbuilder->compile(
- source => $c_file,
- extra_compiler_flags => $self->extra_ccflags,
- include_dirs => \...@include_dirs,
- object_file => $o_file,
- );
}
+ # Compile in multiple child processes to take advantage of multi-CPU
+ # machines.
+ my @children;
+ my $MAX_CHILDREN = 4;
+ for ( my $i = 0; $i < scalar @$c_files; $i += 10 ) {
+ my $pid = fork();
+ if ( !defined $pid ) {
+ die "Fork failed $!\n";
+ }
+ elsif ($pid) {
+ # Parent...
+ push( @children, $pid );
+ }
+ elsif ( $pid == 0 ) {
+ for ( my $j = $i; $j < $i + 10; $j++ ) {
+ # Child...
+ my $c_file = $c_files->[$j];
+ next unless $c_file;
+ my $o_file = $c_file;
+ $o_file =~ s/\.c/$Config{_o}/;
+ next if $self->up_to_date( $c_file, $o_file );
+
+ $cbuilder->compile(
+ source => $c_file,
+ extra_compiler_flags => $self->extra_ccflags,
+ include_dirs => \...@include_dirs,
+ object_file => $o_file,
+ );
+ }
+ exit(0);
+ }
+ while ( _active_kids( \...@children ) >= $MAX_CHILDREN ) {
+ sleep(1);
+ }
+ }
+
+ # Wait for the last few compiles to finish.
+ foreach (@children) {
+ waitpid( $_, 0 );
+ }
+
# .xs => .c
my $perl_binding_c_file = "lib/Lucy.c";
$self->add_to_cleanup($perl_binding_c_file);
@@ -571,6 +607,17 @@
}
}
+sub _active_kids {
+ my $pids = shift;
+ my $active = 0;
+ for my $pid (@$pids) {
+ my $status = waitpid($pid, WNOHANG);
+ next unless $status == 0;
+ $active++;
+ }
+ return $active;
+}
+
sub ACTION_code {
my $self = shift;
