lucy-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Marvin Humphrey <mar...@rectangular.com>
Subject [lucy-dev] Parallel compilation
Date Tue, 16 Nov 2010 02:34:12 GMT
Greets,

My laptop has two cores, but the Lucy build process is single threaded and
doesn't take advantage of the second processor.

I hacked up the patch below for trunk/perl/buildlib/Lucy/Build.pm to try to
speed things up.  It forks off a max of 4 child processes which compile up to
10 C files each.  Here are before-and-after results for "time ./Build code":

    BEFORE:
    real    2m37.562s
    user    2m18.278s
    sys     0m19.448s

    AFTER:
    real    1m49.056s
    user    2m16.673s
    sys     0m20.221s

A nice gain... However, the patch isn't ready for primetime because it doesn't
handle compilation failure gracefully, check for number of CPU cores, or fall
back to single-threaded mode when fork() isn't available.  

Any suggestions about the approach?  If not, I'll pursue this further and fix
the problems when I find another round tuit.

Marvin Humphrey


Index: buildlib/Lucy/Build.pm
===================================================================
--- buildlib/Lucy/Build.pm  (revision 1035418)
+++ buildlib/Lucy/Build.pm  (working copy)
@@ -71,6 +71,8 @@
 use Env qw( @PATH );
 use Fcntl;
 use Carp;
+use POSIX qw( WNOHANG );
+use Time::HiRes qw( sleep );
 
 BEGIN { unshift @PATH, curdir() }
 
@@ -491,7 +493,7 @@
     );
     my @objects;
 
-    # Compile C source files.
+    # Gather C source files, generate list of object files.
     my $c_files = [];
     push @$c_files, @{ $self->rscan_dir( $CORE_SOURCE_DIR,     qr/\.c$/ ) };
     push @$c_files, @{ $self->rscan_dir( $XS_SOURCE_DIR,       qr/\.c$/ ) };
@@ -502,16 +504,50 @@
         my $o_file = $c_file;
         $o_file =~ s/\.c/$Config{_o}/;
         push @objects, $o_file;
-        next if $self->up_to_date( $c_file, $o_file );
         $self->add_to_cleanup($o_file);
-        $cbuilder->compile(
-            source               => $c_file,
-            extra_compiler_flags => $self->extra_ccflags,
-            include_dirs         => \@include_dirs,
-            object_file          => $o_file,
-        );
     }
 
+    # Compile in multiple child processes to take advantage of multi-CPU
+    # machines.
+    my @children;
+    my $MAX_CHILDREN = 4;
+    for ( my $i = 0; $i < scalar @$c_files; $i += 10 ) {
+        my $pid = fork();
+        if ( !defined $pid ) {
+            die "Fork failed $!\n";
+        }
+        elsif ($pid) {
+            # Parent...
+            push( @children, $pid );
+        }
+        elsif ( $pid == 0 ) {
+            for ( my $j = $i; $j < $i + 10; $j++ ) {
+                # Child...
+                my $c_file = $c_files->[$j];
+                next unless $c_file;
+                my $o_file = $c_file;
+                $o_file =~ s/\.c/$Config{_o}/;
+                next if $self->up_to_date( $c_file, $o_file );
+
+                $cbuilder->compile(
+                    source               => $c_file,
+                    extra_compiler_flags => $self->extra_ccflags,
+                    include_dirs         => \@include_dirs,
+                    object_file          => $o_file,
+                );
+            }
+            exit(0);
+        }
+        while ( _active_kids( \@children ) >= $MAX_CHILDREN ) {
+            sleep(1);
+        }
+    }
+
+    # Wait for the last few compiles to finish.
+    foreach (@children) {
+        waitpid( $_, 0 );
+    }
+
     # .xs => .c
     my $perl_binding_c_file = "lib/Lucy.c";
     $self->add_to_cleanup($perl_binding_c_file);
@@ -571,6 +607,17 @@
     }
 }
 
+sub _active_kids {
+    my $pids = shift;
+    my $active = 0;
+    for my $pid (@$pids) {
+        my $status = waitpid($pid, WNOHANG);
+        next unless $status == 0;
+        $active++;
+    }
+    return $active;
+}
+
 sub ACTION_code {
     my $self = shift;
 


Mime
View raw message