pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1199327 - in /pig/trunk: CHANGES.txt test/e2e/pig/drivers/TestDriverPig.pm test/e2e/pig/tests/nightly.conf
Date Tue, 08 Nov 2011 16:23:51 GMT
Author: gates
Date: Tue Nov  8 16:23:50 2011
New Revision: 1199327

URL: http://svn.apache.org/viewvc?rev=1199327&view=rev
Log:
PIG-2352: e2e test harness' use of environment variables causes unintended effects between
tests

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
    pig/trunk/test/e2e/pig/tests/nightly.conf

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1199327&r1=1199326&r2=1199327&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Nov  8 16:23:50 2011
@@ -34,6 +34,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-2352: e2e test harness' use of environment variables causes unintended effects between
tests (gates)
+
 Release 0.10.0 - Unreleased
 
 INCOMPATIBLE CHANGES

Modified: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1199327&r1=1199326&r2=1199327&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Tue Nov  8 16:23:50 2011
@@ -84,6 +84,7 @@ sub replaceParameters
 #    $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
     $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
     $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
+    $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g;
 
     if ( $testCmd->{'hadoopSecurity'} eq "secure" ) { 
       $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g;
@@ -137,6 +138,11 @@ sub globalSetup
     IPC::Run::run(['mkdir', '-p', $globalHash->{'tmpPath'}], \undef, $log, $log) or 
         die "Cannot create temporary directory " . $globalHash->{'tmpPath'} .
         " " . "$ERRNO\n";
+
+    # Create the HDFS temporary directory
+    @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}");
+	print $log "Going to run " . join(" ", @cmd) . "\n";
+    IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'}
. ": $? - $!\n";
 }
 
 sub globalCleanup
@@ -146,6 +152,11 @@ sub globalCleanup
     IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or 
         warn "Cannot remove temporary directory " . $globalHash->{'tmpPath'} .
         " " . "$ERRNO\n";
+
+    # Cleanup the HDFS temporary directory
+    my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'fs', '-rmr', "tmp/$globalHash->{'runid'}");
+	print $log "Going to run " . join(" ", @cmd) . "\n";
+    IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'}
. ": $? - $!\n";
 }
 
 
@@ -329,11 +340,12 @@ sub getPigCmd($$$)
 
     if (defined($testCmd->{'java_params'})) {
 		$ENV{'PIG_OPTS'} = join(" ", @{$testCmd->{'java_params'}});
+    } else {
+        $ENV{'PIG_OPTS'} = undef;
     }
 
-
 	print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
-	print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . "\n";
+	print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to
" . $ENV{'PIG_OPTS'} . "\n";
     return @pigCmd;
 }
 

Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1199327&r1=1199326&r2=1199327&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Tue Nov  8 16:23:50 2011
@@ -3889,14 +3889,13 @@ store E into ':OUTPATH:';\, 
                     {
                         'num' => 1,
                         'pig' => "define bb BuildBloom('Hash.JENKINS_HASH', 'fixed', '128',
'3');
-                                fs -rmr :TMP:/mybloom_1;
                                 A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray,
age:int, gpa:double);
                                 B = filter A by name == 'alice allen';
                                 C = group B all;
                                 D = foreach C generate bb(B.name);
-                                store D into ':TMP:/mybloom_1';
+                                store D into ':HDFSTMP:/mybloom_1';
                                 exec;
-                                define bloom Bloom(':TMP:/mybloom_1');
+                                define bloom Bloom(':HDFSTMP:/mybloom_1');
                                 E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray,
age:int, gpa:double);
                                 F = filter E by bloom(name);
                                 store F into ':OUTPATH:';",
@@ -3908,14 +3907,13 @@ store E into ':OUTPATH:';\, 
                     }, {
                         'num' => 2,
                         'pig' => "define bb BuildBloom('Hash.MURMUR_HASH', 'fixed', '128',
'3');
-                                fs -rmr :TMP:/mybloom_2;
                                 A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray,
age:int, gpa:double);
                                 B = filter A by name == 'alice allen';
                                 C = group B all;
                                 D = foreach C generate bb(B.name);
-                                store D into ':TMP:/mybloom_2';
+                                store D into ':HDFSTMP:/mybloom_2';
                                 exec;
-                                define bloom Bloom(':TMP:/mybloom_2');
+                                define bloom Bloom(':HDFSTMP:/mybloom_2');
                                 E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray,
age:int, gpa:double);
                                 F = filter E by bloom(name);
                                 G = load ':INPATH:/singlefile/votertab10k'as (name:chararray,
age:int, reg:chararray, contrib:float);
@@ -3931,14 +3929,13 @@ store E into ':OUTPATH:';\, 
                     },{
                         'num' => 3,
                         'pig' => "define bb BuildBloom('Hash.JENKINS_HASH', '1', '0.0001');
-                                fs -rmr :TMP:/mybloom_3;
                                 A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray,
age:int, gpa:double);
                                 B = filter A by name == 'alice allen';
                                 C = group B all;
                                 D = foreach C generate bb(B.name);
-                                store D into ':TMP:/mybloom_3';
+                                store D into ':HDFSTMP:/mybloom_3';
                                 exec;
-                                define bloom Bloom(':TMP:/mybloom_3');
+                                define bloom Bloom(':HDFSTMP:/mybloom_3');
                                 E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray,
age:int, gpa:double);
                                 F = filter E by bloom(name);
                                 G = load ':INPATH:/singlefile/votertab10k'as (name:chararray,
age:int, reg:chararray, contrib:float);



Mime
View raw message