pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1170769 [2/7] - in /pig/branches/branch-0.9: ./ test/e2e/harness/ test/e2e/harness/test/conf/ test/e2e/harness/test/tests/ test/e2e/pig/ test/e2e/pig/conf/ test/e2e/pig/deployers/ test/e2e/pig/drivers/ test/e2e/pig/macros/ test/e2e/pig/tes...
Date Wed, 14 Sep 2011 19:03:13 GMT
Modified: pig/branches/branch-0.9/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/drivers/TestDriverPig.pm?rev=1170769&r1=1170768&r2=1170769&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/branches/branch-0.9/test/e2e/pig/drivers/TestDriverPig.pm Wed Sep 14 19:03:12 2011
@@ -26,13 +26,14 @@ use IPC::Run; # don't do qw(run), it scr
 use Digest::MD5 qw(md5_hex);
 use Util;
 use File::Path;
+use Cwd;
 
 use strict;
 use English;
 
 our $className= "TestDriver";
 our @ISA = "$className";
-our $ROOT = (defined $ENV{'PIG_HARNESS_ROOT'} ? $ENV{'PIG_HARNESS_ROOT'} : die "ERROR: You must set environment variable PIG_HARNESS_ROOT\n");
+our $ROOT = (defined $ENV{'HARNESS_ROOT'} ? $ENV{'HARNESS_ROOT'} : die "ERROR: You must set environment variable HARNESS_ROOT\n");
 our $toolpath = "$ROOT/libexec/PigTest";
 
 my $passedStr  = 'passed';
@@ -48,47 +49,10 @@ sub new
     my $class = ref($proto) || $proto;
     my $self = $class->SUPER::new;
 
-    $self->{'exectype'} = "mapred"; # till we know better (in globalSetup())!
-    $self->{'ignore'} = "true";     # till we know better (in globalSetup())!
-
     bless($self, $class);
     return $self;
 }
 
-###############################################################################
-# This method has been copied over from TestDriver to make changes to
-# support skipping tests which do not match current execution mode
-# or which were marked as 'ignore'
-#
-#
-# Static function, can be used by test_harness.pl
-# Print the results so far, given the testStatuses hash.
-# @param testStatuses - reference to hash of test status results.
-# @param log - reference to file handle to print results to.
-# @param prefix - A title to prefix to the results
-# @returns nothing.
-#
-sub printResults
-{
-    my ($testStatuses, $log, $prefix) = @_;
-
-    my ($pass, $fail, $abort, $depend, $skipped) = (0, 0, 0, 0, 0);
-
-    foreach (keys(%$testStatuses)) {
-        ($testStatuses->{$_} eq $passedStr)  && $pass++;
-        ($testStatuses->{$_} eq $failedStr)  && $fail++;
-        ($testStatuses->{$_} eq $abortedStr) && $abort++;
-        ($testStatuses->{$_} eq $dependStr)  && $depend++;
-        ($testStatuses->{$_} eq $skippedStr) && $skipped++;
-    }   
-
-    my $msg = "$prefix, PASSED: $pass FAILED: $fail SKIPPED: $skipped ABORTED: $abort " .
-        "FAILED DEPENDENCY: $depend";
-    print $log "$msg\n";
-	print "$msg\r";
-}
-
-
 sub replaceParameters
 {
 ##!!! Move this to Util.pm
@@ -102,7 +66,7 @@ sub replaceParameters
     $cmd =~ s/:OUTPATH:/$outfile/g;
 
     # $ENV
-    $cmd =~ s/:PIGHARNESS:/$ENV{PIG_HARNESS_ROOT}/g;
+    $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g;
 
     # $testCmd
     $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
@@ -114,9 +78,9 @@ sub replaceParameters
     $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
     $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
     $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
-    $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
-    $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
-    $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
+#    $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
+#    $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
+#    $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
     $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
     $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
 
@@ -147,25 +111,9 @@ sub globalSetup
         $self->{'ignore'} = 'false';
     }
 
-    # if "-x local" was provided on the command line,
-    # it implies pig should be run in "local" mode -so 
-    # change input and output paths 
-    if(defined($globalHash->{'x'}) && $globalHash->{'x'} eq 'local')
-    {
-        $self->{'exectype'} = "local";
-        $globalHash->{'inpathbase'} = $globalHash->{'localinpathbase'};
-        $globalHash->{'outpathbase'} = $globalHash->{'localoutpathbase'};
-    }
     $globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/";
     $globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/";
 
-    # extract the current zebra.jar file path from the classpath
-    # and enter it in the hash for use in the substitution of :ZEBRAJAR:
-    my $zebrajar = $globalHash->{'cp'};
-    $zebrajar =~ s/zebra.jar.*/zebra.jar/;
-    $zebrajar =~ s/.*://;
-    $globalHash->{'zebrajar'} = $zebrajar;
-
     # add libexec location to the path
     if (defined($ENV{'PATH'})) {
         $ENV{'PATH'} = $globalHash->{'scriptPath'} . ":" . $ENV{'PATH'};
@@ -174,45 +122,55 @@ sub globalSetup
         $ENV{'PATH'} = $globalHash->{'scriptPath'};
     }
 
-    my @cmd = (Util::getBasePigCmd($globalHash), '-e', 'mkdir', $globalHash->{'outpath'});
-
-    if($self->{'exectype'} eq "local")
-    {
-        @cmd = ('mkdir', '-p', $globalHash->{'outpath'});
-    }
+    my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'});
 
 
-    print $log join(" ", @cmd);
-
-    if($self->{'exectype'} eq "mapred")
-    {
-       IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
-    }
-    else
-    {
-       IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create directory " . $globalHash->{'outpath'} . "\n";
-    }
+	print $log "Going to run " . join(" ", @cmd) . "\n";
+    IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
 
     IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or 
         die "Cannot create localpath directory " . $globalHash->{'localpath'} .
         " " . "$ERRNO\n";
+
+    # Create the temporary directory
+    IPC::Run::run(['mkdir', '-p', $globalHash->{'tmpPath'}], \undef, $log, $log) or 
+        die "Cannot create temporary directory " . $globalHash->{'tmpPath'} .
+        " " . "$ERRNO\n";
+}
+
+sub globalCleanup
+{
+    my ($self, $globalHash, $log) = @_;
+
+    IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or 
+        warn "Cannot remove temporary directory " . $globalHash->{'tmpPath'} .
+        " " . "$ERRNO\n";
 }
 
 
 sub runTest
 {
-    my ($self, $testCmd, $log, $copyResults) = @_;
+    my ($self, $testCmd, $log) = @_;
     my $subName  = (caller(0))[3];
 
+    # Check that we should run this test.  If the current execution type
+    # doesn't match the execonly flag, then skip this one.
+    if ($self->wrongExecutionMode($testCmd)) {
+        print $log "Skipping test $testCmd->{'group'}" . "_" .
+            $testCmd->{'num'} . " since it is executed only in " .
+            $testCmd->{'execonly'} . " mode and we are executing in " .
+            $testCmd->{'exectype'} . " mode.\n";
+        my %result;
+        return \%result;
+    }
+
     # Handle the various methods of running used in 
     # the original TestDrivers
 
     if ( $testCmd->{'pig'} && $self->hasCommandLineVerifications( $testCmd, $log) ) {
-       return $self->runPigCmdLine( $testCmd, $log, $copyResults );
+       return $self->runPigCmdLine( $testCmd, $log, 1);
     } elsif( $testCmd->{'pig'} ){
-       return $self->runPig( $testCmd, $log, $copyResults );
-   #} elsif(  $testCmd->{'pigsql'} ){
-   #   return $self->runPigSql( $testCmd, $log, $copyResults );
+       return $self->runPig( $testCmd, $log, 1);
     } elsif(  $testCmd->{'script'} ){
        return $self->runScript( $testCmd, $log );
     } else {
@@ -249,7 +207,7 @@ sub runPigCmdLine
     close(FH);
 
     # Build the command
-    my @baseCmd = Util::getBasePigCmd($testCmd);
+    my @baseCmd = $self->getPigCmd($testCmd, $log);
     my @cmd = @baseCmd;
 
     # Add option -l giving location for secondary logs
@@ -341,6 +299,43 @@ sub runScript
 }
 
 
+sub getPigCmd($$$)
+{
+    my ($self, $testCmd, $log) = @_;
+
+    my @pigCmd;
+
+    # set the PIG_CLASSPATH environment variable
+	my $pcp .= $testCmd->{'jythonjar'} if (defined($testCmd->{'jythonjar'}));
+    $pcp .= ":" . $testCmd->{'classpath'} if (defined($testCmd->{'classpath'}));
+    $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local");
+
+    # Set it in our current environment.  It will get inherited by the IPC::Run
+    # command.
+    $ENV{'PIG_CLASSPATH'} = $pcp;
+
+    @pigCmd = ("$testCmd->{'pigpath'}/bin/pig");
+
+    if (defined($testCmd->{'additionaljars'})) {
+        push(@pigCmd, '-Dpig.additional.jars='.$testCmd->{'additionaljars'});
+    }
+
+    if ($testCmd->{'exectype'} eq "local") {
+		push(@{$testCmd->{'java_params'}}, "-Xmx1024m");
+        push(@pigCmd, ("-x", "local"));
+    }
+
+    if (defined($testCmd->{'java_params'})) {
+		$ENV{'PIG_OPTS'} = join(" ", @{$testCmd->{'java_params'}});
+    }
+
+
+	print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
+	print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . "\n";
+    return @pigCmd;
+}
+
+
 
 sub runPig
 {
@@ -361,7 +356,7 @@ sub runPig
 
 
     # Build the command
-    my @baseCmd = Util::getBasePigCmd($testCmd);
+    my @baseCmd = $self->getPigCmd($testCmd, $log);
     my @cmd = @baseCmd;
 
     # Add option -l giving location for secondary logs
@@ -382,6 +377,7 @@ sub runPig
 
 
     # Run the command
+    print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n";
     print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n";
 
     IPC::Run::run(\@cmd, \undef, $log, $log) or
@@ -392,66 +388,35 @@ sub runPig
     # Get results from the command locally
     my $localoutfile;
     my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
-    my @SQLQuery = @{$testCmd->{'queries'}}; # here only used to determine if single-guery of multi-query
+    my $stores = $self->countStores($testCmd);
        
-    # mapreduce
-    if($self->{'exectype'} eq "mapred")
-    {
-        # single query
-        if ($#SQLQuery == 0) {
-            if ($copyResults) {
-              $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
-              $result{'originalOutput'} = "$localdir/out_original"; # populated by postProcessSingleOutputFile
-            } else {
-              $result{'output'} = "NO_COPY";
-            }
-        }
-        # multi query
-        else {
-            my @outfiles = ();
-            for (my $id = 1; $id <= ($#SQLQuery + 1); $id++) {
-                $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out/$id";
-                $localoutfile = $outfile . ".$id";
-
-                # Copy result file out of hadoop
-                my $testOut;
-                if ($copyResults) {
-                  $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log);
-                } else {
-                  $testOut = "NO_COPY";
-                }
-                push(@outfiles, $testOut);
-            }
-            ##!!! originalOutputs not set! Needed?
-            $result{'outputs'} = \@outfiles;
-        }
-    }
-    # local mode
-    else 
-    {
-        # single query
-        if ($#SQLQuery == 0) {
-            $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".dir";
-            mkdir $localdir;
+    # single query
+    if ($stores == 1) {
+        if ($copyResults) {
             $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
             $result{'originalOutput'} = "$localdir/out_original"; # populated by postProcessSingleOutputFile
-        } 
-        # multi query
-        else {
-            my @outfiles = ();
-            for (my $id = 1; $id <= ($#SQLQuery + 1); $id++) {
-                $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
-                mkdir $localdir;
-                $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out/$id";
-                mkdir $localdir;
-                $localoutfile = $outfile . ".$id";
+        } else {
+            $result{'output'} = "NO_COPY";
+        }
+    }
+    # multi query
+    else {
+        my @outfiles = ();
+        for (my $id = 1; $id <= ($stores); $id++) {
+            $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out/$id";
+            $localoutfile = $outfile . ".$id";
 
-                my $testRes = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log);
-                push(@outfiles, $testRes);
+            # Copy result file out of hadoop
+            my $testOut;
+            if ($copyResults) {
+              $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log);
+            } else {
+              $testOut = "NO_COPY";
             }
-            ##!!! originalOutputs not set!
-            $result{'outputs'} = \@outfiles;
+            push(@outfiles, $testOut);
         }
+        ##!!! originalOutputs not set! Needed?
+        $result{'outputs'} = \@outfiles;
     }
 
     # Compare doesn't get the testCmd hash, so I need to stuff the necessary
@@ -469,15 +434,12 @@ sub postProcessSingleOutputFile
     my ($self, $outfile, $localdir, $baseCmd, $testCmd, $log) = @_;
     my $subName  = (caller(0))[3];
 
-    # Copy to local if results on HDFS
-    if ($self->{'exectype'} eq "mapred") {
-        my @baseCmd = @{$baseCmd};
-        my @copyCmd = @baseCmd;
-        push(@copyCmd, ('-e', 'copyToLocal', $outfile, $localdir)); 
-        print $log "$0::$className::$subName INFO: Going to run pig command: @copyCmd\n";
+    my @baseCmd = @{$baseCmd};
+    my @copyCmd = @baseCmd;
+    push(@copyCmd, ('-e', 'copyToLocal', $outfile, $localdir)); 
+    print $log "$0::$className::$subName INFO: Going to run pig command: @copyCmd\n";
  
-       IPC::Run::run(\@copyCmd, \undef, $log, $log) or die "Cannot copy results from HDFS $outfile to $localdir\n";
-    }
+    IPC::Run::run(\@copyCmd, \undef, $log, $log) or die "Cannot copy results from HDFS $outfile to $localdir\n";
 
 
     # Sort the result if necessary.  Keep the original output in one large file.
@@ -485,14 +447,13 @@ sub postProcessSingleOutputFile
     
     # Build command to:
     # 1. Combine part files
-    my $fppCmd = 
-            ($self->{'exectype'} eq "mapred") ? "cat $localdir/map* $localdir/part* 2>/dev/null" : 
-            (-d $outfile )                    ? "cat $outfile/part* 2>/dev/null" :
-                                                "cat $outfile";
+    my $fppCmd = "cat $localdir/map* $localdir/part* 2>/dev/null";
     
     # 2. Standardize float precision
-    if (defined $testCmd->{'floatpostprocess'} && defined $testCmd->{'delimiter'}) {
-        $fppCmd .= " | $toolpath/floatpostprocessor.pl '" . $testCmd->{'delimiter'} . "'";
+    if (defined $testCmd->{'floatpostprocess'} &&
+            defined $testCmd->{'delimiter'}) {
+        $fppCmd .= " | $toolpath/floatpostprocessor.pl '" .
+            $testCmd->{'delimiter'} . "'";
     }
     
     $fppCmd .= " > $localdir/out_original";
@@ -515,68 +476,31 @@ sub generateBenchmark
 
     my %result;
 
-    my @SQLQuery = @{$testCmd->{'queries'}};
- 
-    if ($#SQLQuery == 0) {
-        my $outfile = $self->generateSingleSQLBenchmark($testCmd, $SQLQuery[0], undef, $log);
-        $result{'output'} = $outfile;
-    } else {
-        my @outfiles = ();
-        for (my $id = 0; $id < ($#SQLQuery + 1); $id++) {
-            my $sql = $SQLQuery[$id];
-            my $outfile = $self->generateSingleSQLBenchmark($testCmd, $sql, ($id+1), $log); 
-            push(@outfiles, $outfile);
-        }
-        $result{'outputs'} = \@outfiles;
-    }
-
-    return \%result;
-}
-
-sub generateSingleSQLBenchmark
-{
-    my ($self, $testCmd, $sql, $id, $log) = @_;
-
-    my $qmd5 = substr(md5_hex($testCmd->{'pig'}), 0, 5);
-    my $outfile = $testCmd->{'benchmarkPath'} . "/" . $testCmd->{'group'} . "_" . $testCmd->{'num'};
-    $outfile .= defined($id) ? ".$id" . ".expected." . $qmd5 :  ".expected." . $qmd5;
-    
-    if (-e $outfile) {
-        return $outfile;
-    }
-
-    my @cmd = ('psql', '-U', $testCmd->{'dbuser'}, '-d', $testCmd->{'dbdb'},
-        '-c', $sql, '-t', '-A', '--pset', "fieldsep=	", '-o', $outfile);
-
-    print $log "Running SQL command [" . join(" ", @cmd) . "\n";
-    IPC::Run::run(\@cmd, \undef, $log, $log) or do {
-        print $log "Sql command <" . $sql .
-            " failed for >>$testCmd->{group}_$testCmd->{num}<<\n";
-        unlink $outfile if ( -e $outfile  );
-    
-        die "Sql command failed for >>$testCmd->{group}_$testCmd->{num}<<\n";
-    };
-    
-    # Sort and postprocess the result if necessary
-    my $sortCmd = "cat $outfile";
-    if (defined $testCmd->{'floatpostprocess'} && defined $testCmd->{'delimiter'}) {
-        $sortCmd .= " | $toolpath/floatpostprocessor.pl '" . $testCmd->{'delimiter'} . "' ";
-    }
-    
-    # Use system not IPC run so that the '*' gets interpolated by the shell.
-    $sortCmd .= " | sort";
-    if (defined $testCmd->{'sortBenchmarkArgs'}) {
-        $sortCmd .= " " . join(" ", @{$testCmd->{'sortBenchmarkArgs'}});
-    }
-    my $tmpfile = $outfile . ".tmp";
-    $sortCmd .= " > $tmpfile";
-    print $log "$sortCmd\n";
-    system($sortCmd);
-    unlink $outfile;
-
-    IPC::Run::run ['mv', $tmpfile, $outfile];
-
-    return $outfile;
+    # Check that we should run this test.  If the current execution type
+    # doesn't match the execonly flag, then skip this one.
+    if ($self->wrongExecutionMode($testCmd)) {
+        return \%result;
+    }
+
+    if ($self->hasCommandLineVerifications($testCmd, $log)) {
+        # Do nothing, no benchmark to geneate
+        return \%result;
+    }
+
+    # If they specified an alternate Pig Latin script, use that on the current
+    # version.  Otherwise use a previous version of Pig.
+	my %modifiedTestCmd = %{$testCmd};
+	if (defined $testCmd->{'verify_pig_script'}) {
+		$modifiedTestCmd{'pig'} = $testCmd->{'verify_pig_script'};
+	}
+    else {
+		# Change so we're looking at the old version of Pig
+		$modifiedTestCmd{'pigpath'} = $testCmd->{'oldpigpath'};
+	}
+	# Modify the test number so we don't run over the actual test output
+	# and logs
+	$modifiedTestCmd{'num'} = $testCmd->{'num'} . "_benchmark";
+	return $self->runPig(\%modifiedTestCmd, $log, 1);
 }
 
 sub hasCommandLineVerifications
@@ -598,6 +522,13 @@ sub compare
     my ($self, $testResult, $benchmarkResult, $log, $testCmd) = @_;
     my $subName  = (caller(0))[3];
 
+    # Check that we should run this test.  If the current execution type
+    # doesn't match the execonly flag, then skip this one.
+    if ($self->wrongExecutionMode($testCmd)) {
+        # Special magic value
+        return $self->{'wrong_execution_mode'}; 
+    }
+
     # For now, if the test has 
     # - testCmd pig, and 'sql' for benchmark, then use comparePig, i.e. using benchmark
     # - any verification directives formerly used by CmdLine or Script drivers (rc, regex on out and err...)
@@ -605,21 +536,14 @@ sub compare
     # - testCmd script, then use compareScript
     # - testCmd pig, and none of the above, then use comparePig
     #
-    # (The first condition included because nightly.conf actually had a test, Types_2, that 
-    # specified both 'sql' and 'expected_err_regex'.)
-    #
     # Later, should add ability to have same tests both verify with the 'script' directives, 
     # and do a benchmark compare, if it was a pig cmd. E.g. 'rc' could still be checked when 
     # doing the benchmark compare.
 
-    if( defined $testCmd->{'sql'} ){
-       return $self->comparePig ( $testResult, $benchmarkResult, $log, $testCmd);
-    } elsif( $testCmd->{'script'} || $self->hasCommandLineVerifications( $testCmd, $log) ){
+    if ( $testCmd->{'script'} || $self->hasCommandLineVerifications( $testCmd, $log) ){
        return $self->compareScript ( $testResult, $log, $testCmd);
     } elsif( $testCmd->{'pig'} ){
        return $self->comparePig ( $testResult, $benchmarkResult, $log, $testCmd);
-   #} elsif( $testCmd->{'pigsql'} ){
-   #   return $self->comparePigSql ( $testResult, $benchmarkResult, $log, $testCmd);
     } else {
        # Should have been caught by runTest, still...
        print $log "$0.$subName WARNING Did not find a testCmd that I know how to handle\n";
@@ -633,6 +557,7 @@ sub compareScript
     my ($self, $testResult, $log, $testCmd) = @_;
     my $subName  = (caller(0))[3];
 
+
     # IMPORTANT NOTES:
     #
     # If you are using a regex to compare stdout or stderr
@@ -663,7 +588,7 @@ sub compareScript
       print $log "$0::$subName INFO Checking test stdout' " .
               "as exact match against expected <$testCmd->{'expected_out'}>\n";
       if ($testResult->{'stdout'} ne $testCmd->{'expected_out'}) {
-        print $log "$0::$subName INFO Check failed: exact match of <$testCmd->{'expected_out'}> expected in stdout: $testResult->{'output'}\n";
+        print $log "$0::$subName INFO Check failed: exact match of <$testCmd->{'expected_out'}> expected in stdout: $testResult->{'stdout'}\n";
         $result = 0;
       }
     } 
@@ -672,7 +597,7 @@ sub compareScript
       print $log "$0::$subName INFO Checking test stdout " .
               "as NOT exact match against expected <$testCmd->{'expected_out'}>\n";
       if ($testResult->{'stdout'} eq $testCmd->{'not_expected_out'}) {
-        print $log "$0::$subName INFO Check failed: NON-match of <$testCmd->{'expected_out'}> expected to stdout: $testResult->{'output'}\n";
+        print $log "$0::$subName INFO Check failed: NON-match of <$testCmd->{'expected_out'}> expected to stdout: $testResult->{'stdout'}\n";
         $result = 0;
       }
     } 
@@ -681,7 +606,7 @@ sub compareScript
       print $log "$0::$subName INFO Checking test stdout " .
               "for regular expression <$testCmd->{'expected_out_regex'}>\n";
       if ($testResult->{'stdout'} !~ $testCmd->{'expected_out_regex'}) {
-        print $log "$0::$subName INFO Check failed: regex match of <$testCmd->{'expected_out_regex'}> expected in stdout: $testResult->{'output'}\n";
+        print $log "$0::$subName INFO Check failed: regex match of <$testCmd->{'expected_out_regex'}> expected in stdout: $testResult->{'stdout'}\n";
         $result = 0;
       }
     } 
@@ -690,7 +615,7 @@ sub compareScript
       print $log "$0::$subName INFO Checking test stdout " .
               "for NON-match of regular expression <$testCmd->{'not_expected_out_regex'}>\n";
       if ($testResult->{'stdout'} =~ $testCmd->{'not_expected_out_regex'}) {
-        print $log "$0::$subName INFO Check failed: regex NON-match of <$testCmd->{'not_expected_out_regex'}> expected in stdout: $testResult->{'output'}\n";
+        print $log "$0::$subName INFO Check failed: regex NON-match of <$testCmd->{'not_expected_out_regex'}> expected in stdout: $testResult->{'stdout'}\n";
         $result = 0;
       }
     } 
@@ -736,35 +661,25 @@ sub compareScript
 }
 
 
-# sub comparePigSql
-# {
-#     my ($self, $testResult, $benchmarkResult, $testCmd, $log) = @_;
-#     my $subName  = (caller(0))[3];
-# 
-#     my $result;
-# ##!!! STUB
-# }
-
-
 sub comparePig
 {
     my ($self, $testResult, $benchmarkResult, $log, $testCmd) = @_;
     my $subName  = (caller(0))[3];
 
     my $result;
-    my @SQLQuery = @{$testCmd->{'queries'}};
+    my $stores = $self->countStores($testCmd);
     
-    if ($#SQLQuery == 0) {
+    if ($stores == 1) {
         $result = $self->compareSingleOutput($testResult, $testResult->{'output'},
                 $benchmarkResult->{'output'}, $log);
     } else {
         my $res = 0;
-        for (my $id = 0; $id < ($#SQLQuery + 1); $id++) {
+        for (my $id = 0; $id < ($stores); $id++) {
             my $testOutput = ($testResult->{'outputs'})->[$id];
             my $benchmarkOutput = ($benchmarkResult->{'outputs'})->[$id];
             $res += $self->compareSingleOutput($testResult, $testOutput,
                                                $benchmarkOutput, $log);
-            $result = ($res == ($#SQLQuery + 1)) ? 1 : 0;
+            $result = ($res == ($stores)) ? 1 : 0;
         }
     }
 
@@ -776,6 +691,8 @@ sub compareSingleOutput
 {
     my ($self, $testResult, $testOutput, $benchmarkOutput, $log) = @_;
 
+print $log "testResult: $testResult testOutput: $testOutput benchmarkOutput: $benchmarkOutput\n";
+
     # cksum the the two files to see if they are the same
     my ($testChksm, $benchmarkChksm);
     IPC::Run::run((['cat', $testOutput], '|', ['cksum']), \$testChksm,
@@ -792,7 +709,7 @@ sub compareSingleOutput
         print $log "Test output checksum does not match benchmark checksum\n";
         print $log "Test checksum = <$testChksm>\n";
         print $log "Expected checksum = <$benchmarkChksm>\n";
-        print $log "RESULTS DIFFER: vimdiff $testOutput $benchmarkOutput\n";
+        print $log "RESULTS DIFFER: vimdiff " . cwd . "/$testOutput " . cwd . "/$benchmarkOutput\n";
     } else {
         $result = 1;
     }
@@ -815,298 +732,43 @@ sub compareSingleOutput
     return $result;
 }
 
-###############################################################################
-# This method has been copied over from TestDriver to make changes to
-# support skipping tests which do not match current execution mode
-#
-#
-# Run all the tests in the configuration file.
-# @param testsToRun - reference to array of test groups and ids to run
-# @param testsToMatch - reference to array of test groups and ids to match.
-# If a test group_num matches any of these regular expressions it will be run.
-# @param cfg - reference to contents of cfg file
-# @param log - reference to a stream pointer for the logs
-# @param dbh - reference database connection
-# @param testStatuses- reference to hash of test statuses
-# @param confFile - config file name
-# @param startat - test to start at.
-# @returns nothing
-# failed.
+##############################################################################
+# Count the number of stores in a Pig Latin script, so we know how many files
+# we need to compare.
 #
-sub run
+sub countStores($$)
 {
-    my ($self, $testsToRun, $testsToMatch, $cfg, $log, $dbh, $testStatuses,
-        $confFile, $startat, $logname ) = @_;
-    my $subName  = (caller(0))[3];
-
-    my $msg="";
-    my $duration=0;
-    my $totalDuration=0;
-    my $groupDuration=0;
+    my ($self, $testCmd) = @_;
 
-
-    my $sawstart = !(defined $startat);
-    # Rather than make each driver handle our multi-level cfg, we'll flatten
-    # the hashes into one for it.
-    my %globalHash;
-
-    my $runAll = ((scalar(@$testsToRun) == 0) && (scalar(@$testsToMatch) == 0));
-
-    # Read the global keys
-    foreach (keys(%$cfg)) {
-        next if $_ eq 'groups';
-        $globalHash{$_} = $cfg->{$_};
+    # Special work around for queries with more than one store that are not
+    # actually multiqueries.
+    if (defined $testCmd->{'notmq'}) {
+        return 1;
     }
 
-    # Do the global setup
-    $self->globalSetup(\%globalHash, $log);
-
-    my $report=0;
-    my $properties= new Properties( 0, $globalHash{'propertiesFile'} );
+    my $count;
 
-    # For the xml directory, use the default directory from the configuration file
-    # unless the directory was specified in the command line
-
-    my $xmlDir= $globalHash{'localxmlpathbase'} ."/run".  $globalHash{'UID'};
-    if ( $globalHash{'reportdir'} ) {
-        $xmlDir = $globalHash{'reportdir'};
-    } 
-
-    my %groupExecuted;
-
-    foreach my $group (@{$cfg->{'groups'}}) {
-        my %groupHash = %globalHash;
-        $groupHash{'group'} = $group->{'name'};
-
-        # Read the group keys
-        foreach (keys(%$group)) {
-            next if $_ eq 'tests';
-            $groupHash{$_} = $group->{$_};
-        }
-
-        # Run each test
-        foreach my $test (@{$group->{'tests'}}) {
-            # Check if we're supposed to run this one or not.
-            if (!$runAll) {
-                # check if we are supposed to run this test or not.
-                my $foundIt = 0;
-                foreach (@$testsToRun) {
-                    if (/^$groupHash{'group'}(_[0-9]+)?$/) {
-                        if (not defined $1) {
-                            # In this case it's just the group name, so we'll
-                            # run every test in the group
-                            $foundIt = 1;
-                            last;
-                        } else {
-                            # maybe, it at least matches the group
-                            my $num = "_" . $test->{'num'};
-                            if ($num eq $1) {
-                                $foundIt = 1;
-                                last;
-                            }
-                        }
-                    }
-                }
-                foreach (@$testsToMatch) {
-                    my $protoName = $groupHash{'group'} . "_" .  $test->{'num'};
-                    if ($protoName =~ /$_/) {
-                        if (not defined $1) {
-                            # In this case it's just the group name, so we'll
-                            # run every test in the group
-                            $foundIt = 1;
-                            last;
-                        } else {
-                            # maybe, it at least matches the group
-                            my $num = "_" . $test->{'num'};
-                            if ($num eq $1) {
-                                $foundIt = 1;
-                                last;
-                            }
-                        }
-                    }
-                }
-
-                next unless $foundIt;
-            }
-
-            # This is a test, so run it.
-            my %testHash = %groupHash;
-            foreach (keys(%$test)) {
-                $testHash{$_} = $test->{$_};
-            }
-            my $testName = $testHash{'group'} . "_" . $testHash{'num'};
-
-            if ( $groupExecuted{ $group->{'name'} }== 0 ){
-               $groupExecuted{ $group->{'name'} }=1;
-
-               mkpath( [ $xmlDir ] , 1, 0777) if ( ! -e $xmlDir );
-
-               my $filename = $group->{'name'}.".xml";
-               $report = new TestReport ( $properties, "$xmlDir/$filename" );
-               $report->purge();
-
-            }
-
-            # Have we not reached the starting point yet?
-            if (!$sawstart) {
-                if ($testName eq $startat) {
-                    $sawstart = 1;
-                } else {
-                    next;
-                }
-            }
-
-            # Check that this test doesn't depend on an earlier test or tests
-            # that failed, or that the test wasn't marked as "ignore".
-            # Don't abort if that test wasn't run, just assume the
-            # user knew what they were doing and set it up right.
-            my $skipThisOne = 0;
-            foreach (keys(%testHash)) {
-                if (/^depends_on/ && defined($testStatuses->{$testHash{$_}}) &&
-                        $testStatuses->{$testHash{$_}} ne $passedStr) {
-                    print $log "Skipping test $testName, it depended on " .
-                        "$testHash{$_} which returned a status of " .
-                        "$testStatuses->{$testHash{$_}}\n";
-                    $testStatuses->{$testName} = $dependStr;
-                    $skipThisOne = 1;
-                    last;
-                }
-                # if the test is not applicable to current execution mode
-                # ignore it
-                if(/^exectype$/i && $testHash{$_} !~ /$self->{'exectype'}/i)
-                {
-                    print $log "\nIGNORING test since running mode ($self->{'exectype'}) and exectype in test ($testHash{'exectype'}) do not match\n";
-                    $testStatuses->{$testName} = $skippedStr;
-                    $skipThisOne = 1;
-                    last;
-                }
-
-                # if the test is marked as 'ignore',
-                # ignore it... unless option to ignore the ignore is in force
-                if(/^ignore$/i && ($self->{'ignore'} eq 'true'))
-                {
-                    print $log "$0::$className::$subName TEST IGNORED <$testName> at " . time . ". Message: $testHash{'ignore'}\n";
-                    $testStatuses->{$testName} = $skippedStr;
-                    $skipThisOne = 1;
-                    last;
-                }
-            }
-
-            if ($skipThisOne) {
-                printResults($testStatuses, $log, "Results so far");
-                next;
-            }
-
-            # Check if output comparison should be skipped.
-            my $dontCompareThisOne = 0; # true for tests with key 'noverify'
-            my $copyResults        = 1; # no need to copy output to local if noverify
-            foreach (keys(%testHash)) {
-
-                if(/^noverify$/i )
-                {
-                    $dontCompareThisOne = 1;
-                    $copyResults = 0;
-                    last;
-                }
-            }
-
-            print $log "Beginning test $testName at " . time . "\n";
-            my %dbinfo = (
-                'testrun_id' => $testHash{'trid'},
-                'test_type'  => $testHash{'driver'},
-               #'test_file'  => $testHash{'file'},
-                'test_file'  => $confFile,
-                'test_group' => $testHash{'group'},
-                'test_num'   => $testHash{'num'},
-            );
-            my $beginTime = time;
-            my ($testResult, $benchmarkResult);
-            my $msg;
-            eval {
-                
-
-                my  @SQLQuery = split /;/, $testHash{'sql'};
-
-                # Throw out the last one if it is just space
-                if ($SQLQuery[$#SQLQuery] =~ /^\s*$/) { $#SQLQuery--; }
-
-                # If the last one is a comment, decrement the count
-                if ($#SQLQuery > 0 && $SQLQuery[$#SQLQuery] !~ /select/i && $SQLQuery[$#SQLQuery] =~ /--/) {
-                    $#SQLQuery--;
-                }
-
-                $testHash{'queries'} = \@SQLQuery;
-
-                $testResult = $self->runTest(\%testHash, $log, $copyResults);
-                my $endTime = time;
-
-                $benchmarkResult = $self->generateBenchmark(\%testHash, $log);
-
-                my $result;
-                if( $dontCompareThisOne ) {
-                    $result = 1;
-                    print $log "$0::$className::$subName TEST MARKED NOVERIFY <$testName>\n";
-                } else {
-                    $result = $self->compare($testResult, $benchmarkResult, $log, \%testHash);
-                    print $log "Test $testName\n";
-                }
-
-                if ($result) {
-                        $msg = "TEST SUCCEEDED <$testName> at " . time . "\n";
-                        $testStatuses->{$testName} = $passedStr;
-                } else {
-                        $msg= "TEST FAILED <$testName> at " . time . "\n";
-                        $testStatuses->{$testName} = $failedStr;
-                }
-                print $log $msg;
-
-                $dbinfo{'duration'} = $endTime - $beginTime;
-                $self->recordResults($result, $testResult, $benchmarkResult,
-                    \%dbinfo, $log);
-            };
-
-            if ($@) {
-                my $endTime = time;
-                print $log "$0::$subName FAILED: Failed to run test $testName <$@>\n";
-                $testStatuses->{$testName} = $abortedStr;
-                $dbinfo{'duration'} = $beginTime - $endTime;
-            }
-
-            eval {
-                $dbinfo{'status'} = $testStatuses->{$testName};
-                if($dbh) {
-                    $dbh->insertTestCase(\%dbinfo);
-                }
-            };
-            if ($@) {
-                chomp $@;
-                warn "Failed to insert test case info, error <$@>\n";
-            }
-
-            $self->cleanup($testStatuses->{$testName}, \%testHash, $testResult,
-                $benchmarkResult, $log);
+    # hope they don't have more than store per line
+    # also note that this won't work if you comment out a store
+    my @q = split(/\n/, $testCmd->{'pig'});
+        for (my $i = 0; $i < @q; $i++) {
+            $count += $q[$i] =~ /store\s+[a-zA-Z][a-zA-Z0-9_]*\s+into/i;
+    }
 
-            #$report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName}, $testResult ) if ( $report );
-            $report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName} ) if ( $report );
-            $groupDuration = $groupDuration + $duration;
-            $totalDuration = $totalDuration + $duration;
-            printResults($testStatuses, $log, "Results so far");
-        }
-           if ( $report ) {
-              my $reportname= $group->{'name'};
-              if ( $globalHash{'reportname'} ) {
-                 $reportname= $globalHash{'reportname'};
-              }
-              $report->systemOut( $logname, $reportname );
-              printGroupResultsXml( $report, $reportname, $testStatuses, $groupDuration );
-           }
-           $report = 0;
-           $groupDuration=0;
+    return $count;
+}
 
-    }
+##############################################################################
+# Check whether we should be running this test or not.
+#
+sub wrongExecutionMode($$)
+{
+    my ($self, $testCmd) = @_;
 
-    # Do the global cleanup
-    $self->globalCleanup(\%globalHash, $log);
+    # Check that we should run this test.  If the current execution type
+    # doesn't match the execonly flag, then skip this one.
+    return (defined $testCmd->{'execonly'} &&
+            $testCmd->{'execonly'} ne $testCmd->{'exectype'});
 }
 
 ##############################################################################

Added: pig/branches/branch-0.9/test/e2e/pig/macros/macro1.pig
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/macros/macro1.pig?rev=1170769&view=auto
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/macros/macro1.pig (added)
+++ pig/branches/branch-0.9/test/e2e/pig/macros/macro1.pig Wed Sep 14 19:03:12 2011
@@ -0,0 +1,4 @@
+define simple_macro(in_relation, min_gpa, max_age) returns c {
+    b = filter $in_relation by gpa >= $min_gpa and age <= $max_age;
+    $c = foreach b generate age, name;
+};

Added: pig/branches/branch-0.9/test/e2e/pig/macros/macro_bad1.pig
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/macros/macro_bad1.pig?rev=1170769&view=auto
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/macros/macro_bad1.pig (added)
+++ pig/branches/branch-0.9/test/e2e/pig/macros/macro_bad1.pig Wed Sep 14 19:03:12 2011
@@ -0,0 +1,6 @@
+-- macro with syntax error
+
+define simple_macro(in_relation, min_gpa, max_age) returns c {
+    b = fiter $in_relation by gpa >= $min_gpa and age <= $max_age;
+    c = foreach b generate age, name;
+};

Modified: pig/branches/branch-0.9/test/e2e/pig/tests/bigdata.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/tests/bigdata.conf?rev=1170769&r1=1170768&r2=1170769&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/tests/bigdata.conf (original)
+++ pig/branches/branch-0.9/test/e2e/pig/tests/bigdata.conf Wed Sep 14 19:03:12 2011
@@ -24,7 +24,8 @@
 
 $cfg = {
 	'driver' => 'Pig',
-	'nummachines' => 10,
+    'execonly' => 'mapred',
+    'execonly' => 'mapred',
 
 	'groups' => [
 		{
@@ -32,6 +33,8 @@ $cfg = {
 		'tests' => [
 			{
 			'num' => 1,
+            ,'floatpostprocess' => 1
+            ,'delimiter' => '	',
 			'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
 b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
 c = filter a by age < '50';
@@ -42,37 +45,21 @@ g = group f by registration parallel 20;
 h = foreach g generate group, SUM(f.d::contributions);
 i = order h by $1, $0 parallel 20;
 store i into ':OUTPATH:';\,
-			'sql' => "select registration, sum(contributions)
-from studenttab20m as a full outer join votertab10k as b using (name, age)
-where a.age < 50 and b.age < 50
-group by registration
-order by 2, 1;",
-			'floatpostprocess' => 1,
-			'delimiter' => '	',
-			},
-#			{
-#			'num' => 2,
-#			'sortBenchmark' => 1,
-#			'sortResults' => 1,
-#			'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
-#a1 = filter a by age < '50';
-#b = group a1 by (name, age);
-#c = foreach b generate group as g, AVG(a1.gpa);
-#d = filter c by $1 > '3.0';
-#d1 = foreach d generate g.$0 as name, g.$1 as age, $1 as gpa; 
-#e = group d1 by name;
-#f = foreach e generate group, AVG(d1.age);
-#store f into ':OUTPATH:';\,
-#			'sql' => "select name, avg(age)
-#from (select name, age, avg(gpa) from studenttab20m where age < 50 group by name, age having avg(gpa) > 3.0) as T
-#group by name;",
-#			'floatpostprocess' => 1,
-#			'delimiter' => '	',
-#			},
+			},
+			{
+			'num' => 2,
+			'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
+a1 = filter a by age < '50';
+b = group a1 by (name, age);
+c = foreach b generate group as g, AVG(a1.gpa);
+d = filter c by $1 > 3.0;
+d1 = foreach d generate g.$0 as name, g.$1 as age, $1 as gpa; 
+e = group d1 by name;
+f = foreach e generate group, AVG(d1.age);
+store f into ':OUTPATH:';\,
+			},
 			{
 			'num' => 3,
-			'sortBenchmark' => 1,
-			'sortResults' => 1,
 			'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
 b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
 c = foreach a generate name;
@@ -81,70 +68,46 @@ e = filter c by name matches '.*allen$';
 f = union d, e;
 g = distinct f parallel 20;
 store g into ':OUTPATH:';\,
-			'sql' => q\select name from studenttab20m where name like '%allen'
-union 
-select name from votertab10k;\,
 			},
 			{
 			'num' => 4,
-			'sortBenchmark' => 1,
-			'sortResults' => 1,
 			'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
 b = group a all parallel 20;
 c = foreach b generate COUNT(a.$0);
 store c into ':OUTPATH:';\,
-			'sql' => "select count(*) from studenttab20m;",
 			},
             		{
 			'num' => 5,
-			'sortBenchmark' => 1,
-			'sortResults' => 1,
 			'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
 b = group a by name parallel 20;
 c = foreach b generate group, COUNT($1);
 store c into ':OUTPATH:';\,
-			'sql' => "select name, count(*) from studenttab20m group by name;",
 			},
             		{
 			'num' => 6,
-			'sortBenchmark' => 1,
-			'sortResults' => 1,
 			'pig' => q\
 define cmd `perl PigStreaming.pl` ship(':SCRIPTHOMEPATH:/PigStreaming.pl') stderr('CMD' limit 3);
 a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
 b = stream a through cmd as (n, a, g);
 c = foreach b generate n, a;
 store c into ':OUTPATH:';\,
-			'sql' => "select name, age from studenttab20m;",
 			},
 			{
 			'num' => 7,
-			'sortBenchmark' => 1,
-			'sortResults' => 1,
 			'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
 b = order a by name parallel 20;
 store b into ':OUTPATH:';\,
-			'sql' => "select * from studenttab20m;",
 			'sortArgs' => ['-t', '	', '+0', '-1'],
-			'floatpostprocess' => 1,
-			'delimiter' => '	',
 			},
 			{
 			'num' => 8,
-			'sortBenchmark' => 1,
-			'sortResults' => 1,
 			'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name:chararray, age:int, gpa:double);
 b = order a by name, age desc parallel 20;
 store b into ':OUTPATH:';\,
-			'sql' => "select * from studenttab20m;",
 			'sortArgs' => ['-t', '	', '+0', '-1', '+1rn', '-2'],
-			'floatpostprocess' => 1,
-			'delimiter' => '	',
 			},
 			{
 			'num' => 9,
-			'sortBenchmark' => 1,
-			'sortResults' => 1,
 			'pig' => q\A = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
 B = filter A by age > 20;
 C = group B by name;
@@ -152,18 +115,8 @@ D = foreach C generate group, COUNT(B) P
 E = order D by $0 PARALLEL 16;
 F = limit E 10;
 store F into ':OUTPATH:';\,
-			'sql' => "select name, count(*) from studenttab20m where age > 20 group by name order by name limit 10;",
 			'sortArgs' => ['-t', '	', '+0', '-1'],
 			},
-                        {
-                        'num' => 10,
-			'floatpostprocess' => 1,
-			'delimiter' => '	',
-                        'pig' =>q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
-store a into ':OUTPATH:';\,
-                        'sql' => "select name, age, gpa from studenttab20m;"
-                        }
-
 
 		]
 		},

Modified: pig/branches/branch-0.9/test/e2e/pig/tests/cmdline.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/tests/cmdline.conf?rev=1170769&r1=1170768&r2=1170769&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/tests/cmdline.conf (original)
+++ pig/branches/branch-0.9/test/e2e/pig/tests/cmdline.conf Wed Sep 14 19:03:12 2011
@@ -35,6 +35,7 @@ $cfg = {
 		'name' => 'Describe_cmdline',
 		'floatpostprocess' => 0,
 		'delimiter' => '	',
+        'ignore' => 1, # Need to investigate failures
 		'tests' => [
                         #JIRA[PIG-372]
                         #JIRA[PIG-374] 
@@ -148,7 +149,7 @@ describe D;\,
                         # C: {name: chararray,age: integer}
                         # D: {group: chararray C: {name: chararray,age: integer}}
                         'expected_out_regex' => 
-"A: {name: chararray,age: int,gpa: float}\nB: {name: chararray,age: int}\nC: {name: chararray,age: int}\nD: {group: chararray,C: {name: chararray,age: int}}\n",
+"A: {name: chararray,age: int,gpa: float}\nB: {name: chararray,age: int}\nC: {name: chararray,age: int}\nD: {group: chararray,C: {(name: chararray,age: int)}}",
 
 			},
 			{
@@ -168,7 +169,7 @@ describe D;\,
                         # C: {name: chararray,age: integer}
                         # D: {group: chararray C: {name: chararray,age: integer}}
                         'expected_out_regex' => 
-"A: {name: chararray,age: int,gpa: float}\nB: {name: chararray,age: int}\nC: {name: chararray,age: int}\nD: {group: chararray,C: {name: chararray,age: int}}\n",
+"A: {name: chararray,age: int,gpa: float}\nB: {name: chararray,age: int}\nC: {name: chararray,age: int}\nD: {group: chararray,C: {(name: chararray,age: int)}}\n",
 
 			},
 			{
@@ -188,7 +189,7 @@ describe A;\,
                         # C: {name: chararray,age: integer}
                         # D: {group: chararray C: {name: chararray,age: integer}}
                         'expected_out_regex' => 
-"D: {group: chararray,C: {name: chararray,age: int}}\nC: {name: chararray,age: int}\nB: {name: chararray,age: int}\nA: {name: chararray,age: int,gpa: float}",
+"D: {group: chararray,C: {(name: chararray,age: int)}}\nC: {name: chararray,age: int}\nB: {name: chararray,age: int}\nA: {name: chararray,age: int,gpa: float}",
 
 			},
 			{
@@ -247,6 +248,7 @@ dump A;\,
 		{
 		'name' => 'Warning',
 		'floatpostprocess' => 0,
+		'execonly' => 'mapred', # Warnings use counters, which don't work in local mode
 		'delimiter' => '	',
 		'tests' => [
 		
@@ -297,6 +299,66 @@ describe b;\,
             },
         ],
         },
+        {
+
+# 1 Test that a nested foreach gives instant feedback: after user issue the foreach statement in Grunt  	   	  
+# 2 Test that a nested foreach gives instant feedback: as part of execution: when we run this foreach statement, we will dump the schema for the nested alias 	 	 
+# 3 Describe single Alias resulting from a nested Foreach 	 	 
+# 4 Describe multiple Alias resulting from a nested Foreach 	 	 
+# 5 Describe resulting from a nested Foreach that contains a positional parameter 	 	 
+# 6 Describe for child Alias resulting from a nested Foreach where the child alias had multiple assignments 	  	The expected behavior is that the last assignment will destermine the result of the describe statement.
+# 7 Describe for an Alias resulting from a nested Foreach where the projection for the nested alias is empty 	 	 
+# 8 Describe within a foreach statement
+# 9 Describe for alias with complex data types 
+# 10 Describe that uses references an alias from an AS clause 
+
+                'name' => 'NestedDescribe',
+                'tests' => [
+                        {
+# 2 Test that a nested foreach gives instant feedback: as part of execution: when we run this foreach statement, we will dump the schema for the nested alias 	 	 
+# 3 Describe single Alias resulting from a nested Foreach 	 	 
+                        'num' => 1,
+                        'pig' => q\
+A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+B = group A by name;
+C = foreach B { D = distinct A.age; generate COUNT(D), group;}
+describe C::D;
+\,
+                          ,'expected_out_regex' => "D: {age: bytearray}"
+
+                      },{
+# 2 Test that a nested foreach gives instant feedback: as part of execution: when we run this foreach statement, we will dump the schema for the nested alias 	 	 
+# 7 Describe for an Alias resulting from a nested Foreach where the projection for the nested alias is empty 	 	 
+                        'num' => 2,
+                        'pig' => q\
+A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+B = group A by name;
+C = foreach B { D = distinct A.age; E= filter D by age > 1000; generate COUNT(E), group;}
+describe C;
+\,
+                          ,'expected_out_regex' => "C: {long,group: bytearray}"
+
+
+                      },{
+# 1 Test that a nested foreach gives instant feedback: after user issue the foreach statement in Grunt  	   	  
+# 2 Test that a nested foreach gives instant feedback: as part of execution: when we run this foreach statement, we will dump the schema for the nested alias 	 	 
+# 3 Describe single Alias resulting from a nested Foreach 	 	 
+# 4 Describe multiple Alias resulting from a nested Foreach 	 	 
+# 5 Describe resulting from a nested Foreach that contains a positional parameter 	 	 
+                        'num' => 3,
+                        'pig' => q\
+A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+B = group A by name;
+C = foreach B { D = distinct A.$1; generate COUNT(D), group;}
+describe C::D;
+\,
+                          ,'expected_out_regex' => "D: {age: bytearray}"
+
+
+                      },
+                ]
+
+            }
 	]
 }
 ;

Modified: pig/branches/branch-0.9/test/e2e/pig/tests/grunt.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/tests/grunt.conf?rev=1170769&r1=1170768&r2=1170769&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/tests/grunt.conf (original)
+++ pig/branches/branch-0.9/test/e2e/pig/tests/grunt.conf Wed Sep 14 19:03:12 2011
@@ -36,448 +36,89 @@ $cfg = {
                 'name' => 'Grunt',
                 'tests' => [
                         {
-# TEST Simple Test using ls
-                        'num' => 100,
-                        'pig' => q\
-ls :INPATH:
-sh ls 
-sh echo hello
-sh ls -l :PIGHARNESS:
-sh echo hello2
-\,
-                          ,'expected_out_regex' => "hdfs:.*singlefile"
-                          ,'expected_out_regex' => "hello"
-                          ,'expected_out_regex' => "hello2"
-                          ,'expected_out_regex' => "libexec"
-                          ,'rc' => 0
-
-                      },{
-# TEST Extensive Test using ls
                         'num' => 1,
-                        'pig' => q\
-sh ls -a :PIGHARNESS:/in/sql/address.txt
-sh ls -l :PIGHARNESS:/in/sql/dates.txt
-sh ls :PIGHARNESS:
-sh bash -c 'ls :PIGHARNESS:/in/sql/student*'
-sh bash -c 'ls -l :PIGHARNESS:/in/sql/c*.txt'
-sh ls /bin
-ls
-\,
-                          ,'expected_out_regex' => "address.txt"
-                          ,'expected_out_regex' => "dates.txt"
-                          ,'expected_out_regex' => "libexec"
-                          ,'expected_out_regex' => "-r.-r.-r--"
-                          ,'expected_out_regex' => "student"
-                          ,'expected_out_regex' => "customer"
-                          ,'expected_out_regex' => "rmdir"
-                          ,'rc' => 0
-
+                        'pig' => "sh ls :PIGHARNESS:",
+                        'expected_out_regex' => "libexec",
+                        'rc' => 0
                       },{
-# TEST pwd
                         'num' => 2,
-                        'pig' => q\
-pwd
-sh pwd
-\,
-                          ,'expected_out_regex' => "hdfs:"
-                          ,'rc' => 0
-
-                      },{
-# TEST for cat 
-                        'num' => 3,
-                        'pig' => q\
-sh cat :PIGHARNESS:/in/sql/numbers.txt
-\,
-                          ,'expected_out_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2"
-                          ,'rc' => 0
+                        'pig' => "pwd",
+                        'execonly' => 'mapred', # don't have a clue what their cwd will be for local mode
+                        'expected_out_regex' => "hdfs:",
+                        'rc' => 0
 
                       },{
-# TEST for pipe using cat | wc
-#BROKEN should be fixed with nightly - pipe fixed
                         'num' => 4,
-                        'pig' => q\
-sh bash -c 'cat :PIGHARNESS:/in/sql/numbers.txt | wc'
-\,
-                          ,'expected_out_regex' => "5000.*5000.*199455"
+                        'pig' => "ls :INPATH:",
+                          ,'expected_out_regex' => ".*singlefile"
                           ,'rc' => 0
 
-# TEST for cd
-                      },{
-                        'num' => 5,
-                        'pig' => q\
-sh bash -c cd :LOCALINPATH:
-\,
-                          ,'rc' => 0
-
-# TEST for touch
                       },{
                         'num' => 6,
                         'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test/newfile.txt
-sh ls -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test/newfile.txt
-\,
-                          ,'expected_out_regex' => "Grunt_6.test/newfile.txt"
-                          ,'rc' => 0
-                      },{
-# TEST for rm
-                        'num' => 7,
-                        'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test/newfile.txt
-sh rm -fv :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test/newfile.txt
-\,
-                          ,'expected_out_regex' => "removed.*Grunt_7.test/newfile.txt"
-                          ,'rc' => 0
-                      },{
-# TEST for mkdir
-                        'num' => 8,
-                        'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir/newfile.txt
-sh ls -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir/newfile.txt
-\,
-                          ,'expected_out_regex' => "-r.-r.*newfile.txt"
-                          ,'rc' => 0
-                      },{
-# TEST for mkdir  
-# TEST for chmod 775 file
-                        'num' => 9,
-                        'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test/newfile.txt
-sh chmod -v 755 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test/newfile.txt
-\,
-                          ,'expected_out_regex' => "changed to 0755"
-                          ,'rc' => 0
-                      },{
-                        'num' => 10,
-                        'pig' => q\
-sh mkdir -vp :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_10.test/newdir
-sh rm -Rfv :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_10.test/newdir
-\,
-                          ,'expected_out_regex' => "mkdir: created directory.*newdir"
-                          ,'expected_out_regex' => "removed directory.*newdir"
-                          ,'rc' => 0
-                      },{
-# TEST for diff at directory level
-                        'num' => 11,
-                        'pig' => q\
-A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B  = limit A 5;
-C  = limit A 10;
-store B into ':OUTPATH:/B.out';
-store C into ':OUTPATH:/C.out';
-copyToLocal  :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp
-sh bash -c 'diff :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp/B.out/part-r-00000 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp/C.out/part-r-00000'
-\,
-                          ,'expected_out_regex' => "1000.*9.*1.*904.*5018.*5018.*904.501.*904.501"
-                          ,'rc' => 0
-                      },{
-# TEST for diff two file where one is empty file
-                        'num' => 12,
-                        'pig' => q\
-A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B  = limit A 10;
-Empty  = filter A by id > 99999;
-store B into ':OUTPATH:/B.out';
-store Empty into ':OUTPATH:/Empty.out';
-copyToLocal  :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test
-sh bash -c 'diff :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test/B.out/part-r-00000 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test/Empty.out/part-m-00000'
-\,
-                          ,'expected_out_regex'=>"1000.*10.*2.*900.*5051.*5051.*10.2.*10.2"
-                          ,'rc' => 0
-
-                      },{
-# TEST form chmod +x file
-                        'num' => 13,
-                        'pig' => q\
-A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B  = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal  :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_13.test
-sh bash -c 'chmod -v -x :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_13.test/part-r-00000'
-\,
-                          ,'expected_out_regex' => "changed.*to.*0666.*rw-rw-rw-"
-                          ,'rc' => 0
-
-                      },{
-# TEST form chmod +775 file
-                        'num' => 14,
-                        'pig' => q\
-A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B  = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal  :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_14.test
-sh bash -c 'chmod -v 775 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_14.test/part-r-00000'
-\,
-                          ,'expected_out_regex' => "changed.*to.*0775.*rwxrwxr-x"
-
-                          ,'rc' => 0
-
-                      },{
-# TEST for sort
-# TEST command "tail"
-# TEST multple unix commands in a single line
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
-                        'num' => 15,
-                        'pig' => q\
-A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B  = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal  :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test
-sh bash -c 'sort :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/part-r-00000 > :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/out.sorted'
-sh bash -c 'tail -n 2 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/out.sorted'
-\,
-                          ,'expected_out_regex' => "1000.*9.*1.*904.*5018.*5018.*904.501.*904.501"
-
-                          ,'rc' => 0
-
-                      },{
-# TEST for sort
-# TEST command "head"
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
-                        'num' => 16,
-                        'pig' => q\
-A  = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B  = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal  :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test
-sh bash -c 'sort :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/part-r-00000 > :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/out.sorted'
-sh bash -c 'head -n 2 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/out.sorted'
-\,
-                          ,'expected_out_regex' => "1000.*1.*3.*90.*5084.*5084.*8.3.*8.3"
-
-                          ,'rc' => 0
-
-
-                      },{
-                        'num' => 17,
-                        'pig' => q\
-sh whoami
-\,
-                          ,'expected_out_regex' => "hadoopqa"
-                          ,'rc' => 0
-
-                      },{
-# TEST for comment
-# TEST for wc
-# TEST for wc with pipe
-# COMMENTS NOT SUPPORTED
-# sh #THIS IS  A COMMENT
-#sh cat :INPATH:/types/numbers.txt | wc 
-                        'num' => 18,
-                        'pig' => q\
-copyToLocal  :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers.txt
-copyToLocal  :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers2.txt
-sh wc :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers.txt
-sh wc -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers2.txt
-\,
-                          ,'expected_out_regex' => "5000.*5000.*199455.*numbers.txt"
-                          ,'expected_out_regex' => "5000.*numbers2.txt"
-                          ,'rc' => 0
-
-                      },{
-# TEST for grep
-# TEST foringle quotes
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
-#May want t rewrite this so that we use pig to generate results to compare against
-                        'num' => 19,
-                        'pig' => q#
-copyToLocal  :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt
-sh grep 503 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt
-sh grep :506*: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt
-sh bash -c 'grep \':501:\' :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt'
-#,
-                          ,'expected_out_regex' => "1000:501:5:90:5019:5019L:8.5f:8.5"
-                          ,'expected_out_regex' => "1000:503:3:90:5064:5064L:0.506490f:0.506490"
-                          ,'expected_out_regex' => "5000:4972:5:90:5066:5066L:5.9f:5.9"
-                          ,'rc' => 0
-
-                      },{
-# TEST for uniq
-                        'num' => 20,
-                        'pig' => q\
-copyToLocal  :INPATH:/singlefile/studenttab10k :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_20.tmp/studenttab10k
-sh uniq :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_20.tmp/studenttab10k
-\,
-                          ,'expected_out_regex' => "gabriella.*brown.*60.*1.71"
-                          ,'rc' => 0
-
-# TEST for find
-# TEST for wildcard
-#sh find :INPATH: -name studenttab10k
-                      },{
-                        'num' => 21,
-                        'pig' => q\
-sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp/singlefile
-copyToLocal  :INPATH:/singlefile/studenttab10k :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp/singlefile/studenttab10k
-sh find :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp -name student*
-\,
-                          ,'expected_out_regex' => "studenttab10k"
-                          ,'rc' => 0
-
-                      },{
-# TEST for df
-                        'num' => 22,
-                        'pig' => q\
-sh df
-\,
-                          ,'expected_out_regex' => "Filesystem.*Used.*Available"
-                          ,'rc' => 0
-
-                      },{
-# TEST for du
-                        'num' => 23,
-                        'pig' => q\
-sh du :PIGHARNESS:/in
-\,
-                          ,'expected_out_regex' => "sql"
-                          ,'rc' => 0
-
-                      },{
-# TEST for comment - not supported
-#sh #IGNORE THIS
-# TEST for echo
-                        'num' => 24,
-                        'pig' => q\
-
-sh echo This little piggy went to market
-sh echo This little piggy stayed home
-sh echo This little piggy had roast beef
-sh echo This little piggy had none
-sh echo This little piggy went wee wee wee, all the way home
-\,
-                          ,'expected_out_regex' => "market"
-                          ,'expected_out_regex' => "stayed"
-                          ,'expected_out_regex' => "roast"
-                          ,'expected_out_regex' => "none"
-                          ,'expected_out_regex' => "all the way home"
-                          ,'rc' => 0
-
-                      },{
-# TEST for pipe
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
-                        'num' => 25,
-                        'pig' => q\
-
-sh bash -c 'echo This little piggy went to market | wc'
-sh bash -c 'echo This little piggy stayed home | wc'
-sh bash -c 'echo This little piggy had roast beef | wc'
-sh bash -c 'echo This little piggy had none | wc'
-sh bash -c 'echo This little piggy went wee wee wee, all the way home | wc' 
-\,
-                          ,'expected_out_regex' => "1.*6.*33"
-                          ,'expected_out_regex' => "1.*5.*30"
-                          ,'expected_out_regex' => "1.*5.*27"
-
-                          ,'rc' => 0
-
-
-                      },{
-# TEST for environmet variable
-                        'num' => 26,
-                        'pig' => q\
-sh bash -c 'echo $PIG_HARNESS_ROOT'
-sh bash -c 'ls $PIG_HARNESS_ROOT'
-\,
-                          ,'expected_out_regex' => "libexec"
-                          ,'rc' => 0
-
-
-
-# TEST read file you don't have read rights for
-                      },{
-                        'num' => 27,
-                        'pig' => q\
 sh touch /bin/bad
 \,
                           ,'expected_err_regex' => "Permission denied"
                           ,'rc' => 0
 
-                      },{
-                        'num' => 28,
-                        'pig' => q\
-sh ls -l doesnotexist1
-sh ls -l doesnotexist2/bad
-sh ls -l  :OUTPATH:/doesnotexist3.txt
-\,
-                          ,'expected_err_regex' => "doesnotexist1.*No such file or directory"
-                          ,'expected_err_regex' => "doesnotexist2/bad.*No such file or directory"
-                          ,'expected_err_regex' => "doesnotexist3.txt.*No such file or directory"
-
-                          ,'rc' => 0
-
-                      },{
-# TEST run a shell script
-# BUG-3990418 May result in "../../../../..: Permission denied" Error if the top
-# level directory does not have read permissions, this will not be fixed
-
-                        'num' => 29,
-                        'pig' => "
-sh $ENV{PIG_HARNESS_ROOT}/bin/run_fb.sh -h
-",
-                          ,'expected_out_regex' => "Usage.*run_fb.sh"
-                          ,'rc' => 0
-                      },{
-# TEST run awk script
-#BROKEN - removing this awk script for now
-#                        'num' => 30,
-#                        'pig' => q\
-#sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/types
-#copyToLocal  :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/numbers.txt
-#sh bash -c 'awk '{ print }' :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/numbers.txt'
-#\,
-#                          ,'expected_err_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2"
-#
-#                          ,'rc' => 0
-#                      },{
-# TEST run sed script
-                        'num' => 31,
-                        'pig' => q\
-sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types
-copyToLocal  :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types/numbers.txt
-sh bash -c 'sed -e "1d" :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types/numbers.txt'
-\,
-                          ,'expected_err_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2"
-                          ,'rc' => 0
-
-                      },{
-#TEST run perl script
-                        'num' => 32,
-                        'pig' => q\
-sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types
-copyToLocal  :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types/numbers.txt
-sh bash -c 'perl -p -e "s/:/+/g;" :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types/numbers.txt'
-\,
-                      },{
-#TEST run perl script
-#BROKEN Causes a 2998 Unhandled Exception, check to see if fixed in nightly
-                        'num' => 33,
-                        'pig' => q\
-sh bash -c ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls
-\,
-#sh bash -c 'ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls'
-                          ,'expected_err_regex' => "address.txt"
-
-                          ,'rc' => 0
-
-#IGNORE THIS TEST ISN'T READY
-#                      },{
-#                        'num' => 33,
-#                        'pig' => q\
-#sh echo "export FOO=MyEnvVariable" > :OUTPATH:/env.txt
-#sh source :OUTPATH:/env.txt
-#sh source :OUTPATH:/env.txt; echo $FOO
-#\,
-#                          ,'expected_out_regex' => "E: {age: bytearray}"
-#                          ,'rc' => 0
-
-                      }
-
-#ADD
-#wildcards
-#Add sort with tab delimeter
+                        },{
+                            'num' => 7,
+                            'pig' => "fs -ls :INPATH:",
+                            'expected_out_regex' => ".*singlefile",
+                            'rc' => 0
+                        },{
+                            'num' => 8,
+                            'pig' => "cat :INPATH:/singlefile/studenttab10k",
+                            'expected_out_regex' => "gabriella brown\\t60\\t1.71",
+                            'rc' => 0
+                        },{
+                            'num' => 9,
+                            'pig' => "cd :INPATH:
+                                      pwd",
+                            'expected_out_regex' => ".*data",
+                            'rc' => 0
+                        },{
+                            'num' => 10,
+                            'pig' => "cp :INPATH:/singlefile/studenttab10k .
+                                      ls .",
+                            'expected_out_regex' => ".*studenttab10k",
+                            'rc' => 0
+                        },{
+                            'num' => 11,
+                            'pig' => "cp :INPATH:/singlefile/studenttab10k ./fred
+                                      ls .",
+                            'expected_out_regex' => ".*fred",
+                            'rc' => 0
+                        },{
+                            'num' => 12,
+                            'pig' => "cp :INPATH:/singlefile/studenttab10k ./jim
+                                      mv ./jim ./bob
+                                      ls .",
+                            'expected_out_regex' => ".*bob",
+                            'rc' => 0
+                        },{
+                            'num' => 13,
+                            'pig' => "copyToLocal :INPATH:/singlefile/studenttab10k :TMP:
+                                      sh cat :TMP:/studenttab10k",
+                            'expected_out_regex' => "gabriella brown\\t60\\t1.71",
+                            'rc' => 0
+                        },{
+                            'num' => 14,
+                            'pig' => "copyToLocal :INPATH:/singlefile/votertab10k :TMP:
+                                      copyFromLocal :TMP:/votertab10k ./joe
+                                      cat ./joe",
+                            'expected_out_regex' => "jessica xylophone\\t41\\tgreen\\t638.13",
+                            'rc' => 0
+                        },{
+                            'num' => 15,
+                            'pig' => "rm fred bob joe",
+                            'not_expected_out_regex' => "joe",
+                            'rc' => 0
+                        },{
+                            'num' => 16,
+                            'pig' => "rmf jill",
+                            'rc' => 0
+                        }
                 ]
             },
         ],



Mime
View raw message