incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1325181 [2/2] - in /incubator/hcatalog/trunk: ./ bin/ hive/ ivy/ scripts/ src/docs/src/documentation/content/xdocs/ src/test/e2e/hcatalog/ src/test/e2e/hcatalog/conf/ src/test/e2e/hcatalog/deployers/ src/test/e2e/hcatalog/drivers/ src/test...
Date Thu, 12 Apr 2012 10:08:02 GMT
Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1325181&r1=1325180&r2=1325181&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm Thu Apr 12 10:08:00 2012
@@ -52,52 +52,6 @@ sub new
     return $self;
 }
 
-sub replaceParameters
-{
-##!!! Move this to Util.pm
-
-    my ($self, $cmd, $outfile, $testCmd, $log) = @_;
-
-    # $self
-    $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
-
-    # $outfile
-    $cmd =~ s/:OUTPATH:/$outfile/g;
-
-    # $ENV
-    $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g;
-
-    # $testCmd
-    $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
-    $cmd =~ s/:OUTPATH:/$outfile/g;
-    $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
-    $cmd =~ s/:PIGPATH:/$testCmd->{'pigpath'}/g;
-    $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
-    $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
-    $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g;
-    $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
-    $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
-    $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
-#    $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
-#    $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
-#    $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
-    $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
-    $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
-    $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g;
-
-    if ( $testCmd->{'hadoopSecurity'} eq "secure" ) { 
-      $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g;
-    } else {
-      $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g;
-    }
-
-    $cmd =~ s/:THRIFTSERVER:/$testCmd->{'thriftserver'}/g;
-    $cmd =~ s/:HADOOP_CLASSPATH:/$testCmd->{'hadoop_classpath'}/g;
-    $cmd =~ s/:HCAT_JAR:/$testCmd->{'hcatalog.jar'}/g;
-
-    return $cmd;
-}
-
 sub globalSetup
 {
     my ($self, $globalHash, $log) = @_;
@@ -127,10 +81,7 @@ sub globalSetup
         $ENV{'PATH'} = $globalHash->{'scriptPath'};
     }
 
-    my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'});
-
-    print $log "Going to run " . join(" ", @cmd) . "\n";
-    IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
+    Util::runHadoopCmd($globalHash, $log, "fs -mkdir $globalHash->{'outpath'}");
 
     IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or
         die "Cannot create localpath directory " . $globalHash->{'localpath'} .
@@ -145,10 +96,7 @@ sub globalSetup
         die "Cannot create temporary directory " . $globalHash->{'tmpPath'} .
         " " . "$ERRNO\n";
 
-    # Create the HDFS temporary directory
-    @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}");
-        print $log "Going to run " . join(" ", @cmd) . "\n";
-    IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
+    Util::runHadoopCmd($globalHash, $log, "fs -mkdir tmp/$globalHash->{'runid'}");
 }
 
 sub globalCleanup
@@ -199,8 +147,8 @@ sub runTest
                    my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
 
                    # Copy result file out of hadoop
-                   my @baseCmd = $self->getPigCmd($testCmd, $log);
-                   my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
+                   my @baseCmd = Util::getPigCmd($testCmd, $log);
+                   my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log);
                    $outputs[$i] = $testOut;
                    $id++;
                }
@@ -236,18 +184,16 @@ sub dumpPigTable
 
 
     # Build the command
-    my @baseCmd = $self->getPigCmd($testCmd, $log);
+    my @baseCmd = Util::getPigCmd($testCmd, $log);
     my @cmd = @baseCmd;
 
     push(@cmd, $pigfile);
 
 
     # Run the command
-    print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n";
     print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n";
 
-    IPC::Run::run(\@cmd, \undef, $log, $log) or
-        die "Failed running $pigfile\n";
+    IPC::Run::run(\@cmd, \undef, $log, $log) or die "Failed running $pigfile\n";
     $result{'rc'} = $? >> 8;
 
 
@@ -255,23 +201,17 @@ sub dumpPigTable
     my $localoutfile;
     my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out";
        
-    $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
+    $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log);
     return $outfile;
 }
 
 sub postProcessSingleOutputFile
 {
-    my ($self, $outfile, $localdir, $baseCmd, $testCmd, $log) = @_;
+    my ($self, $outfile, $localdir, $testCmd, $log) = @_;
     my $subName  = (caller(0))[3];
 
-    my @baseCmd = @{$baseCmd};
-    my @copyCmd = @baseCmd;
-    push(@copyCmd, ('-e', 'copyToLocal', $outfile, $localdir)); 
-    print $log "$0::$className::$subName INFO: Going to run pig command: @copyCmd\n";
+    Util::runHadoopCmd($globalHash, $log, "fs -copyToLocal $outfile $localdir");
  
-    IPC::Run::run(\@copyCmd, \undef, $log, $log) or die "Cannot copy results from HDFS $outfile to $localdir\n";
-
-
     # Sort the result if necessary.  Keep the original output in one large file.
     # Use system not IPC run so that the '*' gets interpolated by the shell.
     
@@ -301,8 +241,6 @@ sub postProcessSingleOutputFile
 }
 
 sub runHadoop
-# Being modified from runPig
-# !!! Works, but need to add other arguments, like queue...???
 {
     my ($self, $testCmd, $log) = @_;
     my $subName  = (caller(0))[3];
@@ -313,7 +251,13 @@ sub runHadoop
     my $hadoopfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".hadoop";
     my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
 
-    my $hadoopcmd = $self->replaceParameters( $testCmd->{'hadoop'}, $outfile, $testCmd, $log );
+    # Get all of the additional jars we'll need.
+    my $additionalJars = Util::getHBaseLibs($testCmd, $log); #hbase before hive for precedence over bundled hbase
+    $additionalJars .= Util::getHiveLibs($testCmd, $log);
+    $additionalJars .= Util::getHCatLibs($testCmd, $log);
+    $testCmd->{'libjars'} = $additionalJars;
+    $testCmd->{'libjars'} =~ s/:/,/g;
+    my $hadoopcmd = Util::replaceParameters( $testCmd->{'hadoop'}, $outfile, $testCmd, $log );
 
     # adjust for the leading and trailing new line often seen in the conf file's command directives
     $hadoopcmd =~ s/^\s*(.*?)\s*$/\1/s;
@@ -338,16 +282,12 @@ sub runHadoop
     my $cp = $testCmd->{'hcatalog.jar'}; 
     $cp =~ s/,/:/g;
     # Add in the hcat config file
-    $cp .= ":" . $testCmd->{'hive.conf.dir'};
+    $cp .= ":" . $testCmd->{'hiveconf'};
+    $cp .= ":" . $additionalJars;
     $ENV{'HADOOP_CLASSPATH'} = $cp;
 
-    if (defined($testCmd->{'hbaseconfigpath'})) {
-        $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconfigpath'}";
-    }
-
-    if (defined($testCmd->{'metastore.principal'}) && ($testCmd->{'metastore.principal'} =~ m/\S+/)) {
-        $ENV{'HADOOP_OPTS'} = "$ENV{'HADOOP_OPTS'} -Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
-        $ENV{'HADOOP_CLIENT_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
+    if (defined($testCmd->{'hbaseconf'})) {
+        $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconf'}";
     }
 
     # Add su user if provided
@@ -373,10 +313,10 @@ sub runHadoop
         die "Failed running $script\n";
 
     my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out";
-    my @baseCmd = $self->getPigCmd($testCmd, $log);
+    my @baseCmd = Util::getPigCmd($testCmd, $log);
     if ($self->countStores($testCmd)==1) {
         @outputs = ();
-        $outputs[0] = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
+        $outputs[0] = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log);
         $result{'outputs'} = \@outputs;
     }
 
@@ -535,7 +475,7 @@ sub runPig
     my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig";
     my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
 
-    my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
+    my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
 
     open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
     print FH $pigcmd . "\n";
@@ -543,7 +483,8 @@ sub runPig
 
 
     # Build the command
-    my @baseCmd = $self->getPigCmd($testCmd, $log);
+    #my @baseCmd = $self->getPigCmd($testCmd, $log);
+    my @baseCmd = Util::getPigCmd($testCmd, $log);
     my @cmd = @baseCmd;
 
     # Add option -l giving location for secondary logs
@@ -580,7 +521,7 @@ sub runPig
     # single query
     if ($stores == 1) {
         if ($copyResults) {
-            $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
+            $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, $testCmd, $log);
             $result{'originalOutput'} = "$localdir/out_original"; # populated by postProcessSingleOutputFile
         } else {
             $result{'output'} = "NO_COPY";
@@ -596,7 +537,7 @@ sub runPig
             # Copy result file out of hadoop
             my $testOut;
             if ($copyResults) {
-              $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log);
+              $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, $testCmd, $log);
             } else {
               $testOut = "NO_COPY";
             }
@@ -615,47 +556,6 @@ sub runPig
     return \%result;
 }
 
-sub getPigCmd($$$)
-{
-    my ($self, $testCmd, $log) = @_;
-
-    my @pigCmd;
-
-    # set the PIG_CLASSPATH environment variable
-	my $pcp .= $testCmd->{'jythonjar'} if (defined($testCmd->{'jythonjar'}));
-    $pcp .= ":" . $testCmd->{'classpath'} if (defined($testCmd->{'classpath'}));
-    $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'}));
-    # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined
-    $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local"); #&& (! defined $ENV{'HADOOP_HOME'});
-    $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local" && defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'} ne ""));
-
-    # Set it in our current environment.  It will get inherited by the IPC::Run
-    # command.
-    $ENV{'PIG_CLASSPATH'} = $pcp;
-
-    @pigCmd = ("$testCmd->{'pigpath'}/bin/pig");
-
-    if (defined($testCmd->{'additionaljars'})) {
-        push(@pigCmd, '-Dpig.additional.jars='.$testCmd->{'additionaljars'});
-    }
-
-    if ($testCmd->{'exectype'} eq "local") {
-		push(@{$testCmd->{'java_params'}}, "-Xmx1024m");
-        push(@pigCmd, ("-x", "local"));
-    }
-
-    my $opts .= "-Dhive.metastore.uris=$testCmd->{'thriftserver'}";
-    if (defined($testCmd->{'java_params'})) {
-        $opts = $opts . " " . join(" ", @{$testCmd->{'java_params'}});
-    }
-
-    $ENV{'PIG_OPTS'} = $opts;
-
-	print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
-	print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to " . $ENV{'PIG_OPTS'} . "\n";
-    return @pigCmd;
-}
-
 sub compareSingleOutput
 {
     my ($self, $testResult, $testOutput, $benchmarkOutput, $log) = @_;

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHive.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHive.pm?rev=1325181&r1=1325180&r2=1325181&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHive.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHive.pm Thu Apr 12 10:08:00 2012
@@ -53,35 +53,11 @@ sub new
     return $self;
 }
 
-sub replaceParameters
-{
-##!!! Move this to Util.pm
-
-    my ($self, $cmd, $outfile, $testCmd, $log) = @_;
-
-    # $self
-    $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
-
-    # $outfile
-    $cmd =~ s/:OUTPATH:/$outfile/g;
-
-    # $ENV
-    $cmd =~ s/:HARNESS:/$ENV{HARNESS_ROOT}/g;
-
-    # $testCmd
-    $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
-
-    return $cmd;
-}
-
 sub globalSetup
 {
     my ($self, $globalHash, $log) = @_;
     my $subName = (caller(0))[3];
 
-    # Set up values for the metastore
-    Util::setupHiveProperties($globalHash, $log);
-
     # Setup the output path
     my $me = `whoami`;
     chomp $me;

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm?rev=1325181&r1=1325180&r2=1325181&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm Thu Apr 12 10:08:00 2012
@@ -52,48 +52,6 @@ sub new
     return $self;
 }
 
-sub replaceParameters
-{
-##!!! Move this to Util.pm
-
-    my ($self, $cmd, $outfile, $testCmd, $log) = @_;
-
-    # $self
-    $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
-
-    # $outfile
-    $cmd =~ s/:OUTPATH:/$outfile/g;
-
-    # $ENV
-    $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g;
-
-    # $testCmd
-    $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
-    $cmd =~ s/:OUTPATH:/$outfile/g;
-    $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
-    $cmd =~ s/:PIGPATH:/$testCmd->{'pigpath'}/g;
-    $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
-    $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
-    $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g;
-    $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
-    $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
-    $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
-#    $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
-#    $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
-#    $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
-    $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
-    $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
-    $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g;
-
-    if ( $testCmd->{'hadoopSecurity'} eq "secure" ) { 
-      $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g;
-    } else {
-      $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g;
-    }
-
-    return $cmd;
-}
-
 sub globalSetup
 {
     my ($self, $globalHash, $log) = @_;
@@ -123,7 +81,7 @@ sub globalSetup
         $ENV{'PATH'} = $globalHash->{'scriptPath'};
     }
 
-    my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'});
+    my @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'});
 
 
     print $log "Going to run " . join(" ", @cmd) . "\n";
@@ -143,7 +101,7 @@ sub globalSetup
         " " . "$ERRNO\n";
 
     # Create the HDFS temporary directory
-    @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}");
+    @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}");
 	print $log "Going to run " . join(" ", @cmd) . "\n";
     IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
 }
@@ -157,7 +115,7 @@ sub globalCleanup
         " " . "$ERRNO\n";
 
     # Cleanup the HDFS temporary directory
-    my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'fs', '-rmr', "tmp/$globalHash->{'runid'}");
+    my @cmd = (Util::getPigCmd($globalHash, $log), '-e', 'fs', '-rmr', "tmp/$globalHash->{'runid'}");
 	print $log "Going to run " . join(" ", @cmd) . "\n";
     IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
 }
@@ -221,7 +179,7 @@ sub runTest
                    my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
 
                    # Copy result file out of hadoop
-                   my @baseCmd = $self->getPigCmd($testCmd, $log);
+                   my @baseCmd = Util::getPigCmd($testCmd, $log);
                    my $testOut = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
                    $outputs[$i] = $testOut;
                    $id++;
@@ -264,22 +222,16 @@ sub runPigCmdLine
     }
 
     # Write the pig script to a file.
-    my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
+    my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
 
     open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
     print FH $pigcmd . "\n";
     close(FH);
 
     # Build the command
-    my @baseCmd = $self->getPigCmd($testCmd, $log);
+    my @baseCmd = Util::getPigCmd($testCmd, $log);
     my @cmd = @baseCmd;
 
-    # Add option -l giving location for secondary logs
-    ##!!! Should that even be here? 
-    my $locallog = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".log";
-    push(@cmd, "-logfile");
-    push(@cmd, $locallog);
-
     # Add pig parameters if they're provided
     if (defined($testCmd->{'pig_params'})) {
         # Processing :PARAMPATH: in parameters
@@ -333,7 +285,7 @@ sub runScript
     }
 
     # Write the script to a file
-    my $cmd = $self->replaceParameters( $testCmd->{'script'}, $outfile, $testCmd, $log );
+    my $cmd = Util::replaceParameters( $testCmd->{'script'}, $outfile, $testCmd, $log );
 
     open(FH, ">$script") or die "Unable to open file $script to write script, $ERRNO\n";
     print FH $cmd . "\n";
@@ -363,88 +315,6 @@ sub runScript
 }
 
 
-sub getPigCmd($$$)
-{
-    my ($self, $testCmd, $log) = @_;
-
-    my @pigCmd;
-
-    # set the PIG_CLASSPATH environment variable
-	my $pcp .= $testCmd->{'jythonjar'} if (defined($testCmd->{'jythonjar'}));
-    $pcp .= ":" . $testCmd->{'classpath'} if (defined($testCmd->{'classpath'}));
-    $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'}));
-    # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined
-    $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local"); #&& (! defined $ENV{'HADOOP_HOME'});
-    $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local" && defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'} ne ""));
-
-    # Set it in our current environment.  It will get inherited by the IPC::Run
-    # command.
-    $ENV{'PIG_CLASSPATH'} = $pcp;
-
-    @pigCmd = ("$testCmd->{'pigpath'}/bin/pig");
-
-    if (defined($testCmd->{'additionaljars'})) {
-        push(@pigCmd, '-Dpig.additional.jars='.$testCmd->{'additionaljars'});
-    }
-
-    if ($testCmd->{'exectype'} eq "local") {
-		push(@{$testCmd->{'java_params'}}, "-Xmx1024m");
-        push(@pigCmd, ("-x", "local"));
-    }
-
-    my $opts .= "-Dhive.metastore.uris=$testCmd->{'thriftserver'}";
-    if (defined($testCmd->{'java_params'})) {
-        $opts = $opts . " " . join(" ", @{$testCmd->{'java_params'}});
-    }
-
-    $ENV{'PIG_OPTS'} = $ENV{'PIG_OPTS'} . " " . $opts;
-
-	print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
-	print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to " . $ENV{'PIG_OPTS'} . "\n";
-    return @pigCmd;
-}
-
-sub dumpPigTable
-{
-    my ($self, $testCmd, $table, $log, $id) = @_;
-    my $subName  = (caller(0))[3];
-
-    my %result;
-
-    # Write the pig script to a file.
-    my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.pig";
-    my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'}  . $id . "dump.out";
-
-    open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
-    print FH "a = load '$table' using org.apache.hcatalog.pig.HCatLoader(); store a into '$outfile';\n";
-    close(FH);
-
-
-    # Build the command
-    my @baseCmd = $self->getPigCmd($testCmd, $log);
-    my @cmd = @baseCmd;
-
-    push(@cmd, $pigfile);
-
-
-    # Run the command
-    print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n";
-    print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n";
-
-    IPC::Run::run(\@cmd, \undef, $log, $log) or
-        die "Failed running $pigfile\n";
-    $result{'rc'} = $? >> 8;
-
-
-    # Get results from the command locally
-    my $localoutfile;
-    my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . $id . ".dump.out";
-    my $stores = $self->countStores($testCmd);
-       
-    $outfile = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
-    return $outfile;
-}
-
 sub runPig
 {
     my ($self, $testCmd, $log, $copyResults, $noFailOnFail) = @_;
@@ -456,7 +326,7 @@ sub runPig
     my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig";
     my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
 
-    my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
+    my $pigcmd = Util::replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
 
     open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
     print FH $pigcmd . "\n";
@@ -464,7 +334,7 @@ sub runPig
 
 
     # Build the command
-    my @baseCmd = $self->getPigCmd($testCmd, $log);
+    my @baseCmd = Util::getPigCmd($testCmd, $log);
     my @cmd = @baseCmd;
 
     # Add option -l giving location for secondary logs
@@ -485,8 +355,9 @@ sub runPig
 
 
     # Run the command
-    print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n";
     print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n";
+    print $log "With PIG_CLASSPATH set to $ENV{'PIG_CLASSPATH'}\n";
+    print $log "and HADOOP_HOME set to $ENV{'HADOOP_HOME'}\n";
 
     my $runrc = IPC::Run::run(\@cmd, \undef, $log, $log);
 

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm?rev=1325181&r1=1325180&r2=1325181&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm Thu Apr 12 10:08:00 2012
@@ -27,12 +27,14 @@
 package Util;
 
 use IPC::Run qw(run);
+use strict;
+use English;
 
 sub prepareHCat
 {
     my ($self, $testCmd, $log) = @_;
     my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
-    my $hcatCmd = $self->replaceParameters( $testCmd->{'hcat_prep'}, $outfile, $testCmd, $log);
+    my $hcatCmd = replaceParameters( $testCmd->{'hcat_prep'}, $outfile, $testCmd, $log);
 
     my @hivefiles = ();
     my @outfiles = ();
@@ -50,46 +52,6 @@ sub prepareHCat
     Util::runHCatCmdFromFile($testCmd, $log, $hivefiles[0]);
 }
 
-##############################################################################
-#  Sub: setupHiveProperties
-#
-#  Assure that necessary values are set in config in order to set Hive
-#  Java properties.
-#
-#  Returns:
-#  Nothing
-sub  setupHiveProperties($$)
-{
-    my ($cfg, $log) = @_;
-
-    # Set up values for the metastore
-    if (defined($cfg->{'metastore_thrift'}) && $cfg->{'metastore_thrift'} == 1) {
-        if (! defined $cfg->{'metastore_host'} || $cfg->{'metastore_host'} eq "") {
-            print $log "When using thrift, you must set the key " .
-                " 'metastore_host' to the machine your metastore is on\n";
-            die "metastore_host is not set in existing.conf\n";
-        }
-
-        $cfg->{'metastore_connection'} =
-            "jdbc:$cfg->{'metastore_db'}://$cfg->{'metastore_host'}/hivemetastoredb?createDatabaseIfNotExist=true";
-   
-        if (! defined $cfg->{'metastore_passwd'} || $cfg->{'metastore_passwd'} eq "") {
-            $cfg->{'metastore_passwd'} = 'hive';
-        }
-
-        if (! defined $cfg->{'metastore_port'} || $cfg->{'metastore_port'} eq "") {
-            $cfg->{'metastore_port'} = '9933';
-        }
-
-        $cfg->{'metastore_uri'} =
-            "thrift://$cfg->{'metastore_host'}:$cfg->{'metastore_port'}";
-    } else {
-        $cfg->{'metastore_connection'} =
-            "jdbc:derby:;databaseName=metastore_db;create=true";
-        $cfg->{'metastore_driver'} = "org.apache.derby.jdbc.EmbeddedDriver";
-    }
-}
-
 sub getHadoopCmd
 {
     my ( $properties ) = @_;
@@ -113,7 +75,8 @@ sub getHadoopCmd
     }
     push (@baseCmd, $cmd);
 
-    push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'});
+    push (@baseCmd, '--config', $properties->{'hadoopconfdir'})
+        if defined($properties->{'hadoopconfdir'});
 
     return @baseCmd;
 }
@@ -148,31 +111,11 @@ sub runHiveCmdFromFile($$;$$$$)
     $outfile = $log if (!defined($outfile));
     $errfile = $log if (!defined($errfile));
 
-    my @cmd = ("$cfg->{'hivehome'}/bin/hive");
-
-    # Add all of the modified properties we want to set
-#   push(@cmd, "--hiveconf", "hive.metastore.uris=$cfg->{'thriftserver'}");
-#   push(@cmd, "--hiveconf", "hive.metastore.local=false");
-
-#   if( defined($cfg->{'metastore.principal'}) && ($cfg->{'metastore.principal'} =~ m/\S+/)
-#       &&  ($cfg->{'metastore.principal'} ne '${metastore.principal}')){
-#       push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=true",  "--hiveconf", "hive.metastore.kerberos.principal=$cfg->{'metastore.principal'}");
-#   } else {
-#       push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=false");
-#   }
-
-    $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hive.conf.dir'}";
-
-    if (defined($cfg->{'hive.additionaljars'})) {
-        $ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hive.additionaljars'};
-    }
-
-#   if (defined($cfg->{'hiveconf'})) {
-#       foreach my $hc (@{$cfg->{'hiveconf'}}) {
-#           push(@cmd, "--hiveconf", $hc);
-#       }
-#   }
-
+    my @cmd = ($cfg->{'hivebin'});
+  
+    $ENV{'HIVE_CONF_DIR'} = $cfg->{'hiveconf'};
+    $ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hcatshare'};
+ 
     if (defined($cfg->{'hivecmdargs'})) {
         push(@cmd, @{$cfg->{'hivecmdargs'}});
     }
@@ -230,26 +173,28 @@ sub runHCatCmdFromFile($$;$$$$)
     my ($cfg, $log, $sql, $outfile, $errfile, $noFailOnFail) = @_;
 
     if (!defined($ENV{'HADOOP_HOME'})) {
-        die "Cannot run hive when HADOOP_HOME environment variable is not set.";
+        die "Cannot run hcat when HADOOP_HOME environment variable is not set.";
     }
 
     $outfile = $log if (!defined($outfile));
     $errfile = $log if (!defined($errfile));
 
     # unset HADOOP_CLASSPATH
-    $ENV{'HADOOP_CLASSPATH'} = "";
-    $ENV{'HADOOP_CLASSPATH'} = $cfg->{'pigjar'};
+#   $ENV{'HADOOP_CLASSPATH'} = "";
+    $ENV{'HADOOP_CLASSPATH'} = $cfg->{'hbaseconf'};
+    $ENV{'HCAT_CLASSPATH'} = Util::getHBaseLibs($cfg, $log);
 
     my @cmd;
     if (defined($sql)) {
-        @cmd = ("$cfg->{'hcathome'}/bin/hcat", "-f", $sql);
+        @cmd = ("$cfg->{'hcatbin'}", "-f", $sql);
     } else {
-        @cmd = ("$cfg->{'hcathome'}/bin/hcat");
+        @cmd = ("$cfg->{'hcatbin'}");
     }
 
     my $envStr;
     for my $k (keys(%ENV)) {
-        $envStr .= $k . "=" . $ENV{$k} . " " if ($k =~ /HADOOP/ || $k =~ /HIVE/);
+        $envStr .= $k . "=" . $ENV{$k} . " " if ($k =~ /HADOOP/ || $k =~ /HIVE/ ||
+                $k =~ /HCAT/);
     }
     $envStr .= " ";
     print $log "Going to run hcat command [" . join(" ", @cmd) .
@@ -327,198 +272,33 @@ sub show_call_stack {
 
 sub getPigCmd
 {
-    my $subName        = (caller(0))[3];
-    my $jarkey         = shift;
-    my ( $properties ) = @_;
-    my $isPigSqlEnabled= 0;
-    my @baseCmd;
-    die "$0.$subName: null properties" if (! $properties );
-
-show_call_stack();
-    #UGLY HACK for pig sql support
-    if ( $jarkey =~ /testsql/ ) {
-
-       $isPigSqlEnabled= 1;
-       $jarkey = "testjar";
-
-    }
-
-    my $cmd;
-    if ( $properties->{'use-pig.pl'} ) {
-      # The directive gives that
-      # 1) the 'pig' command will be called, as opposed to java
-      # 2) the conf file has full control over what options and parameters are 
-      #    passed to pig. 
-      #    I.e. no parameters should be passed automatically by the script here. 
-      #
-      # This allows for testing of the pig script as installed, and for testin of
-      # the pig script's options, including error testing. 
-
-print 'use-pig.pl?????';
-
-      $cmd = $properties->{'gridstack.root'} . "/pig/" . $properties->{'pigTestBuildName'} . "/bin/pig";
-      if ( ! -x "$cmd" ) {
-        print STDERR "\n$0::$subName WARNING: Can't find pig command: $cmd\n";
-        $cmd = `which pig`;
-        chomp $cmd;
-        print STDERR "$0::$subName WARNING: Instead using command: $cmd\n";
-      }
-      die "\n$0::$subName FATAL: Pig command does not exist: $cmd\n" if ( ! -x $cmd  );
-      push (@baseCmd, $cmd );
-   
-       if(defined($properties->{'additionaljars'})) {
-          push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'});
-        }
-        $ENV{'PIG_CLASSPATH'}=$properties->{'additionaljars'};  
-
-      if ( $properties->{'use-pig.pl'} eq 'raw' ) { # add _no_ arguments automatically
-        # !!! 
-	return @baseCmd;
-      }
-
-    } else {
-        $cmd="java";
+    my ( $cfg, $log ) = @_;
 
-print 'not use-pig.pl?????';
-        # Set JAVA options
+    my @cmd = ("$cfg->{'pigbin'}");
 
-        # User can provide only one of
-        # (-c <cluster>) OR (-testjar <jar> -testconfigpath <path>)
-        # "-c <cluster>" is allowed only in non local mode
-        if(defined($properties->{'cluster.name'})) {
-            # use provided cluster
-            @baseCmd = ($cmd, '-c', $properties->{'cluster.name'});
-        } else {
     
-                die "\n$0::$subName FATAL: The jar file name must be passed in at the command line or defined in the configuration file\n" if ( !defined( $properties->{$jarkey} ) );
-                die "\n$0::$subName FATAL: The jar file does not exist.\n" . $properties->{$jarkey}."\n" if ( ! -e  $properties->{$jarkey}  );
-    
-            # use user provided jar
-                my $classpath;
-
-				if (defined $properties->{'jythonjar'}) {
-					$classpath = "$classpath:" . $properties->{'jythonjar'};
-				}
-                if( $properties->{'exectype'} eq "local") {
-                   # in local mode, we should not use
-                   # any hadoop-site.xml
-                   $classpath= "$classpath:" . $properties->{$jarkey};
-                   $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) );
-                   @baseCmd = ($cmd, '-cp', $classpath, '-Xmx1024m');
-    
-                } else {
-    
-                   # non local mode, we also need to specify
-                   # location of hadoop-site.xml
-                   die "\n$0::$subName FATAL: The hadoop configuration file name must be passed in at the command line or defined in the configuration file\n" 
-			if ( !defined( $properties->{'testconfigpath'} ) );
-                   die "\n$0::$subName FATAL $! " . $properties->{'testconfigpath'}."\n\n"  
-                   	if (! -e $properties->{'testconfigpath'} );
-
-                   $classpath= "$classpath:" . $properties->{$jarkey}.":".$properties->{'testconfigpath'};
-                   $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) );
-                   $classpath= "$classpath:$properties->{'howl.jar'}" if ( defined( $properties->{'howl.jar'} ) );
-                   @baseCmd = ($cmd, '-cp', $classpath );
-            }
-        }
-    
-        # sets the queue, for exampel "grideng"
-        if(defined($properties->{'queue'})) {
-          push( @baseCmd,'-Dmapred.job.queue.name='.$properties->{'queue'});
-        }
-    
-        if(defined($properties->{'additionaljars'})) {
-          push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'});
-        }
-    
-        if( ( $isPigSqlEnabled == 1 ) ){
-
-	    if(defined($properties->{'metadata.uri'})) {
-		push( @baseCmd, '-Dmetadata.uri='.$properties->{'metadata.uri'});
-	    }
-
-	    if(defined($properties->{'metadata.impl'})) {
-		push( @baseCmd, '-Dmetadata.impl='.$properties->{'metadata.impl'});
-	    }else{
-		push( @baseCmd, '-Dmetadata.impl=org.apache.hadoop.owl.pig.metainterface.OwlPigMetaTables');
-	    }
-        }
-
-        # Add howl support
-	if(defined($properties->{'howl.metastore.uri'})) {
-	  push( @baseCmd, '-Dhowl.metastore.uri='.$properties->{'howl.metastore.uri'});
-	}
+    # sets the queue, for exampel "grideng"
+    if(defined($cfg->{'queue'})) {
+        push( @cmd,'-Dmapred.job.queue.name='.$cfg->{'queue'});
+    }
     
-      # Set local mode property
-      # if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) {
-      # Removed above 'if...' for Pig 8.
-        my $java=`which java`;
-        my $version=`file $java`;
-        if ( $version =~ '32-bit' ){
-           push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-i386-32');
-        } else {
-           push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-amd64-64');
-        }
-      # }
-
-
-        # Add user provided java options if they exist
-        if (defined($properties->{'java_params'})) {
-          push(@baseCmd, @{$properties->{'java_params'}});
-        }
+    my $cp = Util::getHCatLibs($cfg, $log) .  Util::getHiveLibsForPig($cfg, $log) .
+        Util::getHBaseLibs($cfg, $log);
+    push(@cmd, ('-Dpig.additional.jars='. $cp));
+    $cp .= ':' . $cfg->{'hiveconf'};
+    $cp .= ':' . $cfg->{'hbaseconf'};
+    $ENV{'PIG_CLASSPATH'} = $cp;
     
-        if(defined($properties->{'hod'})) {
-          push( @baseCmd, '-Dhod.server=');
-        }
-
-      # sets the permissions on the jobtracker for the logs
-      push( @baseCmd,'-Dmapreduce.job.acl-view-job=*');
+    # sets the permissions on the jobtracker for the logs
+    push( @cmd,'-Dmapreduce.job.acl-view-job=*');
 
 
-      # Add Main
-      push(@baseCmd, 'org.apache.pig.Main');
-
       # Set local mode PIG option
-      if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) {
-          push(@baseCmd, '-x');
-          push(@baseCmd, 'local');
-      }
-
-      # Set Pig SQL options
-      if( ( $isPigSqlEnabled == 1 ) && defined($properties->{'metadata.uri'})) {
-  
-         if ( defined($properties->{'testoutpath'}) ) {
-           push( @baseCmd, '-u' );
-           push( @baseCmd, $properties->{'testoutpath'} );
-         }
-  
-         push( @baseCmd, '-s' );
-         push( @baseCmd, '-f' );
-      }
-
-    } # end else of if use-pig.pl
-
-
-    # Add -latest or -useversion 
-    if ( $cmd =~ 'pig$' ) {
-      # Add -latest, or -useversion if 'current' is not target build
-      if ( defined($properties->{'pigTestBuildName'})) {
-        if ($properties->{'pigTestBuildName'} eq 'latest') {
-            push(@baseCmd, '-latest');
-        } elsif ($properties->{'pigTestBuildName'} ne 'current') {
-            push(@baseCmd, '-useversion', "$properties->{'pigTestBuildName'}");
-        }
-      }
-    } elsif ( $cmd =~ 'java' ) {
-
-      # is this ever used: ???
-      # Add latest if it's there
-      if (defined($properties->{'latest'})) {
-          push(@baseCmd, '-latest');
-      }
+    if ( defined($cfg->{'exectype'}) && $cfg->{'exectype'}=~ "local" ) {
+        push(@cmd, ('-x', 'local'));
     }
 
-    return @baseCmd;
+    return @cmd;
 }
 
 
@@ -527,18 +307,18 @@ sub setLocale
    my $locale= shift;
 #   $locale = "en_US.UTF-8" if ( !$locale );
 $locale = "ja_JP.utf8" if ( !$locale );
-   $ENV[LC_CTYPE]="$locale";
-   $ENV[LC_NUMERIC]="$locale";
-   $ENV[LC_TIME]="$locale";
-   $ENV[LC_COLLATE]="$locale";
-   $ENV[LC_MONETARY]="$locale";
-   $ENV[LC_MESSAGES]="$locale";
-   $ENV[LC_PAPER]="$locale";
-   $ENV[LC_NAME]="$locale";
-   $ENV[LC_ADDRESS]="$locale";
-   $ENV[LC_TELEPHONE]="$locale";
-   $ENV[LC_MEASUREMENT]="$locale";
-   $ENV[LC_IDENTIFICATION]="$locale";
+   $ENV['LC_CTYPE']="$locale";
+   $ENV['LC_NUMERIC']="$locale";
+   $ENV['LC_TIME']="$locale";
+   $ENV['LC_COLLATE']="$locale";
+   $ENV['LC_MONETARY']="$locale";
+   $ENV['LC_MESSAGES']="$locale";
+   $ENV['LC_PAPER']="$locale";
+   $ENV['LC_NAME']="$locale";
+   $ENV['LC_ADDRESS']="$locale";
+   $ENV['LC_TELEPHONE']="$locale";
+   $ENV['LC_MEASUREMENT']="$locale";
+   $ENV['LC_IDENTIFICATION']="$locale";
 }
 
 sub getLocaleCmd 
@@ -560,4 +340,154 @@ sub getLocaleCmd 
           ."export LC_IDENTIFICATION=\"$locale\"";
 }
 
+sub replaceParameters
+{
+
+    my ($cmd, $outfile, $testCmd, $log) = @_;
+
+    # $self
+# $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
+
+    # $outfile
+    $cmd =~ s/:OUTPATH:/$outfile/g;
+
+    # $ENV
+    $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g;
+
+    # $testCmd
+    $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
+    $cmd =~ s/:OUTPATH:/$outfile/g;
+    $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
+    $cmd =~ s/:PIGPATH:/$testCmd->{'pighome'}/g;
+    $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
+    $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
+    $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g;
+    $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
+    $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
+    $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
+#    $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
+#    $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
+#    $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
+    $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
+    $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
+    $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g;
+    $cmd =~ s/:HCAT_JAR:/$testCmd->{'libjars'}/g;
+
+    if ( $testCmd->{'hadoopSecurity'} eq "secure" ) { 
+      $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g;
+    } else {
+      $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g;
+    }
+
+    return $cmd;
+}
+
+sub getHiveLibs($$)
+{
+    my ($cfg, $log) = @_;
+
+    my $cp;
+    opendir(LIB, $cfg->{'hivelib'}) or die "Cannot open $cfg->{'hivelib'}, $!\n";
+    my @jars = readdir(LIB);
+    foreach (@jars) {
+        /\.jar$/ && do {
+            $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+        };
+    }
+    closedir(LIB);
+    return $cp;
+}
+
+# Pig needs a limited set of the Hive libs, since they include some of the same jars
+# and we get version mismatches if it picks up all the libraries.
+sub getHiveLibsForPig($$)
+{
+    my ($cfg, $log) = @_;
+
+    my $cp;
+    opendir(LIB, $cfg->{'hivelib'}) or die "Cannot open $cfg->{'hivelib'}, $!\n";
+    my @jars = readdir(LIB);
+    foreach (@jars) {
+        /hive-.*\.jar$/ && do {
+            $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+        };
+        /libfb303.jar/ && do {
+            $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+        };
+        /libthrift.jar/ && do {
+            $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+        };
+        /datanucleus-.*\.jar$/ && do {
+            $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+        };
+        /jdo2-api-.*\.jar$/ && do {
+            $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+        };
+        /commons-dbcp-.*\.jar$/ && do {
+            $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+        };
+        /commons-pool-.*\.jar$/ && do {
+            $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+        };
+#       /hbase-.*\.jar$/ && do {
+#           $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+#       };
+#       /zookeeper-.*\.jar$/ && do {
+#           $cp .= $cfg->{'hivelib'} . '/' . $_ . ':';
+#       };
+    }
+    closedir(LIB);
+    return $cp;
+}
+
+sub getHBaseLibs($$)
+{
+    my ($cfg, $log) = @_;
+
+    my $cp;
+    opendir(LIB, $cfg->{'hbaselibdir'}) or die "Cannot open $cfg->{'hbaselibdir'}, $!\n";
+    my @jars = readdir(LIB);
+    foreach (@jars) {
+        /hbase-.*\.jar$/ && do {
+            $cp .= $cfg->{'hbaselibdir'} . '/' . $_ . ':';
+        };
+    }
+    closedir(LIB);
+    opendir(LIB, $cfg->{'zklibdir'}) or die "Cannot open $cfg->{'zklibdir'}, $!\n";
+    my @jars = readdir(LIB);
+    foreach (@jars) {
+        /zookeeper.*\.jar$/ && do {
+            $cp .= $cfg->{'zklibdir'} . '/' . $_ . ':';
+        };
+    }
+    closedir(LIB);
+    return $cp;
+}
+ 
+
+sub getHCatLibs($$)
+{
+    my ($cfg, $log) = @_;
+
+    my $cp;
+    opendir(LIB, $cfg->{'hcatshare'}) or die "Cannot open $cfg->{'hcatshare'}, $!\n";
+    my @jars = readdir(LIB);
+    foreach (@jars) {
+        /hcatalog-[0-9].*\.jar$/ && do {
+            $cp .= $cfg->{'hcatshare'} . '/' . $_ . ':';
+        };
+    }
+    closedir(LIB);
+    opendir(LIB, $cfg->{'hcatlib'}) or die "Cannot open $cfg->{'hcatlib'}, $!\n";
+    my @jars = readdir(LIB);
+    foreach (@jars) {
+        /hbase-storage-handler.*\.jar$/ && do {
+            $cp .= $cfg->{'hcatlib'} . '/' . $_ . ':';
+        };
+    }
+    closedir(LIB);
+    return $cp;
+}
+        
+
 1;

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl?rev=1325181&r1=1325180&r2=1325181&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl Thu Apr 12 10:08:00 2012
@@ -361,8 +361,8 @@ sub findHiveJars()
         die 'Please set $HIVE_HOME\n';
     }
 
-    my $execJar = `ls ../../../../../hive/external/build/ql/hive-exec-*.jar`;
-    my $cliJar = `ls ../../../../../hive/external/build/cli/hive-cli-*.jar`;
+    my $execJar = `ls $ENV{HIVE_HOME}/lib/hive-exec-*.jar`;
+    my $cliJar = `ls $ENV{HIVE_HOME}/lib/hive-cli-*.jar`;
     chomp $execJar;
     chomp $cliJar;
     return ($execJar, $cliJar);

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml?rev=1325181&r1=1325180&r2=1325181&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml Thu Apr 12 10:08:00 2012
@@ -22,8 +22,6 @@
     <path id="udf-classpath">
        <fileset file="../../../../../../build/hcatalog/*.jar" />
        <fileset file="../../../../../../build/ivy/lib/hcatalog/*.jar" />
-       <fileset file="../../../../../../hive/external/build/dist/lib/*.jar" />
-       <fileset file="../../../../../../hive/external/build/hadoopcore/hadoop-0.20.1/*.jar" />
     </path>
 
     <target name="init">

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1325181&r1=1325180&r2=1325181&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java Thu Apr 12 10:08:00 2012
@@ -50,9 +50,9 @@ import org.junit.Test;
 
 public class TestHCatLoaderComplexSchema {
 
-  private static MiniCluster cluster = MiniCluster.buildCluster();
+  //private static MiniCluster cluster = MiniCluster.buildCluster();
   private static Driver driver;
-  private static Properties props;
+  //private static Properties props;
 
   private void dropTable(String tablename) throws IOException, CommandNeedRetryException{
     driver.run("drop table "+tablename);
@@ -87,8 +87,8 @@ public class TestHCatLoaderComplexSchema
     hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
     driver = new Driver(hiveConf);
     SessionState.start(new CliSessionState(hiveConf));
-    props = new Properties();
-    props.setProperty("fs.default.name", cluster.getProperties().getProperty("fs.default.name"));
+    //props = new Properties();
+    //props.setProperty("fs.default.name", cluster.getProperties().getProperty("fs.default.name"));
 
   }
 
@@ -180,7 +180,7 @@ public class TestHCatLoaderComplexSchema
     MockLoader.setData(tablename+"Input", data);
     try {
       createTable(tablename, tableSchema);
-      PigServer server = new PigServer(ExecType.LOCAL, props);
+      PigServer server = new PigServer(ExecType.LOCAL);
       server.setBatchOn();
       server.registerQuery("A = load '"+tablename+"Input' using org.apache.hcatalog.pig.MockLoader() AS "+pigSchema+";");
       Schema dumpedASchema = server.dumpSchema("A");

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/build.xml?rev=1325181&r1=1325180&r2=1325181&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/build.xml (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/build.xml Thu Apr 12 10:08:00 2012
@@ -75,7 +75,7 @@
 
     <!-- ivy properteis set here -->
     <property name="ivy.repo.dir" value="${user.home}/ivyrepo" />
-    <property name="ivy.dir" location="ivy" />
+    <property name="ivy.dir" location="../../ivy" />
     <loadproperties srcfile="${ivy.dir}/libraries.properties"/>
     <property name="asfrepo" value="https://repository.apache.org"/>
     <property name="asfsnapshotrepo" value="${asfrepo}/content/repositories/snapshots"/>
@@ -103,29 +103,14 @@
         <fileset dir="${ivy.lib.dir}" includes="*.jar"/>
         <fileset dir="${hcatalog.dir}/build/hcatalog" includes=" *.jar"/>
         <path refid="common.classpath"/>
-        <fileset dir="${hive.root}/build/hbase-handler" includes="*.jar"/>
     </path>
 
     <path id="test.classpath">
         <pathelement location="${test.build.classes}" />
         <pathelement location="${build.classes}" />
         <pathelement location="conf"/>
-        <pathelement location="${hive.conf.dir}"/>
-        <!-- jars Hadoop depends on -->
-        <fileset dir="${hive.root}/build/hadoopcore/hadoop-${hadoop.version}/" >
-            <include name="**/lib/*.jar" />
-            <include name="hadoop-test-*.jar" />
-        </fileset>
-        <path refid="classpath"/>
-        <pathelement location="${handler.jar}"/>
-        <!-- jars Hive depends on -->
-        <fileset dir="${hive.root}/build/ivy/lib/default/">
-            <include name="**/*.jar" />
-            <exclude name="*hbase*.jar" />
-            <exclude name="zookeeper*.jar" />
-            <exclude name="guava*.jar" />
-            <exclude name="slf4j*.jar" />
-        </fileset>
+        <fileset dir="${ivy.lib.dir}" includes="*.jar"/>
+        <fileset dir="${hcatalog.dir}/build/hcatalog" includes=" *.jar"/>
     </path>
 
     <!--
@@ -184,6 +169,11 @@
         <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
                       pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}" conf="common"/>
         <ivy:cachepath pathid="compile.classpath" conf="common"/>
+        <ivy:resolve settingsRef="${ant.project.name}.ivy.settings" conf="default"/>
+        <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+                      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+                      conf="default"/>
+        <ivy:cachepath pathid="compile.classpath" conf="default"/>
     </target>
 
     <target name="ivy-releaseaudit" depends="ivy-init" description="Resolve, Retrieve Ivy-managed artifacts for releaseaudit configuration">

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml?rev=1325181&r1=1325180&r2=1325181&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml Thu Apr 12 10:08:00 2012
@@ -47,10 +47,81 @@
             <artifact name="zookeeper" type="test-jar" ext="jar" m:classifier="tests"/>
         </dependency>
 
-    <!-- hbase test dependencies -->
-    <dependency org="com.github.stephenc.high-scale-lib" name="high-scale-lib" rev="${high-scale-lib.version}" conf="common->master" />
-    <dependency org="com.google.guava" name="guava" rev="${guava.version}" conf="common->master" />
-    <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}" conf="common->master" />
+        <dependency org="org.apache.hadoop" name="hadoop-core"
+          rev="${hadoop-core.version}" conf="common->master" />
+        <dependency org="org.apache.hadoop" name="hadoop-test"
+          rev="${hadoop-test.version}" conf="common->master" />
+        <dependency org="org.apache.hive" name="hive-metastore"
+          rev="${hive.version}" conf="common->master"/>
+        <dependency org="org.apache.hive" name="hive-common"
+          rev="${hive.version}" conf="common->master"/>
+        <dependency org="org.apache.hive" name="hive-cli"
+          rev="${hive.version}" conf="common->master"/>
+        <dependency org="org.apache.hive" name="hive-exec"
+          rev="${hive.version}" conf="common->master"/>
+        <dependency org="junit" name="junit" rev="${junit.version}"
+          conf="common->master"/>
+        <dependency org="commons-lang" name="commons-lang" rev="${commons-lang.version}"
+          conf="common->master"/>
+        <dependency org="commons-logging" name="commons-logging"
+          rev="${commons-logging.version}" conf="common->master"/>
+        <dependency org="commons-logging" name="commons-logging-api"
+          rev="${commons-logging.version}" conf="common->master"/>
 
+        <!-- hbase test dependencies -->
+        <dependency org="com.github.stephenc.high-scale-lib" name="high-scale-lib" rev="${high-scale-lib.version}" conf="common->master" />
+        <dependency org="com.google.guava" name="guava" rev="${guava.version}" conf="common->master" />
+        <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}" conf="common->master" />
+
+        <!-- needed to run tests -->
+        <dependency org="commons-configuration" name="commons-configuration"
+          rev="${commons-configuration.version}" conf="default"/>
+        <dependency org="org.codehaus.jackson" name="jackson-mapper-asl"
+          rev="${jackson.version}" conf="common->master"/>
+        <dependency org="org.codehaus.jackson" name="jackson-core-asl"
+          rev="${jackson.version}" conf="common->master"/>
+        <dependency org="org.slf4j" name="slf4j-api" rev="${slf4j.version}"
+          conf="common->master"/>
+        <dependency org="org.slf4j" name="slf4j-log4j12" rev="${slf4j.version}"
+          conf="common->master"/>
+        <dependency org="log4j" name="log4j" rev="${log4j.version}"
+          conf="common->master"/>
+        <dependency org="javax.jdo" name="jdo2-api" rev="${jdo.version}"
+          conf="default"/>
+        <dependency org="org.datanucleus" name="datanucleus-core"
+          rev="${datanucleus-core.version}" conf="default"/>
+        <dependency org="org.datanucleus" name="datanucleus-connectionpool"
+          rev="${datanucleus-connectionpool.version}" conf="default"/>
+        <dependency org="org.datanucleus" name="datanucleus-enhancer"
+          rev="${datanucleus-enhancer.version}" conf="default"/>
+        <dependency org="org.datanucleus" name="datanucleus-rdbms"
+          rev="${datanucleus-rdbms.version}" conf="default"/>
+        <dependency org="commons-dbcp" name="commons-dbcp" rev="${commons-dbcp.version}"
+            conf="common->master">
+          <exclude module="commons-pool" />
+          <exclude org="org.apache.geronimo.specs" module="geronimo-jta_1.1_spec"/>
+        </dependency>
+        <dependency org="commons-pool" name="commons-pool" rev="${commons-pool.version}"
+          conf="default"/>
+        <dependency org="org.apache.derby" name="derby" rev="${derby.version}"
+          conf="default"/>
+        <dependency org="commons-configuration" name="commons-configuration"
+          rev="${commons-configuration.version}" conf="default"/>
+        <dependency org="commons-httpclient" name="commons-httpclient"
+          rev="${commons-httpclient.version}" conf="default"/>
+        <dependency org="org.apache.hive" name="hive-builtins"
+          rev="${hive.version}" conf="common->master"/>
+        <dependency org="org.mortbay.jetty" name="jetty"
+          rev="${jetty.version}" conf="default"/>
+        <dependency org="org.mortbay.jetty" name="jetty-util"
+          rev="${jetty.version}" conf="default"/>
+        <dependency org="org.apache.thrift" name="libfb303" rev="${fb303.version}"
+          conf="common->master"/>
+        <dependency org="org.antlr" name="antlr-runtime" rev="${antlr.version}"
+          conf="common->master" />
+        <dependency org="org.apache.hive" name="hive-hbase-handler"
+          rev="${hive.version}" conf="common->master"/>
+
+ 
     </dependencies>
 </ivy-module>



Mime
View raw message