incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1365722 [4/11] - in /incubator/hcatalog/trunk: ./ ant/ conf/ hcatalog-pig-adapter/ ivy/ src/docs/src/documentation/content/xdocs/ src/docs/src/documentation/content/xdocs/images/ src/java/org/apache/hcatalog/mapreduce/ src/test/e2e/templet...
Date Wed, 25 Jul 2012 20:29:49 GMT
Added: incubator/hcatalog/trunk/src/test/e2e/templeton/drivers/TestDriverCurl.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/drivers/TestDriverCurl.pm (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/drivers/TestDriverCurl.pm Wed Jul 25 20:29:44 2012
@@ -0,0 +1,1325 @@
+############################################################################           
+#  Licensed to the Apache Software Foundation (ASF) under one or more                  
+#  contributor license agreements.  See the NOTICE file distributed with               
+#  this work for additional information regarding copyright ownership.                 
+#  The ASF licenses this file to You under the Apache License, Version 2.0             
+#  (the "License"); you may not use this file except in compliance with                
+#  the License.  You may obtain a copy of the License at                               
+#                                                                                      
+#      http://www.apache.org/licenses/LICENSE-2.0                                      
+#                                                                                      
+#  Unless required by applicable law or agreed to in writing, software                 
+#  distributed under the License is distributed on an "AS IS" BASIS,                   
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.            
+#  See the License for the specific language governing permissions and                 
+#  limitations under the License.                                                      
+                                                                                       
+package TestDriverCurl;
+
+###########################################################################
+# Class: TestDriver
+# A base class for TestDrivers.
+# 
+
+use TestDriverFactory;
+use TestReport;
+use File::Path;
+use IPC::Run;
+use Data::Dump qw(dump);
+use JSON;
+use HTTP::Daemon;
+use HTTP::Status;
+use Data::Compare;
+use strict;
+use English;
+use Storable qw(dclone);
+use File::Glob ':glob';
+
+my $passedStr = 'passed';
+my $failedStr = 'failed';
+my $abortedStr = 'aborted';
+my $skippedStr = 'skipped';
+my $dependStr = 'failed_dependency';
+
+
+##############################################################################
+#  Sub: printResults
+#  Static function, can be used by test_harness.pl
+#  Print the results so far, given the testStatuses hash.
+#
+# Paramaters:
+# testStatuses - reference to hash of test status results.
+# log    - reference to file handle to print results to.
+# prefix - A title to prefix to the results
+#
+# Returns:
+# None.
+#
+sub printResults
+  {
+    my ($testStatuses, $log, $prefix) = @_;
+
+    my ($pass, $fail, $abort, $depend, $skipped) = (0, 0, 0, 0, 0);
+
+    foreach (keys(%$testStatuses)) {
+      ($testStatuses->{$_} eq $passedStr) && $pass++;
+      ($testStatuses->{$_} eq $failedStr) && $fail++;
+      ($testStatuses->{$_} eq $abortedStr) && $abort++;
+      ($testStatuses->{$_} eq $dependStr) && $depend++;
+      ($testStatuses->{$_} eq $skippedStr) && $skipped++;
+    }
+
+    my $msg = "$prefix, PASSED: $pass FAILED: $fail SKIPPED: $skipped ABORTED: $abort "
+      . "FAILED DEPENDENCY: $depend";
+    print $log "$msg\n";
+    print "$msg\n";
+         
+  }
+
+##############################################################################
+#  Sub: printGroupResultsXml
+#  Print the results for the group using junit xml schema using values from the testStatuses hash.
+#
+# Paramaters:
+# $report       - the report object to use to generate the report
+# $groupName    - the name of the group to report totals for
+# $testStatuses - the hash containing the results for the tests run so far
+# $totalDuration- The total time it took to run the group of tests
+#
+# Returns:
+# None.
+#
+sub printGroupResultsXml
+  {
+    my ( $report, $groupName, $testStatuses,  $totalDuration) = @_;
+    $totalDuration=0 if  ( !$totalDuration );
+
+    my ($pass, $fail, $abort, $depend) = (0, 0, 0, 0);
+
+    foreach my $key (keys(%$testStatuses)) {
+      if ( $key =~ /^$groupName/ ) {
+        ($testStatuses->{$key} eq $passedStr) && $pass++;
+        ($testStatuses->{$key} eq $failedStr) && $fail++;
+        ($testStatuses->{$key} eq $abortedStr) && $abort++;
+        ($testStatuses->{$key} eq $dependStr) && $depend++;
+      }
+    }
+
+    my $total= $pass + $fail + $abort;
+    $report->totals( $groupName, $total, $fail, $abort, $totalDuration );
+
+  }
+
+##############################################################################
+#  Sub: new
+# Constructor, should only be called by TestDriverFactory.
+#
+# Paramaters:
+# None
+#
+# Returns:
+# None.
+sub new
+  {
+    my $proto = shift;
+    my $class = ref($proto) || $proto;
+    my $self = {};
+
+    bless($self, $class);
+
+    $self->{'wrong_execution_mode'} = "_xyz_wrong_execution_mode_zyx_";
+
+    return $self;
+  }
+
+##############################################################################
+#  Sub: globalSetup
+# Set up before any tests are run.  This gives each individual driver a chance to do
+# setup.  This function will only be called once, before all the tests are
+# run.  A driver need not implement it.  It is a virtual function.
+#
+# Paramaters:
+# globalHash - Top level hash from config file (does not have any group
+# or test information in it).
+# log - log file handle
+#
+# Returns:
+# None
+#
+sub globalSetup
+  {
+    my ($self, $globalHash, $log) = @_;
+    my $subName = (caller(0))[3];
+
+
+    # Setup the output path
+    my $me = `whoami`;
+    chomp $me;
+    $globalHash->{'runid'} = $me . "." . time;
+
+    # if "-ignore false" was provided on the command line,
+    # it means do run tests even when marked as 'ignore'
+    if (defined($globalHash->{'ignore'}) && $globalHash->{'ignore'} eq 'false') {
+      $self->{'ignore'} = 'false';
+    }
+
+    if (! defined $globalHash->{'localpathbase'}) {
+      $globalHash->{'localpathbase'} = '/tmp';
+    }
+
+    $globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/";
+    $globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/";
+    $globalHash->{'webhdfs_url'} = $ENV{'WEBHDFS_URL'};
+    $globalHash->{'templeton_url'} = $ENV{'TEMPLETON_URL'};
+    $globalHash->{'current_user'} = $ENV{'USER_NAME'};
+    $globalHash->{'current_group_user'} = $ENV{'GROUP_USER_NAME'};
+    $globalHash->{'current_other_user'} = $ENV{'OTHER_USER_NAME'};
+    $globalHash->{'current_group'} = $ENV{'GROUP_NAME'};
+    $globalHash->{'keytab_dir'} = $ENV{'KEYTAB_DIR'};
+
+
+
+    $globalHash->{'inpdir_local'} = $ENV{'TH_INPDIR_LOCAL'};
+    $globalHash->{'inpdir_hdfs'} = $ENV{'TH_INPDIR_HDFS'};
+
+    $globalHash->{'is_secure_mode'} = $ENV{'SECURE_MODE'};
+
+    # add libexec location to the path
+    if (defined($ENV{'PATH'})) {
+      $ENV{'PATH'} = $globalHash->{'scriptPath'} . ":" . $ENV{'PATH'};
+    } else {
+      $ENV{'PATH'} = $globalHash->{'scriptPath'};
+    }
+
+    IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or 
+        die "Cannot create localpath directory " . $globalHash->{'localpath'} .
+          " " . "$ERRNO\n";
+
+    # Create the temporary directory
+    IPC::Run::run(['mkdir', '-p', $globalHash->{'tmpPath'}], \undef, $log, $log) or 
+        die "Cannot create temporary directory " . $globalHash->{'tmpPath'} .
+          " " . "$ERRNO\n";
+
+  }
+
+###############################################################################
+# Sub: globalCleanup
+# Clean up after all tests have run.  This gives each individual driver a chance to do
+# cleanup.  This function will only be called once, after all the tests are
+# run.  A driver need not implement it.  It is a virtual function.
+#
+# Paramaters:
+# globalHash - Top level hash from config file (does not have any group
+# or test information in it).
+# log - log file handle
+#
+# Returns:
+# None
+sub globalCleanup
+  {
+    my ($self, $globalHash, $log) = @_;
+
+    IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or 
+        warn "Cannot remove temporary directory " . $globalHash->{'tmpPath'} .
+          " " . "$ERRNO\n";
+
+  }
+
+###############################################################################
+# Sub: runTest
+# Run a test.  This is a pure virtual function.
+#
+# Parameters:
+# testcmd - reference to hash with meta tags and command to run tests.
+# Interpretation of the tags and the command are up to the subclass.
+# log - reference to a stream pointer for the logs
+#
+# Returns:
+# @returns reference to hash.  Contents of hash are defined by the subclass.
+#
+sub runTest
+  {
+    my ($self, $testCmd, $log) = @_;
+    my $subName  = (caller(0))[3];
+
+    # Handle the various methods of running used in 
+    # the original TestDrivers
+
+    if ( $testCmd->{'url'} ) {
+      return $self->runCurlCmd( $testCmd, $log);
+    } else {
+      die "$subName FATAL Did not find a testCmd that " .
+        "I know how to handle : " . $testCmd->{'Curl'};
+    }
+
+
+  }
+
+###############################################################################
+
+sub replaceParameters
+  {
+    my ($self, $testCmd, $aPfix, $log) = @_;
+
+    my $url =  $testCmd->{$aPfix . 'url'};
+    $url =~ s/:WEBHDFS_URL:/$testCmd->{'webhdfs_url'}/g;
+    $url =~ s/:TEMPLETON_URL:/$testCmd->{'templeton_url'}/g;
+    $url = $self->replaceParametersInArg($url, $testCmd, $log);
+    $testCmd->{$aPfix . 'url'} = $url;
+
+    $testCmd->{$aPfix . 'upload_file'} = 
+      $self->replaceParametersInArg($testCmd->{$aPfix . 'upload_file'}, $testCmd, $log);
+
+    $testCmd->{$aPfix . 'user_name'} = 
+      $self->replaceParametersInArg($testCmd->{$aPfix . 'user_name'}, $testCmd, $log);
+
+    if (defined $testCmd->{$aPfix . 'post_options'}) {
+      my @options = @{$testCmd->{$aPfix . 'post_options'}};
+      my @new_options = ();
+      foreach my $option (@options) {
+        $option = $self->replaceParametersInArg($option, $testCmd, $log);
+        push @new_options, ($option);
+      }
+      $testCmd->{$aPfix . 'post_options'} = \@new_options;
+    }    
+    if (defined $testCmd->{$aPfix . 'json_field_substr_match'}) {
+      my $json_matches = $testCmd->{$aPfix . 'json_field_substr_match'};
+      my @keys = keys %{$json_matches};
+
+      foreach my $key (@keys) {
+        my $new_value = $self->replaceParametersInArg($json_matches->{$key}, $testCmd, $log);
+        $json_matches->{$key} = $new_value;
+      }
+    }    
+
+
+  }
+
+###############################################################################
+sub replaceParametersInArg
+  {
+    my ($self, $arg, $testCmd, $log) = @_;
+    if(! defined $arg){
+      return $arg;
+    }
+    my $outdir = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'};
+    $arg =~ s/:UNAME:/$testCmd->{'current_user'}/g;
+    $arg =~ s/:UNAME_GROUP:/$testCmd->{'current_group_user'}/g;
+    $arg =~ s/:UNAME_OTHER:/$testCmd->{'current_other_user'}/g;
+    $arg =~ s/:UGROUP:/$testCmd->{'current_group'}/g;
+    $arg =~ s/:OUTDIR:/$outdir/g;
+    $arg =~ s/:INPDIR_HDFS:/$testCmd->{'inpdir_hdfs'}/g;
+    $arg =~ s/:INPDIR_LOCAL:/$testCmd->{'inpdir_local'}/g;
+    $arg =~ s/:TNUM:/$testCmd->{'num'}/g;
+    return $arg;
+  }
+
+
+
+###############################################################################
+
+sub getBaseCurlCmd(){
+  my ($self) = @_; 
+  my @curl_cmd = ("curl", '--silent','--show-error', '-H','Expect:');
+  if (defined $ENV{'SOCKS_PROXY'}) {
+    push @curl_cmd, ('--socks5-hostname', $ENV{'SOCKS_PROXY'});
+  }
+  return @curl_cmd;
+
+}
+
+###############################################################################
+sub runCurlCmd(){
+  my ($self, $testCmd, $log) = @_;
+  if (defined $testCmd->{'upload_file'}) {
+    return $self->upload_file($testCmd,$log);
+  } else {
+    #if there are setup steps, run them first
+    if (defined $testCmd->{'setup'}) {
+      my $i = 0;
+      foreach my $setupCmd (@{$testCmd->{'setup'}}){
+        $i++;
+        print $log "\nRUNNING SETUP COMMAND: $i\n";
+        my $pfix = "setup_${i}_";
+        my $setupTestCmd = $self->createSetupCmd($testCmd, $setupCmd, $pfix, $log);
+        my $setupResult = $self->execCurlCmd($setupTestCmd, $pfix, $log);
+        
+        #if status code is set in setup, check if it matches results
+        if(defined $setupTestCmd->{"${pfix}status_code"}){
+          $self->checkResStatusCode($setupResult, $setupTestCmd->{"${pfix}status_code"}, $log);
+        }
+      }
+    }
+    return $self->execCurlCmd($testCmd, "", $log);
+  }
+}
+###############################################################################
+sub createSetupCmd(){
+  my ($self, $testCmd, $setupCmd, $pfix, $log) = @_;
+  my $newTestCmd = dclone ($testCmd);
+  for my $key (keys %$setupCmd){
+    $newTestCmd->{$pfix . $key} = $setupCmd->{$key};
+  }
+  return $newTestCmd;
+}
+
+###############################################################################
+sub upload_file(){
+  my ($self, $testCmd, $log) = @_;
+  $testCmd->{'method'} = 'PUT';
+  my $result = $self->execCurlCmd($testCmd, "", $log);
+  my $checkRes = $self->checkResStatusCode($result, 100, $log);
+  if ($checkRes == 0) {
+    #fail
+    return 0;
+  }
+  my $header = $result->{'header_fields'};
+
+  #final url where the file should be stored
+  my $location = $header->{'Location'};
+  $testCmd->{'url'} = $location;
+    
+  $result = $self->execCurlCmd($testCmd, "", $log);
+  return $result;
+}
+
+###############################################################################
+sub execCurlCmd(){
+  my ($self, $testCmd, $argPrefix, $log) = @_;
+  my @curl_cmd = $self->getBaseCurlCmd();
+  # Set up file locations
+  my $subName = (caller(0))[3];
+
+  my $filePrefix = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $argPrefix . $testCmd->{'num'}; 
+  my $cmd_body =  $filePrefix . ".cmd_body";
+
+  #results
+  my $res_header = $filePrefix . ".res_header";
+  my $res_body = $filePrefix . ".res_body";
+
+  my $outdir = $filePrefix .  ".out";
+  my $stdoutfile = "$outdir/stdout";
+  my $stderrfile = "$outdir/stderr";
+
+  mkpath( [ $outdir ] , 0, 0755) if ( ! -e outdir );
+  if ( ! -e $outdir ) {
+    print $log "$0.$subName FATAL could not mkdir $outdir\n";
+    die "$0.$subName FATAL could not mkdir $outdir\n";
+  }
+
+  $self->replaceParameters($testCmd, $argPrefix, $log );
+
+  my $method = $testCmd->{ $argPrefix . 'method'};
+
+  my $url = $testCmd->{ $argPrefix . 'url'};
+
+  my @options = ();
+  if (defined $testCmd->{$argPrefix . 'post_options'}) {
+    @options = @{$testCmd->{$argPrefix . 'post_options'}};
+  }
+
+  #handle authentication based on secure mode
+  my $user_name = $testCmd->{ $argPrefix . 'user_name' }; 
+  if (defined $testCmd->{'is_secure_mode'} &&  $testCmd->{'is_secure_mode'} =~ /y.*/i) {
+    push @curl_cmd, ('--negotiate', '-u', ':');
+
+    #if keytab dir is defined, look for a keytab file for user and do a kinit
+    if(defined  $testCmd->{'keytab_dir'} && defined $user_name){
+      $user_name =~ /(.*?)(\/|$)/;
+      my $just_uname = $1; #uname without principal
+      my $keytab_dir = $testCmd->{'keytab_dir'};
+      print $log "regex " .  "${keytab_dir}/*${just_uname}\.*keytab";
+      my @files = bsd_glob(  "${keytab_dir}/*${just_uname}\.*keytab" );
+      if(scalar @files == 0){
+        die "Could not find keytab file for user $user_name in $keytab_dir";
+      } elsif(scalar @files > 1){
+        die "More than one keytab file found for user $user_name in $keytab_dir";
+      }
+      my @cmd = ('kinit', '-k', '-t', $files[0], $user_name);
+      print $log "Command  @cmd";
+      IPC::Run::run(\@cmd, \undef, $log, $log) or 
+          die "Could not kinit as $user_name using " .  $files[0] . " $ERRNO";
+    }
+
+  } else { 
+    #if mode is unsecure
+    if (defined $user_name) {
+      my $user_param = "user.name=${user_name}";
+      if ($method eq 'POST' ) {
+        push @options, $user_param;
+      } else {
+        if ($url =~ /\?/) {
+          #has some parameters in url
+          $url = $url . '&' . $user_param;
+        } else {
+          $url = $url . '?' . $user_param;
+        }
+      }
+    }
+
+  }
+  
+  if (defined $testCmd->{'format_header'}) {
+    push @curl_cmd, ('-H', $testCmd->{'format_header'});
+  }
+
+  
+  
+  
+  if (defined $testCmd->{$argPrefix . 'format_header'}) {
+    push @curl_cmd, ('-H', $testCmd->{$argPrefix . 'format_header'});
+  }
+
+  if (defined $testCmd->{$argPrefix . 'upload_file'}) {
+    push @curl_cmd, ('-T', $testCmd->{$argPrefix . 'upload_file'});
+  }
+
+  #    if(!defined $testCmd->{'post_options'}){
+  #	$testCmd->{'post_options'} = \();
+  #    }
+
+  if (defined $testCmd->{$argPrefix . 'check_call_back'}) {
+    my $d = HTTP::Daemon->new || die;
+    $testCmd->{'http_daemon'} = $d;
+    $testCmd->{'callback_url'} = $d->url . 'templeton/$jobId';
+    push @curl_cmd, ('-d', 'callback=' . $testCmd->{'callback_url'});
+    #	push ${testCmd->{'post_options'}}, ('callback=' . $testCmd->{'callback_url'});
+    #	#my @options = @{$testCmd->{'post_options'}};
+    #	print $log "post options  @options\n";
+  }
+
+  foreach my $option (@options) {
+    push @curl_cmd, ('-d', $option);
+  }
+
+  push @curl_cmd, ("-X", $method, "-o", $res_body, "-D", $res_header);  
+  push @curl_cmd, ($url);
+
+  print $log "$0:$subName Going to run command : " .  join (' ', @curl_cmd);
+  print $log "\n";
+
+
+  my %result;
+  my $out;
+  my $err;
+  IPC::Run::run(\@curl_cmd, \undef, $out, $err) 
+      or die "Failed running curl cmd " . join ' ', @curl_cmd;
+
+  $result{'rc'} = $? >> 8;
+  $result{'stderr'} = $err;
+  $result{'stdout'} = $out;
+  $result{'body'} = `cat $res_body`;
+ 
+  my @full_header = `cat $res_header`;
+  $result{'full_header'} = join '\n', @full_header;
+
+  #find the final http status code
+  for my $line ( @full_header){
+    if($line =~ /.*(HTTP\/1.1)\s+(\S+)/){
+      $result{'status_code'}  = $2;
+    }
+  }
+
+  my %header_field;
+  foreach my $line (@full_header) {
+    chomp $line;
+    $line =~ /(.*?)\s*:\s*(.*)/;
+    if (defined $1 && defined $2 ) {
+      $header_field{$1} = $2;
+    }
+  }
+  $result{'header_fields'} = \%header_field;
+
+  print $log "result : " . dump(%result);
+  #dump(%result);
+    
+  return \%result;
+
+}
+
+
+###############################################################################
+# Sub: generateBenchmark
+# Generate benchmark results.  This is a pure virtual function.
+#
+# Parameters:
+# benchmarkcmd - reference to hash with meta tags and command to
+# generate benchmark.  Interpretation of the tags and the command are up to
+# log - reference to a stream pointer for the logs
+# the subclass.
+#
+# Returns:
+# @returns reference to hash.  Contents of hash are defined by the subclass.
+#
+sub generateBenchmark
+  {
+    my %result;
+    return \%result;
+  }
+
+###############################################################################
+# Sub: compare
+# Compare the results of the test run with the generated benchmark results.  
+# This is a pure virtual function.
+#
+# Parameters:
+# benchmarkcmd - reference to hash with meta tags and command to
+# testResult - reference to hash returned by runTest.
+# benchmarkResult - reference to hash returned by generateBenchmark.
+# log - reference to a stream pointer for the logs
+# testHash - reference to hash with meta tags and commands
+#
+# Returns:
+# @returns reference true if results are the same, false otherwise.  "the
+# same" is defined by the subclass.
+#
+sub compare
+  {
+
+    my ($self, $testResult, $benchmarkResult, $log, $testCmd) = @_;
+    my $subName  = (caller(0))[3];
+
+    my $result = 1;             # until proven wrong...
+    if (defined $testCmd->{'status_code'}) {
+      my $res = $self->checkResStatusCode($testResult, $testCmd->{'status_code'}, $log);
+      if ($res == 0) {
+        $result = 0;
+      }
+    }
+
+    my $json_hash;
+    my %json_info;
+    if (defined $testCmd->{'json_field_substr_match'} || $testCmd->{'json_field_match_object'}) {
+      my $json = new JSON;
+      $json_hash = $json->utf8->decode($testResult->{'body'});
+      my $json_matches = $testCmd->{'json_field_substr_match'};
+      my $json_matches_object = $testCmd->{'json_field_match_object'};
+
+      %json_info = %$json_hash;
+      if (defined $json_info{'info'}) {
+        %json_info = %{$json_info{'info'}};
+        
+      }
+      print $log "\n\n json_info";
+      print $log dump(%json_info);
+      print $log "\n\n";
+
+      if (defined $json_hash->{'id'}) {
+        print STDERR "jobid " . $json_hash->{'id'} . "\n";        
+        $json_info{'id'} = $json_hash->{'id'};
+      }
+
+      if(defined $json_matches->{'location_perms'} || defined $json_matches->{'location_group'}){
+        $self->setLocationPermGroup(\%json_info, $testCmd, $log);
+      }
+
+      foreach my $key (keys %$json_matches) {
+        my $json_field_val = $json_info{$key};
+        if( (ref($json_field_val) && ! UNIVERSAL::isa($json_field_val,'SCALAR')) ||
+            (!ref($json_field_val) && ! UNIVERSAL::isa(\$json_field_val,'SCALAR')) ){
+          #flatten the object into a string
+          $json_field_val = dump($json_field_val);
+        }
+        my $regex_expected_value = $json_matches->{$key};
+        print $log "Comparing $key: $json_field_val with regex /$regex_expected_value/\n";
+
+        if ($json_field_val !~ /$regex_expected_value/s) {
+          print $log "$0::$subName INFO check failed:" 
+            . " json pattern check failed. For field "
+              . "$key, regex <" . $regex_expected_value 
+                . "> did not match the result <" . $json_field_val
+                  . ">\n";
+          $result = 0;
+        }
+      }
+
+      foreach my $key (keys %$json_matches_object) {
+        my $json_field_val = $json_info{$key};
+        my $regex_expected_obj = $json->utf8->decode($json_matches_object->{$key});
+        print $log "Comparing $key: " . dump($json_field_val) . ",expected value:  " . dump($regex_expected_obj);
+
+        if (!Compare($json_field_val, $regex_expected_obj)) {
+          print $log "$0::$subName INFO check failed:" 
+            . " json compare failed. For field "
+              . "$key, regex <" . dump($regex_expected_obj)
+                . "> did not match the result <" . dump($json_field_val)
+                  . ">\n";
+          $result = 0;
+        }
+      }
+
+
+    }
+    
+    #kill it if there is a request to kill
+    if($testCmd->{'kill_job_timeout'}){
+      sleep $testCmd->{'kill_job_timeout'};
+      my $jobid = $json_hash->{'id'};
+      if (!defined $jobid) {
+        print $log "$0::$subName INFO check failed: " 
+          . "no jobid (id field)found in result";
+        $result = 0;
+      } else {
+        $self->killJob($testCmd, $jobid, $log);
+      }
+    }
+
+
+    #try to get the call back url request until timeout
+    if ($result == 1 && defined $testCmd->{'check_call_back'}) {
+      my $d = $testCmd->{'http_daemon'};
+      $d->timeout(300);         #wait for 5 mins
+      my $url_requested;
+      $testCmd->{'callback_url'} =~ s/\$jobId/$json_hash->{'id'}/g;
+      print $log "Expanded callback url : <" . $testCmd->{'callback_url'} . ">\n";
+      do{
+        print $log "Waiting for call back url request\n";
+        if (my $c = $d->accept) {
+          if (my $r = $c->get_request) {
+            my $durl = $d->url;
+            chop $durl;
+            $url_requested = $durl . $r->uri->path ;
+            print $log "Got request at url <" .  $url_requested  . ">\n";
+            $c->send_status_line(200);
+            $c->close;
+          }
+          undef($c);
+        } else {
+          print $log "Timeout on wait on call back url"  . "\n";
+          $result = 0;
+        }
+      }while (defined $url_requested && $url_requested  ne $testCmd->{'callback_url'});
+      $d->close;
+      if (!defined $url_requested || $url_requested  ne $testCmd->{'callback_url'}) {
+        print $log "failed to recieve request on call back url";
+        $result = 0;
+      }
+
+    }
+
+    
+    if (defined $testCmd->{'check_job_created'} || defined  $testCmd->{'check_job_complete'}) {
+      my $jobid = $json_hash->{'id'};
+      if (!defined $jobid) {
+        print $log "$0::$subName INFO check failed: " 
+          . "no jobid (id field)found in result";
+        $result = 0;
+      } else {
+        my $jobResult = $self->getJobResult($testCmd, $jobid, $log);
+        my $json = new JSON;
+        my $res_hash = $json->utf8->decode($jobResult->{'body'});
+        if (! defined $res_hash->{'status'}) {
+          print $log "$0::$subName INFO check failed: " 
+            . "jobresult not defined ";
+          $result = 0;
+        }
+        if ($testCmd->{'check_job_complete'}) {
+          my $jobComplete;
+          my $NUM_RETRIES = 60;
+          my $SLEEP_BETWEEN_RETRIES = 5;
+
+          #first wait for job completion
+          while ($NUM_RETRIES-- > 0) {
+            $jobComplete = $res_hash->{'status'}->{'jobComplete'};
+            if (defined $jobComplete && lc($jobComplete) eq "true") {
+              last;
+            }
+            sleep $SLEEP_BETWEEN_RETRIES;
+            $jobResult = $self->getJobResult($testCmd, $jobid, $log);
+            $json = new JSON;
+            $res_hash = $json->utf8->decode($jobResult->{'body'});
+          }
+          if ( (!defined $jobComplete) || lc($jobComplete) ne "true") {
+            print $log "$0::$subName INFO check failed: " 
+              . " timeout on wait for job completion ";
+            $result = 0;
+          } else { 
+            # job has completed, check the runState value
+            my $runState = $res_hash->{'status'}->{'runState'};
+            my $runStateVal = $self->getRunStateNum($testCmd->{'check_job_complete'});
+            if ( (!defined $runState) || $runState ne $runStateVal) {
+              print $log "check_job_complete failed. got runState  $runState,  expected  $runStateVal";
+              $result = 0;
+            }
+          }
+        }
+      }
+    }
+    return $result;
+  }
+
+###############################################################################
+sub  setLocationPermGroup{
+  my ($self, $job_info, $testCmd, $log) = @_;
+  my $location = $job_info->{'location'};
+  $location =~ /hdfs.*:\d+(\/.*)\/(.*)/;  
+  my $dir = $1;
+  my $file = $2;
+
+  my $testCmdBasics = $self->copyTestBasicConfig($testCmd);
+  $testCmdBasics->{'method'} = 'GET';
+  $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_checkFile";
+  $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1' 
+    . $dir . '?op=LISTSTATUS';
+
+
+  my $result =  $self->execCurlCmd($testCmdBasics, "", $log);
+
+  my $json = new JSON;
+  my $json_hash = $json->utf8->decode($result->{'body'});
+  my @filestatuses = @{$json_hash->{'FileStatuses'}->{'FileStatus'}};
+  foreach my $filestatus (@filestatuses){
+    if($filestatus->{'pathSuffix'} eq $file){
+      $job_info->{'location_perms'} =  numPermToStringPerm($filestatus->{'permission'});
+      $job_info->{'location_group'} = $filestatus->{'group'};
+      $job_info->{'location_owner'} = $filestatus->{'owner'};
+      last;
+    }
+
+  }
+
+}
+
+###############################################################################
+#convert decimal string to binary string
+sub dec2bin {
+    my $decimal = shift;
+    my $binary = unpack("B32", pack("N", $decimal));
+    $binary =~ s/^0+(?=\d)//;   # remove leading zeros                                                                                                                                                                                                                                                   
+    return $binary;
+}
+
+###############################################################################
+#convert single digit of the numeric permission format to string format
+sub digitPermToStringPerm{
+    my $numPerm = shift;
+    my $binaryPerm = dec2bin($numPerm);
+    my $stringPerm = "";
+    if($binaryPerm =~ /1\d\d$/){
+        $stringPerm .= "r";
+      }else{
+        $stringPerm .= "-";
+    }
+
+    if($binaryPerm =~ /\d1\d$/){
+        $stringPerm .= "w";
+      }else{
+        $stringPerm .= "-";
+    }
+
+    if($binaryPerm =~ /\d\d1$/){
+        $stringPerm .= "x";
+      }else{
+        $stringPerm .= "-";
+    }
+
+}
+
+###############################################################################
+# convert numeric permission format to string format
+sub numPermToStringPerm{
+    my $numPerm = shift;
+    $numPerm =~ /(\d)(\d)(\d)$/;
+    return digitPermToStringPerm($1)
+        . digitPermToStringPerm($2)
+        . digitPermToStringPerm($3);
+
+}
+
+###############################################################################
+sub getRunStateNum{
+  my ($self, $job_complete_state) = @_;
+  if (lc($job_complete_state) eq 'success') {
+    return 2;
+  } elsif (lc($job_complete_state) eq 'failure') {
+    return 3;
+  } elsif (lc($job_complete_state) eq 'killed') {
+    return 5;
+  }
+
+}
+
+
+###############################################################################
+sub getJobResult{
+  my ($self, $testCmd, $jobid, $log) = @_;
+  my $testCmdBasics = $self->copyTestBasicConfig($testCmd);
+  $testCmdBasics->{'method'} = 'GET';
+  $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_jobStatusCheck";
+  $testCmdBasics->{'url'} = ':TEMPLETON_URL:/templeton/v1/queue/' 
+    . $jobid . '?' . "user.name=:UNAME:" ;
+  return $self->execCurlCmd($testCmdBasics, "", $log);
+}
+###############################################################################
+sub killJob{
+  my ($self, $testCmd, $jobid, $log) = @_;
+  my $testCmdBasics = $self->copyTestBasicConfig($testCmd);
+  $testCmdBasics->{'method'} = 'DELETE';
+  $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_killJob";
+  $testCmdBasics->{'url'} = ':TEMPLETON_URL:/templeton/v1/queue/' 
+    . $jobid . '?' . "user.name=:UNAME:" ;
+  return $self->execCurlCmd($testCmdBasics, "", $log);
+}
+###############################################################################
+#Copy test config essential for running a sub command
+sub copyTestBasicConfig{
+  my ($self, $testCmd) = @_;
+  my %testCmdBasics;
+  foreach my $key (keys %$testCmd) {
+    if ($key ne 'method'
+        && $key ne 'url'
+        && $key ne 'upload_file'
+        && $key ne 'post_options'
+       ) {
+      $testCmdBasics{$key} = $testCmd->{$key};
+    }
+  }
+  #   $testCmdBasics{'localpath'} = $testCmd->{'localpath'};
+  #   $testCmdBasics{'group'} = $testCmd->{'group'};
+  #   $testCmdBasics{'num'} = $testCmd->{'num'};
+  return \%testCmdBasics;
+}
+###############################################################################
+sub checkResStatusCode{
+  my ($self, $testResult, $e_status_code, $log) = @_;
+  my $subName  = (caller(0))[3];
+
+  #    print STDERR "expected " . $e_status_code . " was " . $testResult->{'status_code'};
+
+  if (!defined $testResult->{'status_code'} || 
+      $testResult->{'status_code'} != $e_status_code) {
+    print $log "$0::$subName INFO Check failed: status_code " .
+      "$e_status_code expected, test returned " .
+        "<$testResult->{'status_code'}>\n";
+    return 0;
+  }
+  return 1;
+
+}
+
+###############################################################################
+# Sub: recordResults
+# Record results of the test run.  The required fields are filled in by the
+# test harness.  This call gives an individual driver a chance to fill in
+# additional fields of cmd, cmd_id, expected_results, and actual_results.
+# this function does not have to be implemened.
+# This is a virtual function.
+#
+# Parameters:
+# status - status of test passing, true or false
+# testResult - reference to hash returned by runTest.
+# benchmarkResult - reference to hash returned by generateBenchmark.
+# dbinfo - reference to hash that will be used to fill in db.
+# log - reference to hash that will be used to fill in db.
+#
+# Returns:
+# None
+#
+sub recordResults
+  {
+  }
+
+###############################################################################
+# Sub: cleanup
+# Clean up after a test.  This gives each individual driver a chance to do
+# cleanup.  A driver need not implement it.  It is a virtual function.
+#
+# Parameters:
+# status - status of test passing, true or false
+# testHash - reference to hash that was passed to runTest() and
+# generateBenchmark().
+# testResult - reference to hash returned by runTest.
+# benchmarkResult - reference to hash returned by generateBenchmark.
+# log - reference to a stream pointer for the logs
+# 
+# Returns:
+# None
+#
+sub cleanup
+  {
+  }
+
+###############################################################################
+# Sub: run
+# Run all the tests in the configuration file.
+#
+# Parameters:
+# testsToRun - reference to array of test groups and ids to run
+# testsToMatch - reference to array of test groups and ids to match.
+# If a test group_num matches any of these regular expressions it will be run.
+# cfg - reference to contents of cfg file
+# log - reference to a stream pointer for the logs
+# dbh - reference database connection
+# testStatuses- reference to hash of test statuses
+# confFile - config file name
+# startat - test to start at.
+# logname - name of the xml log for reporting results
+#
+# Returns:
+# @returns nothing
+# failed.
+#
+sub run
+  {
+    my ($self, $testsToRun, $testsToMatch, $cfg, $log, $dbh, $testStatuses,
+        $confFile, $startat, $logname ) = @_;
+
+    my $subName      = (caller(0))[3];
+    my $msg="";
+    my $duration=0;
+    my $totalDuration=0;
+    my $groupDuration=0;
+    my $sawstart = !(defined $startat);
+    # Rather than make each driver handle our multi-level cfg, we'll flatten
+    # the hashes into one for it.
+    my %globalHash;
+
+    my $runAll = ((scalar(@$testsToRun) == 0) && (scalar(@$testsToMatch) == 0));
+
+    # Read the global keys
+    foreach (keys(%$cfg)) {
+      next if $_ eq 'groups';
+      $globalHash{$_} = $cfg->{$_};
+    }
+
+    $globalHash{$_} = $cfg->{$_};
+    # Do the global setup
+    $self->globalSetup(\%globalHash, $log);
+
+    my $report=0;
+    #        my $properties= new Properties(0, $globalHash{'propertiesFile'});
+
+    my %groupExecuted;
+    foreach my $group (@{$cfg->{'groups'}}) {
+ 
+      print $log "INFO $subName at ".__LINE__.": Running TEST GROUP(".$group->{'name'}.")\n";
+                
+      my %groupHash = %globalHash;
+      $groupHash{'group'} = $group->{'name'};
+
+      # Read the group keys
+      foreach (keys(%$group)) {
+        next if $_ eq 'tests';
+        $groupHash{$_} = $group->{$_};
+      }
+
+
+      # Run each test
+      foreach my $test (@{$group->{'tests'}}) {
+        # Check if we're supposed to run this one or not.
+        if (!$runAll) {
+          # check if we are supposed to run this test or not.
+          my $foundIt = 0;
+          foreach (@$testsToRun) {
+            if (/^$groupHash{'group'}(_[0-9]+)?$/) {
+              if (not defined $1) {
+                # In this case it's just the group name, so we'll
+                # run every test in the group
+                $foundIt = 1;
+                last;
+              } else {
+                # maybe, it at least matches the group
+                my $num = "_" . $test->{'num'};
+                if ($num eq $1) {
+                  $foundIt = 1;
+                  last;
+                }
+              }
+            }
+          }
+          foreach (@$testsToMatch) {
+            my $protoName = $groupHash{'group'} . "_" .  $test->{'num'};
+            if ($protoName =~ /$_/) {
+              if (not defined $1) {
+                # In this case it's just the group name, so we'll
+                # run every test in the group
+                $foundIt = 1;
+                last;
+              } else {
+                # maybe, it at least matches the group
+                my $num = "_" . $test->{'num'};
+                if ($num eq $1) {
+                  $foundIt = 1;
+                  last;
+                }
+              }
+            }
+          }
+
+          next unless $foundIt;
+        }
+
+        # This is a test, so run it.
+        my %testHash = %groupHash;
+        foreach (keys(%$test)) {
+          $testHash{$_} = $test->{$_};
+        }
+
+        my $testName = $testHash{'group'} . "_" . $testHash{'num'};
+
+        #            if ( $groupExecuted{ $group->{'name'} }== 0 ){
+        #                $groupExecuted{ $group->{'name'} }=1;
+        #               
+        #                my $xmlDir= $globalHash{'localxmlpathbase'}."/run".$globalHash->{'UID'};
+        #                mkpath( [ $xmlDir ] , 1, 0777) if ( ! -e $xmlDir );
+        #
+        #                my $filename = $group->{'name'}.".xml";
+        #                $report = new TestReport ( $properties, "$xmlDir/$filename" );
+        #                $report->purge();
+        #            }
+
+        # Check that ignore isn't set for this file, group, or test
+        if (defined $testHash{'ignore'}) {
+          print $log "Ignoring test $testName, ignore message: " .
+            $testHash{'ignore'} . "\n";
+          next;
+        }
+
+        # Have we not reached the starting point yet?
+        if (!$sawstart) {
+          if ($testName eq $startat) {
+            $sawstart = 1;
+          } else {
+            next;
+          }
+        }
+
+        # Check that this test doesn't depend on an earlier test or tests
+        # that failed.  Don't abort if that test wasn't run, just assume the
+        # user knew what they were doing and set it up right.
+        my $skipThisOne = 0;
+        foreach (keys(%testHash)) {
+          if (/^depends_on/ && defined($testStatuses->{$testHash{$_}}) &&
+              $testStatuses->{$testHash{$_}} ne $passedStr) {
+            print $log "Skipping test $testName, it depended on " .
+              "$testHash{$_} which returned a status of " .
+                "$testStatuses->{$testHash{$_}}\n";
+            $testStatuses->{$testName} = $dependStr;
+            $skipThisOne = 1;
+            last;
+          }
+        }
+        if ($skipThisOne) {
+          printResults($testStatuses, $log, "Results so far");
+          next;
+        }
+
+        print $log "\n******************************************************\n";
+        print $log "\nTEST: $confFile::$testName\n";
+        print $log  "******************************************************\n";
+        print $log "Beginning test $testName at " . time . "\n";
+        my %dbinfo = (
+                      'testrun_id' => $testHash{'trid'},
+                      'test_type' => $testHash{'driver'},
+                      #'test_file' => $testHash{'file'},
+                      'test_file' => $confFile,
+                      'test_group' => $testHash{'group'},
+                      'test_num' => $testHash{'num'},
+                     );
+        my $beginTime = time;
+        my $endTime = 0;
+        my ($testResult, $benchmarkResult);
+        eval {
+          $testResult = $self->runTest(\%testHash, $log);
+          $endTime = time;
+          $benchmarkResult = $self->generateBenchmark(\%testHash, $log);
+          my $result =
+            $self->compare($testResult, $benchmarkResult, $log, \%testHash);
+          $msg = "INFO: $subName() at ".__LINE__.":Test $testName";
+
+          if ($result eq $self->{'wrong_execution_mode'}) {
+            $msg .= " SKIPPED";
+            $testStatuses->{$testName} = $skippedStr;
+          } elsif ($result) {
+            $msg .= " SUCCEEDED";
+            $testStatuses->{$testName} = $passedStr;
+
+          } else {
+            $msg .= " FAILED";
+            $testStatuses->{$testName} = $failedStr;
+
+          }
+          $msg= "$msg at " . time . "\n";
+          #print $msg;
+          print $log $msg;
+          $duration = $endTime - $beginTime;
+          $dbinfo{'duration'} = $duration;
+          $self->recordResults($result, $testResult
+                               , $benchmarkResult, \%dbinfo, $log);
+                                  
+        };
+
+
+        if ($@) {
+          $msg= "ERROR $subName at : ".__LINE__." Failed to run test $testName <$@>\n";
+          #print $msg;
+          print $log $msg;
+          $testStatuses->{$testName} = $abortedStr;
+          $dbinfo{'duration'} = $duration;
+        }
+
+
+        eval {
+          $dbinfo{'status'} = $testStatuses->{$testName};
+          if ($dbh) {
+            $dbh->insertTestCase(\%dbinfo);
+          }
+        };
+        if ($@) {
+          chomp $@;
+          warn "Failed to insert test case info, error <$@>\n";
+        }
+
+        $self->cleanup($testStatuses->{$testName}, \%testHash, $testResult,
+                       $benchmarkResult, $log);
+        #$report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName}, $testResult ) if ( $report );
+        $report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName} ) if ( $report );
+        $groupDuration = $groupDuration + $duration;
+        $totalDuration = $totalDuration + $duration;
+        printResults( $testStatuses, $log, "Results so far" );
+      }
+
+      if ( $report ) {
+        $report->systemOut( $logname, $group->{'name'});
+        printGroupResultsXml( $report, $group->{'name'}, $testStatuses, $groupDuration );
+      }
+      $report = 0;
+      $groupDuration=0;
+
+
+    }
+
+    # Do the global cleanup
+    $self->globalCleanup(\%globalHash, $log);
+  }
+
+# TODO These should be removed
+
+sub tmpIPCRun(){
+
+  my $self = shift;
+  my $subName      = (caller(0))[3];
+  my $runningSubName= shift;
+  my $refCmd  = shift;
+  my @cmd     = @$refCmd;
+  my $log    = shift;
+  my $msg    = shift;
+
+  print $log "$0::$subName INFO Running ( @cmd )\n";
+
+  my $result= `@cmd`;
+  if ( $@ ) {
+    my $msg= "$0::$subName FATAL Failed to run from $runningSubName $msg < $@ >\n$result\n";
+    print $log $msg;
+    die "$msg";
+  }
+
+  return $?;
+}
+
+sub tmpIPCRunSplitStdoe {
+
+  my $self = shift;
+  my $subName      = (caller(0))[3];
+  my $runningSubName= shift;
+  my $refCmd  = shift;
+  my @cmd     = @$refCmd;
+  my $dir    = shift;
+  my $log    = shift;
+  my $msg    = shift;
+  my $die    = shift;
+
+
+  my $failed = 0;
+  
+  my $outfilename = $dir."out.tmp";
+  my $errfilename = $dir."err.tmp";
+  print $log "$0::$subName INFO Running from $runningSubName ( @cmd 1>$outfilename 2>$errfilename )\n";
+  #make sure files are writeable
+  open( TMP, ">$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for writing\n";
+  close( TMP );
+  open( TMP, ">$errfilename" ) || die "$0::$subName FATAL: Cannot open $errfilename for writing\n";
+  close( TMP );
+
+  #RUN CMD
+  my $msg;
+  print $log `@cmd 1>$outfilename 2>$errfilename`;
+   
+  my $failed=0;
+  if ( $@ ) { 
+    $msg= "$0::$subName FATAL < $@ >\n"; 
+    $failed++;
+  }
+   
+  #READ FILES
+  my $stdout=""; 
+  my $stderr="";;
+  open( TMP, "$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for reading\n";
+  while ( <TMP> ) {
+    $stdout .= $_;
+  }
+  close( TMP );
+ 
+  open( TMP, "$errfilename" ) || die "$0::$subName FATAL: Cannot open $errfilename for reading\n";
+  while ( <TMP> ) {
+    $stderr .= $_;
+  }
+  close( TMP );
+
+  #DIE IF Test Failed, otherwise return stdout and stderr
+  if ( $failed ) {
+
+    $msg = "$0::$subName FATAL: Faied from $runningSubName \nSTDOUT:" . $stdout . "\nSTDERR:" . $stderr . "\n" if ( $failed );
+    print $log "$msg";
+    die $msg if ( $die != "1" ); #die by defaultast
+    return ( -1, $stdout, $stderr );
+
+  }
+
+  return ( $?, $stdout, $stderr);
+}
+
+sub tmpIPCRunJoinStdoe {
+
+  my $self = shift;
+  my $subName      = (caller(0))[3];
+  my $runningSubName= shift;
+  my $refCmd  = shift;
+  my @cmd     = @$refCmd;
+  my $outfilename= shift;
+  my $log    = shift;
+  my $msg    = shift;
+  my $die    = shift;
+
+  #make sure files are writeable
+  open( TMP, ">$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for writing\n";
+  close( TMP );
+
+  #RUN CMD
+  my $msg;
+  my $failed=0;
+  print $log "$0::$subName INFO Running ( @cmd 2>&1$outfilename 2>/dev/null )\n";
+  print $log `@cmd 2>&1 > $outfilename 2>/dev/null`;
+  if ( $@ ) { 
+    $failed++;
+    $msg= "$0::$subName FATAL < $@ >\n"; 
+  }
+   
+  #READ FILES
+  my $stdoe=""; 
+  open( TMP, "$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for reading\n";
+  while ( <TMP> ) {
+    $stdoe .= $_;
+  }
+  close( TMP );
+
+  if ( $failed ) {
+    print $log "$msg";
+    die $msg if ( $die != "1" ); #die by default
+    return ( -1 ); 
+  }
+  return ( $? );
+}
+
+
+1;

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jaradditionaljars.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jaradditionaljars.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jaradditionaljars.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jaradditionaljars.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1,5 @@
+-- no register command
+l = load '$INPDIR/nums.txt' as (a : chararray);
+f = foreach l generate  org.apache.pig.piggybank.evaluation.string.Reverse($0);
+store f into '$OUTDIR/reversed.txt';
+

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jarregistered.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jarregistered.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jarregistered.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jarregistered.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1,5 @@
+register piggybank.jar
+l = load '$INPDIR/nums.txt' as (a : chararray);
+f = foreach l generate  org.apache.pig.piggybank.evaluation.string.Reverse($0);
+store f into '$OUTDIR/reversed.txt';
+

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/loadstore.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/loadstore.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/loadstore.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/loadstore.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1,2 @@
+l = load '$INPDIR/nums.txt';
+store l into '$OUTDIR/loadstore.out';

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/nums.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/nums.txt?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/nums.txt (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/nums.txt Wed Jul 25 20:29:44 2012
@@ -0,0 +1,2 @@
+1	3
+2	4

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/pythonudf.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/pythonudf.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/pythonudf.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/pythonudf.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1,5 @@
+register 'udfs.py' using jython as myfuncs;
+l = load '$INPDIR/nums.txt' as (a : chararray);
+f = foreach l generate $0, myfuncs.helloworld() ;
+store f into '$OUTDIR/pyhello.txt';
+

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcount_withmacro.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcount_withmacro.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcount_withmacro.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcount_withmacro.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1,5 @@
+import 'rowcountmacro.pig';
+
+l = load '$INPDIR/nums.txt';
+c = row_count(l);
+store c into '$OUTDIR/rowcount.out';

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcountmacro.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcountmacro.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcountmacro.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcountmacro.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1 @@
+DEFINE row_count(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate COUNT($X); };

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/udfs.py
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/udfs.py?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/udfs.py (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/udfs.py Wed Jul 25 20:29:44 2012
@@ -0,0 +1,3 @@
+@outputSchema("word:chararray")
+def helloworld():
+  return 'Hello, World'

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jaradditionaljars.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jaradditionaljars.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jaradditionaljars.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jaradditionaljars.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1,5 @@
+-- no register command
+l = load '$INPDIR/nums.txt' as (a : chararray);
+f = foreach l generate  org.apache.pig.piggybank.evaluation.string.Reverse($0);
+store f into '$OUTDIR/reversed.txt';
+

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jarregistered.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jarregistered.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jarregistered.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jarregistered.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1,5 @@
+register piggybank.jar
+l = load '$INPDIR/nums.txt' as (a : chararray);
+f = foreach l generate  org.apache.pig.piggybank.evaluation.string.Reverse($0);
+store f into '$OUTDIR/reversed.txt';
+

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/loadstore.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/loadstore.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/loadstore.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/loadstore.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1,2 @@
+l = load '$INPDIR/nums.txt';
+store l into '$OUTDIR/loadstore.out';

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pigtest.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pigtest.txt?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pigtest.txt (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pigtest.txt Wed Jul 25 20:29:44 2012
@@ -0,0 +1,49 @@
+checks for all of them -
+- lanucher  launched ok
+- job succeeded (and pig exit status 0)/failed (tests below succeed unless specified)
+
+
+#1
+pig -e 'fs -ls :INPDIR:/'
+- stdout after job finish has list of files including some known one (.*nums.txt.*)
+
+#2
+pig -e 'erroneous syntax'
+- job failed
+- stderr contains syntax error message
+
+#3
+pig -check "l = load 'x';"
+
+#4
+- add loadstore.pig to files to be copied
+pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: loadstore.pig
+
+#5
+- add piggybank.jar to files to be copied
+pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: jarregistered.pig
+
+#6
+#jar specified in cmdline option instead of explicit register in script
+- add piggybank.jar to files to be copied
+pig  -Dpig.additional.jars=piggybank.jar -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR:  jaradditionaljars.pig
+
+
+#7
+- add rowcountmacro.pig to files to be copied
+pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: rowcount_withmacro.pig 
+
+
+#8
+# test error when input file is not present
+pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: no_such_file.pig
+- non zero exit status
+
+#9
+#i haven't been able to get this test working, it complains of not being able to find PyException class
+# would reccomend taking daniel's help to get it working
+- add udfs.py to files to be copied
+pig  -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: pythonudf.pig
+
+
+

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pythonudf.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pythonudf.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pythonudf.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pythonudf.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1,5 @@
+register 'udfs.py' using jython as myfuncs;
+l = load '$INPDIR/nums.txt' as (a : chararray);
+f = foreach l generate $0, myfuncs.helloworld() ;
+store f into '$OUTDIR/pyhello.txt';
+

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcount_withmacro.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcount_withmacro.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcount_withmacro.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcount_withmacro.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1,5 @@
+import 'rowcountmacro.pig';
+
+l = load '$INPDIR/nums.txt';
+c = row_count(l);
+store c into '$OUTDIR/rowcount.out';

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcountmacro.pig
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcountmacro.pig?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcountmacro.pig (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcountmacro.pig Wed Jul 25 20:29:44 2012
@@ -0,0 +1 @@
+DEFINE row_count(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate COUNT($X); };

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/udfs.py
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/udfs.py?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/udfs.py (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/udfs.py Wed Jul 25 20:29:44 2012
@@ -0,0 +1,3 @@
+@outputSchema("word:chararray")
+def helloworld():
+  return 'Hello, World'



Mime
View raw message