From hcatalog-commits-return-972-apmail-incubator-hcatalog-commits-archive=incubator.apache.org@incubator.apache.org Wed Jul 25 20:30:45 2012 Return-Path: X-Original-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 181FBD33B for ; Wed, 25 Jul 2012 20:30:45 +0000 (UTC) Received: (qmail 99600 invoked by uid 500); 25 Jul 2012 20:30:45 -0000 Delivered-To: apmail-incubator-hcatalog-commits-archive@incubator.apache.org Received: (qmail 99522 invoked by uid 500); 25 Jul 2012 20:30:44 -0000 Mailing-List: contact hcatalog-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hcatalog-dev@incubator.apache.org Delivered-To: mailing list hcatalog-commits@incubator.apache.org Received: (qmail 99508 invoked by uid 99); 25 Jul 2012 20:30:44 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Jul 2012 20:30:44 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Jul 2012 20:30:39 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 62E0623889E7; Wed, 25 Jul 2012 20:29:56 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1365722 [4/11] - in /incubator/hcatalog/trunk: ./ ant/ conf/ hcatalog-pig-adapter/ ivy/ src/docs/src/documentation/content/xdocs/ src/docs/src/documentation/content/xdocs/images/ src/java/org/apache/hcatalog/mapreduce/ src/test/e2e/templet... Date: Wed, 25 Jul 2012 20:29:49 -0000 To: hcatalog-commits@incubator.apache.org From: gates@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120725202956.62E0623889E7@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Added: incubator/hcatalog/trunk/src/test/e2e/templeton/drivers/TestDriverCurl.pm URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/drivers/TestDriverCurl.pm (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/drivers/TestDriverCurl.pm Wed Jul 25 20:29:44 2012 @@ -0,0 +1,1325 @@ +############################################################################ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package TestDriverCurl; + +########################################################################### +# Class: TestDriver +# A base class for TestDrivers. +# + +use TestDriverFactory; +use TestReport; +use File::Path; +use IPC::Run; +use Data::Dump qw(dump); +use JSON; +use HTTP::Daemon; +use HTTP::Status; +use Data::Compare; +use strict; +use English; +use Storable qw(dclone); +use File::Glob ':glob'; + +my $passedStr = 'passed'; +my $failedStr = 'failed'; +my $abortedStr = 'aborted'; +my $skippedStr = 'skipped'; +my $dependStr = 'failed_dependency'; + + +############################################################################## +# Sub: printResults +# Static function, can be used by test_harness.pl +# Print the results so far, given the testStatuses hash. +# +# Paramaters: +# testStatuses - reference to hash of test status results. +# log - reference to file handle to print results to. +# prefix - A title to prefix to the results +# +# Returns: +# None. +# +sub printResults + { + my ($testStatuses, $log, $prefix) = @_; + + my ($pass, $fail, $abort, $depend, $skipped) = (0, 0, 0, 0, 0); + + foreach (keys(%$testStatuses)) { + ($testStatuses->{$_} eq $passedStr) && $pass++; + ($testStatuses->{$_} eq $failedStr) && $fail++; + ($testStatuses->{$_} eq $abortedStr) && $abort++; + ($testStatuses->{$_} eq $dependStr) && $depend++; + ($testStatuses->{$_} eq $skippedStr) && $skipped++; + } + + my $msg = "$prefix, PASSED: $pass FAILED: $fail SKIPPED: $skipped ABORTED: $abort " + . "FAILED DEPENDENCY: $depend"; + print $log "$msg\n"; + print "$msg\n"; + + } + +############################################################################## +# Sub: printGroupResultsXml +# Print the results for the group using junit xml schema using values from the testStatuses hash. +# +# Paramaters: +# $report - the report object to use to generate the report +# $groupName - the name of the group to report totals for +# $testStatuses - the hash containing the results for the tests run so far +# $totalDuration- The total time it took to run the group of tests +# +# Returns: +# None. +# +sub printGroupResultsXml + { + my ( $report, $groupName, $testStatuses, $totalDuration) = @_; + $totalDuration=0 if ( !$totalDuration ); + + my ($pass, $fail, $abort, $depend) = (0, 0, 0, 0); + + foreach my $key (keys(%$testStatuses)) { + if ( $key =~ /^$groupName/ ) { + ($testStatuses->{$key} eq $passedStr) && $pass++; + ($testStatuses->{$key} eq $failedStr) && $fail++; + ($testStatuses->{$key} eq $abortedStr) && $abort++; + ($testStatuses->{$key} eq $dependStr) && $depend++; + } + } + + my $total= $pass + $fail + $abort; + $report->totals( $groupName, $total, $fail, $abort, $totalDuration ); + + } + +############################################################################## +# Sub: new +# Constructor, should only be called by TestDriverFactory. +# +# Paramaters: +# None +# +# Returns: +# None. +sub new + { + my $proto = shift; + my $class = ref($proto) || $proto; + my $self = {}; + + bless($self, $class); + + $self->{'wrong_execution_mode'} = "_xyz_wrong_execution_mode_zyx_"; + + return $self; + } + +############################################################################## +# Sub: globalSetup +# Set up before any tests are run. This gives each individual driver a chance to do +# setup. This function will only be called once, before all the tests are +# run. A driver need not implement it. It is a virtual function. +# +# Paramaters: +# globalHash - Top level hash from config file (does not have any group +# or test information in it). +# log - log file handle +# +# Returns: +# None +# +sub globalSetup + { + my ($self, $globalHash, $log) = @_; + my $subName = (caller(0))[3]; + + + # Setup the output path + my $me = `whoami`; + chomp $me; + $globalHash->{'runid'} = $me . "." . time; + + # if "-ignore false" was provided on the command line, + # it means do run tests even when marked as 'ignore' + if (defined($globalHash->{'ignore'}) && $globalHash->{'ignore'} eq 'false') { + $self->{'ignore'} = 'false'; + } + + if (! defined $globalHash->{'localpathbase'}) { + $globalHash->{'localpathbase'} = '/tmp'; + } + + $globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/"; + $globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/"; + $globalHash->{'webhdfs_url'} = $ENV{'WEBHDFS_URL'}; + $globalHash->{'templeton_url'} = $ENV{'TEMPLETON_URL'}; + $globalHash->{'current_user'} = $ENV{'USER_NAME'}; + $globalHash->{'current_group_user'} = $ENV{'GROUP_USER_NAME'}; + $globalHash->{'current_other_user'} = $ENV{'OTHER_USER_NAME'}; + $globalHash->{'current_group'} = $ENV{'GROUP_NAME'}; + $globalHash->{'keytab_dir'} = $ENV{'KEYTAB_DIR'}; + + + + $globalHash->{'inpdir_local'} = $ENV{'TH_INPDIR_LOCAL'}; + $globalHash->{'inpdir_hdfs'} = $ENV{'TH_INPDIR_HDFS'}; + + $globalHash->{'is_secure_mode'} = $ENV{'SECURE_MODE'}; + + # add libexec location to the path + if (defined($ENV{'PATH'})) { + $ENV{'PATH'} = $globalHash->{'scriptPath'} . ":" . $ENV{'PATH'}; + } else { + $ENV{'PATH'} = $globalHash->{'scriptPath'}; + } + + IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or + die "Cannot create localpath directory " . $globalHash->{'localpath'} . + " " . "$ERRNO\n"; + + # Create the temporary directory + IPC::Run::run(['mkdir', '-p', $globalHash->{'tmpPath'}], \undef, $log, $log) or + die "Cannot create temporary directory " . $globalHash->{'tmpPath'} . + " " . "$ERRNO\n"; + + } + +############################################################################### +# Sub: globalCleanup +# Clean up after all tests have run. This gives each individual driver a chance to do +# cleanup. This function will only be called once, after all the tests are +# run. A driver need not implement it. It is a virtual function. +# +# Paramaters: +# globalHash - Top level hash from config file (does not have any group +# or test information in it). +# log - log file handle +# +# Returns: +# None +sub globalCleanup + { + my ($self, $globalHash, $log) = @_; + + IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or + warn "Cannot remove temporary directory " . $globalHash->{'tmpPath'} . + " " . "$ERRNO\n"; + + } + +############################################################################### +# Sub: runTest +# Run a test. This is a pure virtual function. +# +# Parameters: +# testcmd - reference to hash with meta tags and command to run tests. +# Interpretation of the tags and the command are up to the subclass. +# log - reference to a stream pointer for the logs +# +# Returns: +# @returns reference to hash. Contents of hash are defined by the subclass. +# +sub runTest + { + my ($self, $testCmd, $log) = @_; + my $subName = (caller(0))[3]; + + # Handle the various methods of running used in + # the original TestDrivers + + if ( $testCmd->{'url'} ) { + return $self->runCurlCmd( $testCmd, $log); + } else { + die "$subName FATAL Did not find a testCmd that " . + "I know how to handle : " . $testCmd->{'Curl'}; + } + + + } + +############################################################################### + +sub replaceParameters + { + my ($self, $testCmd, $aPfix, $log) = @_; + + my $url = $testCmd->{$aPfix . 'url'}; + $url =~ s/:WEBHDFS_URL:/$testCmd->{'webhdfs_url'}/g; + $url =~ s/:TEMPLETON_URL:/$testCmd->{'templeton_url'}/g; + $url = $self->replaceParametersInArg($url, $testCmd, $log); + $testCmd->{$aPfix . 'url'} = $url; + + $testCmd->{$aPfix . 'upload_file'} = + $self->replaceParametersInArg($testCmd->{$aPfix . 'upload_file'}, $testCmd, $log); + + $testCmd->{$aPfix . 'user_name'} = + $self->replaceParametersInArg($testCmd->{$aPfix . 'user_name'}, $testCmd, $log); + + if (defined $testCmd->{$aPfix . 'post_options'}) { + my @options = @{$testCmd->{$aPfix . 'post_options'}}; + my @new_options = (); + foreach my $option (@options) { + $option = $self->replaceParametersInArg($option, $testCmd, $log); + push @new_options, ($option); + } + $testCmd->{$aPfix . 'post_options'} = \@new_options; + } + if (defined $testCmd->{$aPfix . 'json_field_substr_match'}) { + my $json_matches = $testCmd->{$aPfix . 'json_field_substr_match'}; + my @keys = keys %{$json_matches}; + + foreach my $key (@keys) { + my $new_value = $self->replaceParametersInArg($json_matches->{$key}, $testCmd, $log); + $json_matches->{$key} = $new_value; + } + } + + + } + +############################################################################### +sub replaceParametersInArg + { + my ($self, $arg, $testCmd, $log) = @_; + if(! defined $arg){ + return $arg; + } + my $outdir = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'}; + $arg =~ s/:UNAME:/$testCmd->{'current_user'}/g; + $arg =~ s/:UNAME_GROUP:/$testCmd->{'current_group_user'}/g; + $arg =~ s/:UNAME_OTHER:/$testCmd->{'current_other_user'}/g; + $arg =~ s/:UGROUP:/$testCmd->{'current_group'}/g; + $arg =~ s/:OUTDIR:/$outdir/g; + $arg =~ s/:INPDIR_HDFS:/$testCmd->{'inpdir_hdfs'}/g; + $arg =~ s/:INPDIR_LOCAL:/$testCmd->{'inpdir_local'}/g; + $arg =~ s/:TNUM:/$testCmd->{'num'}/g; + return $arg; + } + + + +############################################################################### + +sub getBaseCurlCmd(){ + my ($self) = @_; + my @curl_cmd = ("curl", '--silent','--show-error', '-H','Expect:'); + if (defined $ENV{'SOCKS_PROXY'}) { + push @curl_cmd, ('--socks5-hostname', $ENV{'SOCKS_PROXY'}); + } + return @curl_cmd; + +} + +############################################################################### +sub runCurlCmd(){ + my ($self, $testCmd, $log) = @_; + if (defined $testCmd->{'upload_file'}) { + return $self->upload_file($testCmd,$log); + } else { + #if there are setup steps, run them first + if (defined $testCmd->{'setup'}) { + my $i = 0; + foreach my $setupCmd (@{$testCmd->{'setup'}}){ + $i++; + print $log "\nRUNNING SETUP COMMAND: $i\n"; + my $pfix = "setup_${i}_"; + my $setupTestCmd = $self->createSetupCmd($testCmd, $setupCmd, $pfix, $log); + my $setupResult = $self->execCurlCmd($setupTestCmd, $pfix, $log); + + #if status code is set in setup, check if it matches results + if(defined $setupTestCmd->{"${pfix}status_code"}){ + $self->checkResStatusCode($setupResult, $setupTestCmd->{"${pfix}status_code"}, $log); + } + } + } + return $self->execCurlCmd($testCmd, "", $log); + } +} +############################################################################### +sub createSetupCmd(){ + my ($self, $testCmd, $setupCmd, $pfix, $log) = @_; + my $newTestCmd = dclone ($testCmd); + for my $key (keys %$setupCmd){ + $newTestCmd->{$pfix . $key} = $setupCmd->{$key}; + } + return $newTestCmd; +} + +############################################################################### +sub upload_file(){ + my ($self, $testCmd, $log) = @_; + $testCmd->{'method'} = 'PUT'; + my $result = $self->execCurlCmd($testCmd, "", $log); + my $checkRes = $self->checkResStatusCode($result, 100, $log); + if ($checkRes == 0) { + #fail + return 0; + } + my $header = $result->{'header_fields'}; + + #final url where the file should be stored + my $location = $header->{'Location'}; + $testCmd->{'url'} = $location; + + $result = $self->execCurlCmd($testCmd, "", $log); + return $result; +} + +############################################################################### +sub execCurlCmd(){ + my ($self, $testCmd, $argPrefix, $log) = @_; + my @curl_cmd = $self->getBaseCurlCmd(); + # Set up file locations + my $subName = (caller(0))[3]; + + my $filePrefix = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $argPrefix . $testCmd->{'num'}; + my $cmd_body = $filePrefix . ".cmd_body"; + + #results + my $res_header = $filePrefix . ".res_header"; + my $res_body = $filePrefix . ".res_body"; + + my $outdir = $filePrefix . ".out"; + my $stdoutfile = "$outdir/stdout"; + my $stderrfile = "$outdir/stderr"; + + mkpath( [ $outdir ] , 0, 0755) if ( ! -e outdir ); + if ( ! -e $outdir ) { + print $log "$0.$subName FATAL could not mkdir $outdir\n"; + die "$0.$subName FATAL could not mkdir $outdir\n"; + } + + $self->replaceParameters($testCmd, $argPrefix, $log ); + + my $method = $testCmd->{ $argPrefix . 'method'}; + + my $url = $testCmd->{ $argPrefix . 'url'}; + + my @options = (); + if (defined $testCmd->{$argPrefix . 'post_options'}) { + @options = @{$testCmd->{$argPrefix . 'post_options'}}; + } + + #handle authentication based on secure mode + my $user_name = $testCmd->{ $argPrefix . 'user_name' }; + if (defined $testCmd->{'is_secure_mode'} && $testCmd->{'is_secure_mode'} =~ /y.*/i) { + push @curl_cmd, ('--negotiate', '-u', ':'); + + #if keytab dir is defined, look for a keytab file for user and do a kinit + if(defined $testCmd->{'keytab_dir'} && defined $user_name){ + $user_name =~ /(.*?)(\/|$)/; + my $just_uname = $1; #uname without principal + my $keytab_dir = $testCmd->{'keytab_dir'}; + print $log "regex " . "${keytab_dir}/*${just_uname}\.*keytab"; + my @files = bsd_glob( "${keytab_dir}/*${just_uname}\.*keytab" ); + if(scalar @files == 0){ + die "Could not find keytab file for user $user_name in $keytab_dir"; + } elsif(scalar @files > 1){ + die "More than one keytab file found for user $user_name in $keytab_dir"; + } + my @cmd = ('kinit', '-k', '-t', $files[0], $user_name); + print $log "Command @cmd"; + IPC::Run::run(\@cmd, \undef, $log, $log) or + die "Could not kinit as $user_name using " . $files[0] . " $ERRNO"; + } + + } else { + #if mode is unsecure + if (defined $user_name) { + my $user_param = "user.name=${user_name}"; + if ($method eq 'POST' ) { + push @options, $user_param; + } else { + if ($url =~ /\?/) { + #has some parameters in url + $url = $url . '&' . $user_param; + } else { + $url = $url . '?' . $user_param; + } + } + } + + } + + if (defined $testCmd->{'format_header'}) { + push @curl_cmd, ('-H', $testCmd->{'format_header'}); + } + + + + + if (defined $testCmd->{$argPrefix . 'format_header'}) { + push @curl_cmd, ('-H', $testCmd->{$argPrefix . 'format_header'}); + } + + if (defined $testCmd->{$argPrefix . 'upload_file'}) { + push @curl_cmd, ('-T', $testCmd->{$argPrefix . 'upload_file'}); + } + + # if(!defined $testCmd->{'post_options'}){ + # $testCmd->{'post_options'} = \(); + # } + + if (defined $testCmd->{$argPrefix . 'check_call_back'}) { + my $d = HTTP::Daemon->new || die; + $testCmd->{'http_daemon'} = $d; + $testCmd->{'callback_url'} = $d->url . 'templeton/$jobId'; + push @curl_cmd, ('-d', 'callback=' . $testCmd->{'callback_url'}); + # push ${testCmd->{'post_options'}}, ('callback=' . $testCmd->{'callback_url'}); + # #my @options = @{$testCmd->{'post_options'}}; + # print $log "post options @options\n"; + } + + foreach my $option (@options) { + push @curl_cmd, ('-d', $option); + } + + push @curl_cmd, ("-X", $method, "-o", $res_body, "-D", $res_header); + push @curl_cmd, ($url); + + print $log "$0:$subName Going to run command : " . join (' ', @curl_cmd); + print $log "\n"; + + + my %result; + my $out; + my $err; + IPC::Run::run(\@curl_cmd, \undef, $out, $err) + or die "Failed running curl cmd " . join ' ', @curl_cmd; + + $result{'rc'} = $? >> 8; + $result{'stderr'} = $err; + $result{'stdout'} = $out; + $result{'body'} = `cat $res_body`; + + my @full_header = `cat $res_header`; + $result{'full_header'} = join '\n', @full_header; + + #find the final http status code + for my $line ( @full_header){ + if($line =~ /.*(HTTP\/1.1)\s+(\S+)/){ + $result{'status_code'} = $2; + } + } + + my %header_field; + foreach my $line (@full_header) { + chomp $line; + $line =~ /(.*?)\s*:\s*(.*)/; + if (defined $1 && defined $2 ) { + $header_field{$1} = $2; + } + } + $result{'header_fields'} = \%header_field; + + print $log "result : " . dump(%result); + #dump(%result); + + return \%result; + +} + + +############################################################################### +# Sub: generateBenchmark +# Generate benchmark results. This is a pure virtual function. +# +# Parameters: +# benchmarkcmd - reference to hash with meta tags and command to +# generate benchmark. Interpretation of the tags and the command are up to +# log - reference to a stream pointer for the logs +# the subclass. +# +# Returns: +# @returns reference to hash. Contents of hash are defined by the subclass. +# +sub generateBenchmark + { + my %result; + return \%result; + } + +############################################################################### +# Sub: compare +# Compare the results of the test run with the generated benchmark results. +# This is a pure virtual function. +# +# Parameters: +# benchmarkcmd - reference to hash with meta tags and command to +# testResult - reference to hash returned by runTest. +# benchmarkResult - reference to hash returned by generateBenchmark. +# log - reference to a stream pointer for the logs +# testHash - reference to hash with meta tags and commands +# +# Returns: +# @returns reference true if results are the same, false otherwise. "the +# same" is defined by the subclass. +# +sub compare + { + + my ($self, $testResult, $benchmarkResult, $log, $testCmd) = @_; + my $subName = (caller(0))[3]; + + my $result = 1; # until proven wrong... + if (defined $testCmd->{'status_code'}) { + my $res = $self->checkResStatusCode($testResult, $testCmd->{'status_code'}, $log); + if ($res == 0) { + $result = 0; + } + } + + my $json_hash; + my %json_info; + if (defined $testCmd->{'json_field_substr_match'} || $testCmd->{'json_field_match_object'}) { + my $json = new JSON; + $json_hash = $json->utf8->decode($testResult->{'body'}); + my $json_matches = $testCmd->{'json_field_substr_match'}; + my $json_matches_object = $testCmd->{'json_field_match_object'}; + + %json_info = %$json_hash; + if (defined $json_info{'info'}) { + %json_info = %{$json_info{'info'}}; + + } + print $log "\n\n json_info"; + print $log dump(%json_info); + print $log "\n\n"; + + if (defined $json_hash->{'id'}) { + print STDERR "jobid " . $json_hash->{'id'} . "\n"; + $json_info{'id'} = $json_hash->{'id'}; + } + + if(defined $json_matches->{'location_perms'} || defined $json_matches->{'location_group'}){ + $self->setLocationPermGroup(\%json_info, $testCmd, $log); + } + + foreach my $key (keys %$json_matches) { + my $json_field_val = $json_info{$key}; + if( (ref($json_field_val) && ! UNIVERSAL::isa($json_field_val,'SCALAR')) || + (!ref($json_field_val) && ! UNIVERSAL::isa(\$json_field_val,'SCALAR')) ){ + #flatten the object into a string + $json_field_val = dump($json_field_val); + } + my $regex_expected_value = $json_matches->{$key}; + print $log "Comparing $key: $json_field_val with regex /$regex_expected_value/\n"; + + if ($json_field_val !~ /$regex_expected_value/s) { + print $log "$0::$subName INFO check failed:" + . " json pattern check failed. For field " + . "$key, regex <" . $regex_expected_value + . "> did not match the result <" . $json_field_val + . ">\n"; + $result = 0; + } + } + + foreach my $key (keys %$json_matches_object) { + my $json_field_val = $json_info{$key}; + my $regex_expected_obj = $json->utf8->decode($json_matches_object->{$key}); + print $log "Comparing $key: " . dump($json_field_val) . ",expected value: " . dump($regex_expected_obj); + + if (!Compare($json_field_val, $regex_expected_obj)) { + print $log "$0::$subName INFO check failed:" + . " json compare failed. For field " + . "$key, regex <" . dump($regex_expected_obj) + . "> did not match the result <" . dump($json_field_val) + . ">\n"; + $result = 0; + } + } + + + } + + #kill it if there is a request to kill + if($testCmd->{'kill_job_timeout'}){ + sleep $testCmd->{'kill_job_timeout'}; + my $jobid = $json_hash->{'id'}; + if (!defined $jobid) { + print $log "$0::$subName INFO check failed: " + . "no jobid (id field)found in result"; + $result = 0; + } else { + $self->killJob($testCmd, $jobid, $log); + } + } + + + #try to get the call back url request until timeout + if ($result == 1 && defined $testCmd->{'check_call_back'}) { + my $d = $testCmd->{'http_daemon'}; + $d->timeout(300); #wait for 5 mins + my $url_requested; + $testCmd->{'callback_url'} =~ s/\$jobId/$json_hash->{'id'}/g; + print $log "Expanded callback url : <" . $testCmd->{'callback_url'} . ">\n"; + do{ + print $log "Waiting for call back url request\n"; + if (my $c = $d->accept) { + if (my $r = $c->get_request) { + my $durl = $d->url; + chop $durl; + $url_requested = $durl . $r->uri->path ; + print $log "Got request at url <" . $url_requested . ">\n"; + $c->send_status_line(200); + $c->close; + } + undef($c); + } else { + print $log "Timeout on wait on call back url" . "\n"; + $result = 0; + } + }while (defined $url_requested && $url_requested ne $testCmd->{'callback_url'}); + $d->close; + if (!defined $url_requested || $url_requested ne $testCmd->{'callback_url'}) { + print $log "failed to recieve request on call back url"; + $result = 0; + } + + } + + + if (defined $testCmd->{'check_job_created'} || defined $testCmd->{'check_job_complete'}) { + my $jobid = $json_hash->{'id'}; + if (!defined $jobid) { + print $log "$0::$subName INFO check failed: " + . "no jobid (id field)found in result"; + $result = 0; + } else { + my $jobResult = $self->getJobResult($testCmd, $jobid, $log); + my $json = new JSON; + my $res_hash = $json->utf8->decode($jobResult->{'body'}); + if (! defined $res_hash->{'status'}) { + print $log "$0::$subName INFO check failed: " + . "jobresult not defined "; + $result = 0; + } + if ($testCmd->{'check_job_complete'}) { + my $jobComplete; + my $NUM_RETRIES = 60; + my $SLEEP_BETWEEN_RETRIES = 5; + + #first wait for job completion + while ($NUM_RETRIES-- > 0) { + $jobComplete = $res_hash->{'status'}->{'jobComplete'}; + if (defined $jobComplete && lc($jobComplete) eq "true") { + last; + } + sleep $SLEEP_BETWEEN_RETRIES; + $jobResult = $self->getJobResult($testCmd, $jobid, $log); + $json = new JSON; + $res_hash = $json->utf8->decode($jobResult->{'body'}); + } + if ( (!defined $jobComplete) || lc($jobComplete) ne "true") { + print $log "$0::$subName INFO check failed: " + . " timeout on wait for job completion "; + $result = 0; + } else { + # job has completed, check the runState value + my $runState = $res_hash->{'status'}->{'runState'}; + my $runStateVal = $self->getRunStateNum($testCmd->{'check_job_complete'}); + if ( (!defined $runState) || $runState ne $runStateVal) { + print $log "check_job_complete failed. got runState $runState, expected $runStateVal"; + $result = 0; + } + } + } + } + } + return $result; + } + +############################################################################### +sub setLocationPermGroup{ + my ($self, $job_info, $testCmd, $log) = @_; + my $location = $job_info->{'location'}; + $location =~ /hdfs.*:\d+(\/.*)\/(.*)/; + my $dir = $1; + my $file = $2; + + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'GET'; + $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_checkFile"; + $testCmdBasics->{'url'} = ':WEBHDFS_URL:/webhdfs/v1' + . $dir . '?op=LISTSTATUS'; + + + my $result = $self->execCurlCmd($testCmdBasics, "", $log); + + my $json = new JSON; + my $json_hash = $json->utf8->decode($result->{'body'}); + my @filestatuses = @{$json_hash->{'FileStatuses'}->{'FileStatus'}}; + foreach my $filestatus (@filestatuses){ + if($filestatus->{'pathSuffix'} eq $file){ + $job_info->{'location_perms'} = numPermToStringPerm($filestatus->{'permission'}); + $job_info->{'location_group'} = $filestatus->{'group'}; + $job_info->{'location_owner'} = $filestatus->{'owner'}; + last; + } + + } + +} + +############################################################################### +#convert decimal string to binary string +sub dec2bin { + my $decimal = shift; + my $binary = unpack("B32", pack("N", $decimal)); + $binary =~ s/^0+(?=\d)//; # remove leading zeros + return $binary; +} + +############################################################################### +#convert single digit of the numeric permission format to string format +sub digitPermToStringPerm{ + my $numPerm = shift; + my $binaryPerm = dec2bin($numPerm); + my $stringPerm = ""; + if($binaryPerm =~ /1\d\d$/){ + $stringPerm .= "r"; + }else{ + $stringPerm .= "-"; + } + + if($binaryPerm =~ /\d1\d$/){ + $stringPerm .= "w"; + }else{ + $stringPerm .= "-"; + } + + if($binaryPerm =~ /\d\d1$/){ + $stringPerm .= "x"; + }else{ + $stringPerm .= "-"; + } + +} + +############################################################################### +# convert numeric permission format to string format +sub numPermToStringPerm{ + my $numPerm = shift; + $numPerm =~ /(\d)(\d)(\d)$/; + return digitPermToStringPerm($1) + . digitPermToStringPerm($2) + . digitPermToStringPerm($3); + +} + +############################################################################### +sub getRunStateNum{ + my ($self, $job_complete_state) = @_; + if (lc($job_complete_state) eq 'success') { + return 2; + } elsif (lc($job_complete_state) eq 'failure') { + return 3; + } elsif (lc($job_complete_state) eq 'killed') { + return 5; + } + +} + + +############################################################################### +sub getJobResult{ + my ($self, $testCmd, $jobid, $log) = @_; + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'GET'; + $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_jobStatusCheck"; + $testCmdBasics->{'url'} = ':TEMPLETON_URL:/templeton/v1/queue/' + . $jobid . '?' . "user.name=:UNAME:" ; + return $self->execCurlCmd($testCmdBasics, "", $log); +} +############################################################################### +sub killJob{ + my ($self, $testCmd, $jobid, $log) = @_; + my $testCmdBasics = $self->copyTestBasicConfig($testCmd); + $testCmdBasics->{'method'} = 'DELETE'; + $testCmdBasics->{'num'} = $testCmdBasics->{'num'} . "_killJob"; + $testCmdBasics->{'url'} = ':TEMPLETON_URL:/templeton/v1/queue/' + . $jobid . '?' . "user.name=:UNAME:" ; + return $self->execCurlCmd($testCmdBasics, "", $log); +} +############################################################################### +#Copy test config essential for running a sub command +sub copyTestBasicConfig{ + my ($self, $testCmd) = @_; + my %testCmdBasics; + foreach my $key (keys %$testCmd) { + if ($key ne 'method' + && $key ne 'url' + && $key ne 'upload_file' + && $key ne 'post_options' + ) { + $testCmdBasics{$key} = $testCmd->{$key}; + } + } + # $testCmdBasics{'localpath'} = $testCmd->{'localpath'}; + # $testCmdBasics{'group'} = $testCmd->{'group'}; + # $testCmdBasics{'num'} = $testCmd->{'num'}; + return \%testCmdBasics; +} +############################################################################### +sub checkResStatusCode{ + my ($self, $testResult, $e_status_code, $log) = @_; + my $subName = (caller(0))[3]; + + # print STDERR "expected " . $e_status_code . " was " . $testResult->{'status_code'}; + + if (!defined $testResult->{'status_code'} || + $testResult->{'status_code'} != $e_status_code) { + print $log "$0::$subName INFO Check failed: status_code " . + "$e_status_code expected, test returned " . + "<$testResult->{'status_code'}>\n"; + return 0; + } + return 1; + +} + +############################################################################### +# Sub: recordResults +# Record results of the test run. The required fields are filled in by the +# test harness. This call gives an individual driver a chance to fill in +# additional fields of cmd, cmd_id, expected_results, and actual_results. +# this function does not have to be implemened. +# This is a virtual function. +# +# Parameters: +# status - status of test passing, true or false +# testResult - reference to hash returned by runTest. +# benchmarkResult - reference to hash returned by generateBenchmark. +# dbinfo - reference to hash that will be used to fill in db. +# log - reference to hash that will be used to fill in db. +# +# Returns: +# None +# +sub recordResults + { + } + +############################################################################### +# Sub: cleanup +# Clean up after a test. This gives each individual driver a chance to do +# cleanup. A driver need not implement it. It is a virtual function. +# +# Parameters: +# status - status of test passing, true or false +# testHash - reference to hash that was passed to runTest() and +# generateBenchmark(). +# testResult - reference to hash returned by runTest. +# benchmarkResult - reference to hash returned by generateBenchmark. +# log - reference to a stream pointer for the logs +# +# Returns: +# None +# +sub cleanup + { + } + +############################################################################### +# Sub: run +# Run all the tests in the configuration file. +# +# Parameters: +# testsToRun - reference to array of test groups and ids to run +# testsToMatch - reference to array of test groups and ids to match. +# If a test group_num matches any of these regular expressions it will be run. +# cfg - reference to contents of cfg file +# log - reference to a stream pointer for the logs +# dbh - reference database connection +# testStatuses- reference to hash of test statuses +# confFile - config file name +# startat - test to start at. +# logname - name of the xml log for reporting results +# +# Returns: +# @returns nothing +# failed. +# +sub run + { + my ($self, $testsToRun, $testsToMatch, $cfg, $log, $dbh, $testStatuses, + $confFile, $startat, $logname ) = @_; + + my $subName = (caller(0))[3]; + my $msg=""; + my $duration=0; + my $totalDuration=0; + my $groupDuration=0; + my $sawstart = !(defined $startat); + # Rather than make each driver handle our multi-level cfg, we'll flatten + # the hashes into one for it. + my %globalHash; + + my $runAll = ((scalar(@$testsToRun) == 0) && (scalar(@$testsToMatch) == 0)); + + # Read the global keys + foreach (keys(%$cfg)) { + next if $_ eq 'groups'; + $globalHash{$_} = $cfg->{$_}; + } + + $globalHash{$_} = $cfg->{$_}; + # Do the global setup + $self->globalSetup(\%globalHash, $log); + + my $report=0; + # my $properties= new Properties(0, $globalHash{'propertiesFile'}); + + my %groupExecuted; + foreach my $group (@{$cfg->{'groups'}}) { + + print $log "INFO $subName at ".__LINE__.": Running TEST GROUP(".$group->{'name'}.")\n"; + + my %groupHash = %globalHash; + $groupHash{'group'} = $group->{'name'}; + + # Read the group keys + foreach (keys(%$group)) { + next if $_ eq 'tests'; + $groupHash{$_} = $group->{$_}; + } + + + # Run each test + foreach my $test (@{$group->{'tests'}}) { + # Check if we're supposed to run this one or not. + if (!$runAll) { + # check if we are supposed to run this test or not. + my $foundIt = 0; + foreach (@$testsToRun) { + if (/^$groupHash{'group'}(_[0-9]+)?$/) { + if (not defined $1) { + # In this case it's just the group name, so we'll + # run every test in the group + $foundIt = 1; + last; + } else { + # maybe, it at least matches the group + my $num = "_" . $test->{'num'}; + if ($num eq $1) { + $foundIt = 1; + last; + } + } + } + } + foreach (@$testsToMatch) { + my $protoName = $groupHash{'group'} . "_" . $test->{'num'}; + if ($protoName =~ /$_/) { + if (not defined $1) { + # In this case it's just the group name, so we'll + # run every test in the group + $foundIt = 1; + last; + } else { + # maybe, it at least matches the group + my $num = "_" . $test->{'num'}; + if ($num eq $1) { + $foundIt = 1; + last; + } + } + } + } + + next unless $foundIt; + } + + # This is a test, so run it. + my %testHash = %groupHash; + foreach (keys(%$test)) { + $testHash{$_} = $test->{$_}; + } + + my $testName = $testHash{'group'} . "_" . $testHash{'num'}; + + # if ( $groupExecuted{ $group->{'name'} }== 0 ){ + # $groupExecuted{ $group->{'name'} }=1; + # + # my $xmlDir= $globalHash{'localxmlpathbase'}."/run".$globalHash->{'UID'}; + # mkpath( [ $xmlDir ] , 1, 0777) if ( ! -e $xmlDir ); + # + # my $filename = $group->{'name'}.".xml"; + # $report = new TestReport ( $properties, "$xmlDir/$filename" ); + # $report->purge(); + # } + + # Check that ignore isn't set for this file, group, or test + if (defined $testHash{'ignore'}) { + print $log "Ignoring test $testName, ignore message: " . + $testHash{'ignore'} . "\n"; + next; + } + + # Have we not reached the starting point yet? + if (!$sawstart) { + if ($testName eq $startat) { + $sawstart = 1; + } else { + next; + } + } + + # Check that this test doesn't depend on an earlier test or tests + # that failed. Don't abort if that test wasn't run, just assume the + # user knew what they were doing and set it up right. + my $skipThisOne = 0; + foreach (keys(%testHash)) { + if (/^depends_on/ && defined($testStatuses->{$testHash{$_}}) && + $testStatuses->{$testHash{$_}} ne $passedStr) { + print $log "Skipping test $testName, it depended on " . + "$testHash{$_} which returned a status of " . + "$testStatuses->{$testHash{$_}}\n"; + $testStatuses->{$testName} = $dependStr; + $skipThisOne = 1; + last; + } + } + if ($skipThisOne) { + printResults($testStatuses, $log, "Results so far"); + next; + } + + print $log "\n******************************************************\n"; + print $log "\nTEST: $confFile::$testName\n"; + print $log "******************************************************\n"; + print $log "Beginning test $testName at " . time . "\n"; + my %dbinfo = ( + 'testrun_id' => $testHash{'trid'}, + 'test_type' => $testHash{'driver'}, + #'test_file' => $testHash{'file'}, + 'test_file' => $confFile, + 'test_group' => $testHash{'group'}, + 'test_num' => $testHash{'num'}, + ); + my $beginTime = time; + my $endTime = 0; + my ($testResult, $benchmarkResult); + eval { + $testResult = $self->runTest(\%testHash, $log); + $endTime = time; + $benchmarkResult = $self->generateBenchmark(\%testHash, $log); + my $result = + $self->compare($testResult, $benchmarkResult, $log, \%testHash); + $msg = "INFO: $subName() at ".__LINE__.":Test $testName"; + + if ($result eq $self->{'wrong_execution_mode'}) { + $msg .= " SKIPPED"; + $testStatuses->{$testName} = $skippedStr; + } elsif ($result) { + $msg .= " SUCCEEDED"; + $testStatuses->{$testName} = $passedStr; + + } else { + $msg .= " FAILED"; + $testStatuses->{$testName} = $failedStr; + + } + $msg= "$msg at " . time . "\n"; + #print $msg; + print $log $msg; + $duration = $endTime - $beginTime; + $dbinfo{'duration'} = $duration; + $self->recordResults($result, $testResult + , $benchmarkResult, \%dbinfo, $log); + + }; + + + if ($@) { + $msg= "ERROR $subName at : ".__LINE__." Failed to run test $testName <$@>\n"; + #print $msg; + print $log $msg; + $testStatuses->{$testName} = $abortedStr; + $dbinfo{'duration'} = $duration; + } + + + eval { + $dbinfo{'status'} = $testStatuses->{$testName}; + if ($dbh) { + $dbh->insertTestCase(\%dbinfo); + } + }; + if ($@) { + chomp $@; + warn "Failed to insert test case info, error <$@>\n"; + } + + $self->cleanup($testStatuses->{$testName}, \%testHash, $testResult, + $benchmarkResult, $log); + #$report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName}, $testResult ) if ( $report ); + $report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName} ) if ( $report ); + $groupDuration = $groupDuration + $duration; + $totalDuration = $totalDuration + $duration; + printResults( $testStatuses, $log, "Results so far" ); + } + + if ( $report ) { + $report->systemOut( $logname, $group->{'name'}); + printGroupResultsXml( $report, $group->{'name'}, $testStatuses, $groupDuration ); + } + $report = 0; + $groupDuration=0; + + + } + + # Do the global cleanup + $self->globalCleanup(\%globalHash, $log); + } + +# TODO These should be removed + +sub tmpIPCRun(){ + + my $self = shift; + my $subName = (caller(0))[3]; + my $runningSubName= shift; + my $refCmd = shift; + my @cmd = @$refCmd; + my $log = shift; + my $msg = shift; + + print $log "$0::$subName INFO Running ( @cmd )\n"; + + my $result= `@cmd`; + if ( $@ ) { + my $msg= "$0::$subName FATAL Failed to run from $runningSubName $msg < $@ >\n$result\n"; + print $log $msg; + die "$msg"; + } + + return $?; +} + +sub tmpIPCRunSplitStdoe { + + my $self = shift; + my $subName = (caller(0))[3]; + my $runningSubName= shift; + my $refCmd = shift; + my @cmd = @$refCmd; + my $dir = shift; + my $log = shift; + my $msg = shift; + my $die = shift; + + + my $failed = 0; + + my $outfilename = $dir."out.tmp"; + my $errfilename = $dir."err.tmp"; + print $log "$0::$subName INFO Running from $runningSubName ( @cmd 1>$outfilename 2>$errfilename )\n"; + #make sure files are writeable + open( TMP, ">$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for writing\n"; + close( TMP ); + open( TMP, ">$errfilename" ) || die "$0::$subName FATAL: Cannot open $errfilename for writing\n"; + close( TMP ); + + #RUN CMD + my $msg; + print $log `@cmd 1>$outfilename 2>$errfilename`; + + my $failed=0; + if ( $@ ) { + $msg= "$0::$subName FATAL < $@ >\n"; + $failed++; + } + + #READ FILES + my $stdout=""; + my $stderr="";; + open( TMP, "$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for reading\n"; + while ( ) { + $stdout .= $_; + } + close( TMP ); + + open( TMP, "$errfilename" ) || die "$0::$subName FATAL: Cannot open $errfilename for reading\n"; + while ( ) { + $stderr .= $_; + } + close( TMP ); + + #DIE IF Test Failed, otherwise return stdout and stderr + if ( $failed ) { + + $msg = "$0::$subName FATAL: Faied from $runningSubName \nSTDOUT:" . $stdout . "\nSTDERR:" . $stderr . "\n" if ( $failed ); + print $log "$msg"; + die $msg if ( $die != "1" ); #die by defaultast + return ( -1, $stdout, $stderr ); + + } + + return ( $?, $stdout, $stderr); +} + +sub tmpIPCRunJoinStdoe { + + my $self = shift; + my $subName = (caller(0))[3]; + my $runningSubName= shift; + my $refCmd = shift; + my @cmd = @$refCmd; + my $outfilename= shift; + my $log = shift; + my $msg = shift; + my $die = shift; + + #make sure files are writeable + open( TMP, ">$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for writing\n"; + close( TMP ); + + #RUN CMD + my $msg; + my $failed=0; + print $log "$0::$subName INFO Running ( @cmd 2>&1$outfilename 2>/dev/null )\n"; + print $log `@cmd 2>&1 > $outfilename 2>/dev/null`; + if ( $@ ) { + $failed++; + $msg= "$0::$subName FATAL < $@ >\n"; + } + + #READ FILES + my $stdoe=""; + open( TMP, "$outfilename" ) || die "$0::$subName FATAL: Cannot open $outfilename for reading\n"; + while ( ) { + $stdoe .= $_; + } + close( TMP ); + + if ( $failed ) { + print $log "$msg"; + die $msg if ( $die != "1" ); #die by default + return ( -1 ); + } + return ( $? ); +} + + +1; Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jaradditionaljars.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jaradditionaljars.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jaradditionaljars.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jaradditionaljars.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1,5 @@ +-- no register command +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate org.apache.pig.piggybank.evaluation.string.Reverse($0); +store f into '$OUTDIR/reversed.txt'; + Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jarregistered.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jarregistered.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jarregistered.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/jarregistered.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1,5 @@ +register piggybank.jar +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate org.apache.pig.piggybank.evaluation.string.Reverse($0); +store f into '$OUTDIR/reversed.txt'; + Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/loadstore.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/loadstore.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/loadstore.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/loadstore.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1,2 @@ +l = load '$INPDIR/nums.txt'; +store l into '$OUTDIR/loadstore.out'; Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/nums.txt URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/nums.txt?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/nums.txt (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/nums.txt Wed Jul 25 20:29:44 2012 @@ -0,0 +1,2 @@ +1 3 +2 4 Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/pythonudf.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/pythonudf.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/pythonudf.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/pythonudf.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1,5 @@ +register 'udfs.py' using jython as myfuncs; +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate $0, myfuncs.helloworld() ; +store f into '$OUTDIR/pyhello.txt'; + Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcount_withmacro.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcount_withmacro.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcount_withmacro.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcount_withmacro.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1,5 @@ +import 'rowcountmacro.pig'; + +l = load '$INPDIR/nums.txt'; +c = row_count(l); +store c into '$OUTDIR/rowcount.out'; Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcountmacro.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcountmacro.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcountmacro.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/rowcountmacro.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1 @@ +DEFINE row_count(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate COUNT($X); }; Added: incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/udfs.py URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/udfs.py?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/udfs.py (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/inpdir/udfs.py Wed Jul 25 20:29:44 2012 @@ -0,0 +1,3 @@ +@outputSchema("word:chararray") +def helloworld(): + return 'Hello, World' Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jaradditionaljars.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jaradditionaljars.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jaradditionaljars.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jaradditionaljars.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1,5 @@ +-- no register command +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate org.apache.pig.piggybank.evaluation.string.Reverse($0); +store f into '$OUTDIR/reversed.txt'; + Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jarregistered.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jarregistered.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jarregistered.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/jarregistered.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1,5 @@ +register piggybank.jar +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate org.apache.pig.piggybank.evaluation.string.Reverse($0); +store f into '$OUTDIR/reversed.txt'; + Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/loadstore.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/loadstore.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/loadstore.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/loadstore.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1,2 @@ +l = load '$INPDIR/nums.txt'; +store l into '$OUTDIR/loadstore.out'; Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pigtest.txt URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pigtest.txt?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pigtest.txt (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pigtest.txt Wed Jul 25 20:29:44 2012 @@ -0,0 +1,49 @@ +checks for all of them - +- lanucher launched ok +- job succeeded (and pig exit status 0)/failed (tests below succeed unless specified) + + +#1 +pig -e 'fs -ls :INPDIR:/' +- stdout after job finish has list of files including some known one (.*nums.txt.*) + +#2 +pig -e 'erroneous syntax' +- job failed +- stderr contains syntax error message + +#3 +pig -check "l = load 'x';" + +#4 +- add loadstore.pig to files to be copied +pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: loadstore.pig + +#5 +- add piggybank.jar to files to be copied +pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: jarregistered.pig + +#6 +#jar specified in cmdline option instead of explicit register in script +- add piggybank.jar to files to be copied +pig -Dpig.additional.jars=piggybank.jar -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: jaradditionaljars.pig + + +#7 +- add rowcountmacro.pig to files to be copied +pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: rowcount_withmacro.pig + + +#8 +# test error when input file is not present +pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: no_such_file.pig +- non zero exit status + +#9 +#i haven't been able to get this test working, it complains of not being able to find PyException class +# would reccomend taking daniel's help to get it working +- add udfs.py to files to be copied +pig -p INPDIR=:INPDIR: -p OUTDIR=:OUTDIR: pythonudf.pig + + + Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pythonudf.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pythonudf.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pythonudf.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/pythonudf.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1,5 @@ +register 'udfs.py' using jython as myfuncs; +l = load '$INPDIR/nums.txt' as (a : chararray); +f = foreach l generate $0, myfuncs.helloworld() ; +store f into '$OUTDIR/pyhello.txt'; + Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcount_withmacro.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcount_withmacro.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcount_withmacro.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcount_withmacro.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1,5 @@ +import 'rowcountmacro.pig'; + +l = load '$INPDIR/nums.txt'; +c = row_count(l); +store c into '$OUTDIR/rowcount.out'; Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcountmacro.pig URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcountmacro.pig?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcountmacro.pig (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/rowcountmacro.pig Wed Jul 25 20:29:44 2012 @@ -0,0 +1 @@ +DEFINE row_count(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate COUNT($X); }; Added: incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/udfs.py URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/udfs.py?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/udfs.py (added) +++ incubator/hcatalog/trunk/src/test/e2e/templeton/newtests/udfs.py Wed Jul 25 20:29:44 2012 @@ -0,0 +1,3 @@ +@outputSchema("word:chararray") +def helloworld(): + return 'Hello, World'