incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1163097 [4/7] - in /incubator/hcatalog/trunk: ./ src/test/e2e/ src/test/e2e/hcatalog/ src/test/e2e/hcatalog/conf/ src/test/e2e/hcatalog/data/ src/test/e2e/hcatalog/deployers/ src/test/e2e/hcatalog/drivers/ src/test/e2e/hcatalog/paramfiles/...
Date Tue, 30 Aug 2011 06:18:18 GMT
Added: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm?rev=1163097&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm (added)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm Tue Aug 30 06:18:16 2011
@@ -0,0 +1,460 @@
+#!/usr/bin/env perl 
+
+############################################################################           
+#  Licensed to the Apache Software Foundation (ASF) under one or more                  
+#  contributor license agreements.  See the NOTICE file distributed with               
+#  this work for additional information regarding copyright ownership.                 
+#  The ASF licenses this file to You under the Apache License, Version 2.0             
+#  (the "License"); you may not use this file except in compliance with                
+#  the License.  You may obtain a copy of the License at                               
+#                                                                                      
+#      http://www.apache.org/licenses/LICENSE-2.0                                      
+#                                                                                      
+#  Unless required by applicable law or agreed to in writing, software                 
+#  distributed under the License is distributed on an "AS IS" BASIS,                   
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.            
+#  See the License for the specific language governing permissions and                 
+#  limitations under the License.                                                      
+                                                                                       
+
+###########################################################################
+# Class: Util
+#
+# A collection of  helper subroutines.
+#
+
+
+package Util;
+
+##############################################################################
+#  Sub: localTime
+# 
+#  Returns:
+#  A string with the local time
+
+sub  localTime() {
+
+   my $retval = time();
+   
+   my $local_time = gmtime( $retval);
+
+   return $local_time;
+
+}
+
+##############################################################################
+#  Sub: formatedTime
+#  Returns the time with following format "$mday/$mon/$year $hour:$min:$sec $weekday[$wday]"
+#
+#  Returns:
+#  formated time
+
+sub  formatedTime() {
+
+   my @weekday = ("Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat");
+
+   my $retval = time();
+   
+   my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime(time);
+   $year = $year + 1900; 
+   $mon += 1;
+   return  "$mday/$mon/$year $hour:$min:$sec $weekday[$wday]\n";
+   
+
+}
+
+##############################################################################
+# Sub: execCmd
+# Records the command in the log and prints it to stdout before executing.
+#
+# Paramaters:
+# $log     - The log object 
+# $subName - The name of the subroutine the message originated at
+# $lineNo  - The line number  of the subroutine the message originated at
+# $cmd     - The command string to execute
+# $msg     - A string, the message to print
+# $level   - (optional)The logging level for the message: DEBUG, INFO, WARN, ERROR, FATAL
+#            defaults to DEBUG.
+#
+#  Returns:
+#  An array containing the  output from the executed command.
+#
+
+sub execCmd() {
+
+    my ( $log, $subName, $lineNo, $cmd, $level ) = @_ ;
+
+    my $count = @_;
+    my $thisSubName = (caller(0))[3];
+
+   #Check for errors in arguments
+   if ( $count < 4 ){
+
+       if ( $log ) {
+          $log->msg( $level, $thisSubName, __LINE__ , "Invalid number of arguments, got
$count=( @_ )" );
+
+       } else {
+          print "ERROR: $0 $thisSubName at ".__LINE__."Invalid number of arguments\n";
+    
+       }
+       return 1;
+   }
+
+   #Log command,  execute commdand, return results
+   $level  = "DEBUG" if ( !$level );
+   $log->msg( $level, $subName, $lineNo , "$cmd");
+
+    my @result = `$cmd`;
+
+    $log->msg( $level, $subName, $lineNo , "@cmd") if ( @cmd );
+
+    return @result;
+}
+
+sub getHadoopCmd
+{
+    my ( $properties ) = @_;
+
+    my $subName        = (caller(0))[3];
+    my @baseCmd;
+
+    die "$0.$subName: null properties" if (! $properties );
+
+    my $cmd;
+
+    $cmd = $properties->{'gridstack.root'} . "/hadoop/current/bin/hadoop";
+    if ( ! -x "$cmd" ) {
+      print STDERR "\n$0::$subName WARNING: Can't find hadoop command: $cmd\n";
+      $cmd = `which hadoop`;
+      chomp $cmd;
+      print STDERR "$0::$subName WARNING: Instead using command: $cmd\n";
+    }
+    if ( ! -x "$cmd" ) {
+      die "\n$0::$subName FATAL: Hadoop command does not exist: $cmd\n";
+    }
+    push (@baseCmd, $cmd);
+
+    push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'});
+
+    return @baseCmd;
+}
+
+
+sub getHiveCmd
+{
+    my ( $properties ) = @_;
+
+    my $subName        = (caller(0))[3];
+    my @baseCmd;
+
+    die "$0.$subName: null properties" if (! $properties );
+
+    my $cmd;
+
+    $cmd = $properties->{'hive_bin_location'} . "/hive";
+    if ( ! -x "$cmd" ) {
+      die "\n$0::$subName FATAL: Hive command does not exist: $cmd\n";
+    }
+    push (@baseCmd, $cmd);
+
+#   push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'});
+
+    return @baseCmd;
+}
+
+sub getHCatCmd
+{
+    my ( $properties ) = @_;
+
+    my $subName        = (caller(0))[3];
+    my @baseCmd;
+
+    die "$0.$subName: null properties" if (! $properties );
+
+    my $cmd;
+
+    $cmd = $properties->{'hcat_bin_location'};
+    if ( ! -x "$cmd" ) {
+         print STDERR "\n$0::$subName WARNING: Can't find hcat command: $cmd\n";
+         $cmd = `which hcat.sh`;
+         chomp $cmd;
+         print STDERR "$0::$subName WARNING: Instead using command: $cmd\n";
+    }
+    die "\n$0::$subName FATAL: hcat command does not exist: $cmd\n" if ( ! -x $cmd  ); 
+    $ENV{"hive.metastore.local"} = "false"; 
+    $ENV{"hive.metastore.uris"}  = "thrift://".$properties->{'thriftserver'}; 
+    push (@baseCmd, $cmd);
+    return @baseCmd;
+}
+
+
+
+sub getPigCmd
+{
+    my $subName        = (caller(0))[3];
+    my $jarkey         = shift;
+    my ( $properties ) = @_;
+    my $isPigSqlEnabled= 0;
+    my @baseCmd;
+    die "$0.$subName: null properties" if (! $properties );
+
+    #UGLY HACK for pig sql support
+    if ( $jarkey =~ /testsql/ ) {
+
+       $isPigSqlEnabled= 1;
+       $jarkey = "testjar";
+
+    }
+
+    my $cmd;
+    if ( $properties->{'use-pig.pl'} ) {
+      # The directive gives that
+      # 1) the 'pig' command will be called, as opposed to java
+      # 2) the conf file has full control over what options and parameters are 
+      #    passed to pig. 
+      #    I.e. no parameters should be passed automatically by the script here. 
+      #
+      # This allows for testing of the pig script as installed, and for testin of
+      # the pig script's options, including error testing. 
+
+
+      $cmd = $properties->{'gridstack.root'} . "/pig/" . $properties->{'pigTestBuildName'}
. "/bin/pig";
+      if ( ! -x "$cmd" ) {
+        print STDERR "\n$0::$subName WARNING: Can't find pig command: $cmd\n";
+        $cmd = `which pig`;
+        chomp $cmd;
+        print STDERR "$0::$subName WARNING: Instead using command: $cmd\n";
+      }
+      die "\n$0::$subName FATAL: Pig command does not exist: $cmd\n" if ( ! -x $cmd  );
+      push (@baseCmd, $cmd );
+   
+       if(defined($properties->{'additionaljars'})) {
+          push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'});
+        }
+    
+
+      if ( $properties->{'use-pig.pl'} eq 'raw' ) { # add _no_ arguments automatically
+        # !!! 
+	return @baseCmd;
+      }
+
+    } else {
+        $cmd="java";
+
+        # Set JAVA options
+
+        # User can provide only one of
+        # (-c <cluster>) OR (-testjar <jar> -testconfigpath <path>)
+        # "-c <cluster>" is allowed only in non local mode
+        if(defined($properties->{'cluster.name'})) {
+            # use provided cluster
+            @baseCmd = ($cmd, '-c', $properties->{'cluster.name'});
+        } else {
+    
+                die "\n$0::$subName FATAL: The jar file name must be passed in at the command
line or defined in the configuration file\n" if ( !defined( $properties->{$jarkey} ) );
+                die "\n$0::$subName FATAL: The jar file does not exist.\n" . $properties->{$jarkey}."\n"
if ( ! -e  $properties->{$jarkey}  );
+    
+            # use user provided jar
+                my $classpath;
+
+				if (defined $properties->{'jythonjar'}) {
+					$classpath = "$classpath:" . $properties->{'jythonjar'};
+				}
+                if( $properties->{'exectype'} eq "local") {
+                   # in local mode, we should not use
+                   # any hadoop-site.xml
+                   $classpath= "$classpath:" . $properties->{$jarkey};
+                   $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'}
) );
+                   @baseCmd = ($cmd, '-cp', $classpath, '-Xmx1024m');
+    
+                } else {
+    
+                   # non local mode, we also need to specify
+                   # location of hadoop-site.xml
+                   die "\n$0::$subName FATAL: The hadoop configuration file name must be
passed in at the command line or defined in the configuration file\n" 
+			if ( !defined( $properties->{'testconfigpath'} ) );
+                   die "\n$0::$subName FATAL $! " . $properties->{'testconfigpath'}."\n\n"
 
+                   	if (! -e $properties->{'testconfigpath'} );
+
+                   $classpath= "$classpath:" . $properties->{$jarkey}.":".$properties->{'testconfigpath'};
+                   $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'}
) );
+                   $classpath= "$classpath:$properties->{'howl.jar'}" if ( defined( $properties->{'howl.jar'}
) );
+                   @baseCmd = ($cmd, '-cp', $classpath );
+            }
+        }
+    
+        # sets the queue, for exampel "grideng"
+        if(defined($properties->{'queue'})) {
+          push( @baseCmd,'-Dmapred.job.queue.name='.$properties->{'queue'});
+        }
+    
+        if(defined($properties->{'additionaljars'})) {
+          push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'});
+        }
+    
+        if( ( $isPigSqlEnabled == 1 ) ){
+
+	    if(defined($properties->{'metadata.uri'})) {
+		push( @baseCmd, '-Dmetadata.uri='.$properties->{'metadata.uri'});
+	    }
+
+	    if(defined($properties->{'metadata.impl'})) {
+		push( @baseCmd, '-Dmetadata.impl='.$properties->{'metadata.impl'});
+	    }else{
+		push( @baseCmd, '-Dmetadata.impl=org.apache.hadoop.owl.pig.metainterface.OwlPigMetaTables');
+	    }
+        }
+
+        # Add howl support
+	if(defined($properties->{'howl.metastore.uri'})) {
+	  push( @baseCmd, '-Dhowl.metastore.uri='.$properties->{'howl.metastore.uri'});
+	}
+    
+      # Set local mode property
+      # if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~
"local" ) {
+      # Removed above 'if...' for Pig 8.
+        my $java=`which java`;
+        my $version=`file $java`;
+        if ( $version =~ '32-bit' ){
+           push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-i386-32');
+        } else {
+           push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-amd64-64');
+        }
+      # }
+
+
+        # Add user provided java options if they exist
+        if (defined($properties->{'java_params'})) {
+          push(@baseCmd, @{$properties->{'java_params'}});
+        }
+    
+        if(defined($properties->{'hod'})) {
+          push( @baseCmd, '-Dhod.server=');
+        }
+
+      # sets the permissions on the jobtracker for the logs
+      push( @baseCmd,'-Dmapreduce.job.acl-view-job=*');
+
+
+      # Add Main
+      push(@baseCmd, 'org.apache.pig.Main');
+
+      # Set local mode PIG option
+      if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~
"local" ) {
+          push(@baseCmd, '-x');
+          push(@baseCmd, 'local');
+      }
+
+      # Set Pig SQL options
+      if( ( $isPigSqlEnabled == 1 ) && defined($properties->{'metadata.uri'}))
{
+  
+         if ( defined($properties->{'testoutpath'}) ) {
+           push( @baseCmd, '-u' );
+           push( @baseCmd, $properties->{'testoutpath'} );
+         }
+  
+         push( @baseCmd, '-s' );
+         push( @baseCmd, '-f' );
+      }
+
+    } # end else of if use-pig.pl
+
+
+    # Add -latest or -useversion 
+    if ( $cmd =~ 'pig$' ) {
+      # Add -latest, or -useversion if 'current' is not target build
+      if ( defined($properties->{'pigTestBuildName'})) {
+        if ($properties->{'pigTestBuildName'} eq 'latest') {
+            push(@baseCmd, '-latest');
+        } elsif ($properties->{'pigTestBuildName'} ne 'current') {
+            push(@baseCmd, '-useversion', "$properties->{'pigTestBuildName'}");
+        }
+      }
+    } elsif ( $cmd =~ 'java' ) {
+
+      # is this ever used: ???
+      # Add latest if it's there
+      if (defined($properties->{'latest'})) {
+          push(@baseCmd, '-latest');
+      }
+    }
+
+    return @baseCmd;
+}
+
+
+sub getBasePigSqlCmd 
+{
+
+    my $subName        = (caller(0))[3];
+   
+   Util::getPigCmd( 'testsql', @_ );
+
+}
+
+sub getBasePigCmd 
+{
+
+    my $subName        = (caller(0))[3];
+   
+   Util::getPigCmd( 'testjar', @_ );
+
+}
+
+sub getLatestBasePigCmd 
+{
+
+    my $subName        = (caller(0))[3];
+   
+   Util::getPigCmd( 'latesttestjar', @_ );
+
+}
+
+
+sub getBenchmarkBasePigCmd 
+{
+    my $subName        = (caller(0))[3];
+    my ( $properties ) = @_;
+
+   Util::getPigCmd( 'benchmarkjar', @_ );
+
+}
+
+sub setLocale
+{
+   my $locale= shift;
+#   $locale = "en_US.UTF-8" if ( !$locale );
+$locale = "ja_JP.utf8" if ( !$locale );
+   $ENV[LC_CTYPE]="$locale";
+   $ENV[LC_NUMERIC]="$locale";
+   $ENV[LC_TIME]="$locale";
+   $ENV[LC_COLLATE]="$locale";
+   $ENV[LC_MONETARY]="$locale";
+   $ENV[LC_MESSAGES]="$locale";
+   $ENV[LC_PAPER]="$locale";
+   $ENV[LC_NAME]="$locale";
+   $ENV[LC_ADDRESS]="$locale";
+   $ENV[LC_TELEPHONE]="$locale";
+   $ENV[LC_MEASUREMENT]="$locale";
+   $ENV[LC_IDENTIFICATION]="$locale";
+}
+
+sub getLocaleCmd 
+{
+  my $locale= shift;
+  $locale = "en_US.UTF-8" if ( !$locale );
+
+  return     "export LC_CTYPE=\"$locale\";"
+          ."export LC_NUMERIC=\"$locale\";"
+          ."export LC_TIME=\"$locale\";"
+          ."export LC_COLLATE=\"$locale\";"
+          ."export LC_MONETARY=\"$locale\";"
+          ."export LC_MESSAGES=\"$locale\";"
+          ."export LC_PAPER=\"$locale\";"
+          ."export LC_NAME=\"$locale\";"
+          ."export LC_ADDRESS=\"$locale\";"
+          ."export LC_TELEPHONE=\"$locale\";"
+          ."export LC_MEASUREMENT=\"$locale\";"
+          ."export LC_IDENTIFICATION=\"$locale\"";
+}
+1;

Added: incubator/hcatalog/trunk/src/test/e2e/hcatalog/paramfiles/params_3
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/paramfiles/params_3?rev=1163097&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/paramfiles/params_3 (added)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/paramfiles/params_3 Tue Aug 30 06:18:16
2011
@@ -0,0 +1,3 @@
+# this is a comment
+fname = studenttab10k
+unusedkey = unusedvalueihope



Mime
View raw message