incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1167355 - in /incubator/hcatalog/trunk: ./ src/test/e2e/hcatalog/ src/test/e2e/hcatalog/conf/ src/test/e2e/hcatalog/drivers/ src/test/e2e/hcatalog/tests/
Date Fri, 09 Sep 2011 20:39:37 GMT
Author: hashutosh
Date: Fri Sep  9 20:39:36 2011
New Revision: 1167355

URL: http://svn.apache.org/viewvc?rev=1167355&view=rev
Log:
HCATALOG-97: Make test harness work with secure cluster

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Fri Sep  9 20:39:36 2011
@@ -25,6 +25,8 @@ Trunk (unreleased changes)
   NEW FEATURES
 
   IMPROVEMENTS
+  HCAT-97. Make test harness work with secure cluster (hashutosh)
+
   HCAT-94. Handle the job tracker tokens properly in har jobs (khorgath via hashutosh)
 
   HCAT-43. hcat.sh does not pick up hcatalog.jar correctly (macyang via hashutosh)

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml Fri Sep  9 20:39:36 2011
@@ -27,6 +27,7 @@
    </fileset>
    <fileset dir="${hive.dir}/build/dist/lib/">
      <include name="*.jar" />
+     <exclude name="antlr*.jar" />
    </fileset>
   <pathelement location="${hcat.jar}" />
   </path>
@@ -103,11 +104,7 @@
       <fileset dir="${tool.src}/test"/>
       <fileset dir="${tool.src}/generate"/>
     </copy>
-<!--
-    <copy todir="${tar.dir}/lib">
-      <fileset dir="${lib.dir}"/>
-    </copy>
--->
+
     <copy todir="${tar.dir}/lib/java">
       <fileset file="${udf.jar}"/>
     </copy>
@@ -151,7 +148,7 @@
 
   <target name="test" depends="property-check, udfs, tar, init-test">
     <!-- If they have not specified tests to run then null it out -->
-    <property name="tests.to.run" value=""/>
+     <property name="tests.to.run" value=""/> 
     <echo />
     <exec executable="./test_harness.pl" dir="${test.location}">
       <env key="HARNESS_ROOT" value="."/>
@@ -165,8 +162,8 @@
       <env key="PH_OLDPIG" value="${harness.old.pig}"/>
       <env key="PH_CLUSTER" value="${harness.cluster.conf}"/>
       <env key="HCAT_URL" value="${hcat.server.url}"/>
+      <env key="METASTORE_PRINCIPAL" value="${metastore.principal}"/>
       <arg line="${tests.to.run}"/>
-      <!-- <arg value="${test.location}/tests/bootstrap_hcat.conf"/> -->
       <arg value="${test.location}/tests/hcat.conf"/>
     </exec>
   </target>

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf Fri Sep  9 20:39:36 2011
@@ -63,7 +63,8 @@ $cfg = {
 
    #HIVE
     , 'hive_bin_location' => "$ENV{HIVE_ROOT}/build/dist/bin" 
-   
+
+    , 'metastore.principal' => "$ENV{METASTORE_PRINCIPAL}"   
   #HCATALOG
   ,'thriftserver' => "$ENV{HCAT_URL}"
      ,'hcatalog.jar' => "$ENV{HCAT_JAR},file://$ENV{HIVE_ROOT}/lib/thrift-fb303-0.5.0.jar,file://$ENV{HIVE_ROOT}/lib/thrift-0.5.0.jar,file://$ENV{HIVE_ROOT}/build/metastore/hive-metastore-0.8.0-SNAPSHOT.jar,file://$ENV{HIVE_ROOT}/build/common/hive-common-0.8.0-SNAPSHOT.jar,file://$ENV{HIVE_ROOT}/build/shims/hive-shims-0.8.0-SNAPSHOT.jar,file://$ENV{HIVE_ROOT}/build/serde/hive-serde-0.8.0-SNAPSHOT.jar,file://$ENV{HIVE_ROOT}/build/ql/hive-exec-0.8.0-SNAPSHOT.jar"

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHCat.pm Fri Sep  9 20:39:36
2011
@@ -442,8 +442,14 @@ sub runHive
     my @cmd = Util::getHiveCmd($testCmd);
 
     #Add metastore info
-    push(@cmd, " --hiveconf hive.metastore.local=false --hiveconf hive.metastore.uris=thrift://".$testCmd->{'thriftserver'});
+    push(@cmd, "--hiveconf hive.metastore.local=false --hiveconf hive.metastore.uris=thrift://".$testCmd->{'thriftserver'});
    
+   
+ if(defined($testCmd->{'metastore.principal'})){
+     push(@cmd, "--hiveconf hive.metastore.sasl.enabled=true  --hiveconf hive.metastore.kerberos.principal=$testCmd->{'metastore.principal'}");
+
+    }
+
     # Add hive command file
     push(@cmd, '-f', $hivefile); 
 
@@ -624,7 +630,11 @@ sub runHadoopCmdLine
         my $hadoop_classpath = $self->replaceParameters( $testCmd->{'hadoop_classpath'},
$outfile, $testCmd, $log );
         $ENV{'HADOOP_CLASSPATH'} = $ENV{'HCAT_EXTRA_JARS'};
     }
-
+    my $hadoop_opts = "-Dhive.metastore.uris=thrift://".$testCmd->{'thriftserver'}." -Dhcat.metastore.uri=thrift://".$testCmd->{'thriftserver'};
+    if (defined($testCmd->{'metastore.principal'})){
+	$hadoop_opts = join '',$hadoop_opts," -Dhive.metastore.sasl.enabled=true -Dhcat.metastore.principal=",$testCmd->{'metastore.principal'},"
-Dhive.metastore.kerberos.principal=",$testCmd->{'metastore.principal'};
+    }    
+    $ENV{'HADOOP_OPTS'} = $hadoop_opts;
     # Run the command
     print $log "$0:$subName Going to run command: $command\n";
     print $log "$0:$subName STD OUT IS IN FILE: $stdoutfile\n";
@@ -840,7 +850,13 @@ sub runPig
     my $locallog = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'}
. ".log";
     push(@cmd, "-logfile");
     push(@cmd, $locallog);
-
+    
+    my $pig_opts = "-Dhive.metastore.uris=thrift://".$testCmd->{'thriftserver'}." -Dhcat.metastore.uri=thrift://".$testCmd->{'thriftserver'};
+    if (defined($testCmd->{'metastore.principal'})){
+	$pig_opts = join '',$pig_opts," -Dhive.metastore.sasl.enabled=true -Dhcat.metastore.principal=",$testCmd->{'metastore.principal'},"
-Dhive.metastore.kerberos.principal=",$testCmd->{'metastore.principal'};
+    }    
+    $ENV{'PIG_OPTS'} = $pig_opts;
+  
     # Add pig parameters if they're provided
     if (defined($testCmd->{'pig_params'})) {
         # Processing :PARAMPATH: in parameters

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm Fri Sep  9 20:39:36 2011
@@ -231,7 +231,7 @@ sub getPigCmd
        if(defined($properties->{'additionaljars'})) {
           push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'});
         }
-    
+        $ENV{'PIG_CLASSPATH'}=$properties->{'additionaljars'};  
 
       if ( $properties->{'use-pig.pl'} eq 'raw' ) { # add _no_ arguments automatically
         # !!! 

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf?rev=1167355&r1=1167354&r2=1167355&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf Fri Sep  9 20:39:36 2011
@@ -346,7 +346,7 @@ limit 5
 			{
 			'num' => 1,
 			'hive' => "show tables;",
-			'expected_out_regex' => "OK",
+			'expected_err_regex' => "OK",
 			},
 			{
 			'num' => 5,
@@ -728,10 +728,10 @@ a = load 'default.boolean_table' using o
 b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('boolean+int', *);
 store b into ':OUTPATH:';
 ",
-                        'rc' => 0, 
-			'expected_err_regex' => "Success",
+                        'rc' => 6, 
+			'expected_err_regex' => "ERROR",
 			# 'expected_out_regex' => "1",
-			# 'not_expected_out_regex' => "[^1\\t]",
+			'not_expected_out_regex' => "Success",
                         },
 
                         
@@ -1427,7 +1427,7 @@ store A into 'default.tmp_pig2pig_stored
                         {
                         'num' => 3,
                         # complex stored by pig
-			'depends_on' => 'hcat_pig2pig_setup_tables_2', # not really, but #4 does
+			# 'depends_on' => 'hcat_pig2pig_setup_tables_2', # not really, but #4 does
                         'hive' => "
 CREATE TABLE tmp_pig2pig_stored_complex_:RUNID: (
   mymap       map<string, string>,
@@ -2249,7 +2249,6 @@ store b into ':OUTPATH:';
                         'pig' => "
 register :FUNCPATH:/testudf.jar;
 a = load 'default.tmp_pig2pig_stored_complex_:RUNID:' using org.apache.hcatalog.pig.HCatLoader();
--- b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('{mymap: map[],mytuple:
(num: int,str: chararray,dbl: double),bagofmap: {(innerfield: map[])},rownum: int', *);
 b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('mymap: map[],mytuple: (num:
int,str: chararray,dbl: double),bagofmap: {(innerfield: map[])},rownum: int', *);
 store b into ':OUTPATH:';
 ",



Mime
View raw message