incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1325149 - in /incubator/hcatalog/branches/branch-0.4: ./ src/test/e2e/hcatalog/ src/test/e2e/hcatalog/conf/ src/test/e2e/hcatalog/drivers/
Date Thu, 12 Apr 2012 07:51:04 GMT
Author: gates
Date: Thu Apr 12 07:51:04 2012
New Revision: 1325149

URL: http://svn.apache.org/viewvc?rev=1325149&view=rev
Log:
HCATALOG-357 e2e harness: allow to run again pre-installed packages

Modified:
    incubator/hcatalog/branches/branch-0.4/CHANGES.txt
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/existing_deployer.conf
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm

Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Thu Apr 12 07:51:04 2012
@@ -75,6 +75,8 @@ Release 0.4.0 - Unreleased
   HCAT-2 Support nested schema conversion between Hive an Pig (julienledem via hashutosh)
 
   IMPROVEMENTS
+  HCAT-357 e2e harness: allow to run again pre-installed packages (thw via gates)
+
   HCAT-352 Need tests for complex types (gates)
 
   HCAT-268 Remove remnants of storage drivers. (rohini via gates)

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml Thu Apr 12 07:51:04
2012
@@ -20,19 +20,7 @@
   <property name="hcat.dir" value="${basedir}/../../../../"/>
   <property name="hive.dir" value="${basedir}/../../../../hive/external/"/>
   <property name="hcat.jar" value="${hcat.dir}/build/hcatalog/hcatalog-${hcatalog.version}.jar"/>
-  
-  <path id="hcat.deps">
-    <fileset dir="${hive.dir}/lib/">
-     <include name="*.jar" />
-   </fileset>
-   <fileset dir="${hive.dir}/build/dist/lib/" erroronmissingdir="false">
-     <include name="*.jar" />
-     <exclude name="antlr*.jar" />
-   </fileset>
-  <pathelement location="${hcat.jar}" />
-  </path>
 
-  <property name="hcat-deps" refid="hcat.deps"/>
   <!-- Separate property name for udfs' build.xml -->
   <property name="hcat.jarfile" value="${hcat.jar}"/>
   <property name="udf.dir" value="${basedir}/udfs"/>
@@ -53,6 +41,8 @@
 
   <property name="harness.dir" value="${basedir}/../harness"/>
   <property name="harness.tar" value="${harness.dir}/harness.tar"/>
+  <property name="harness.PH_LOCAL" value="."/>
+  <property name="harness.PH_OUT" value="."/>
 
   <property name="test.location" value="${basedir}/testdist"/>
   <property name="benchmark.location" value="${test.location}/benchmarks"/>
@@ -158,6 +148,77 @@
       unless="harness.hadoop.home"/>
     <fail message="Please set the property hive.metastore.uris to the hcat thrift server"
       unless="hive.metastore.uris"/>
+    <fail message="Please set the property harness.pig.home to the path of your pig installation"
+      unless="harness.pig.home"/>
+
+  <dirset id="hcat.dist.lib.dir.fs" dir="${hcat.dir}/build" includes="hcatalog-*/share/hcatalog/lib"/>
+  <property name="hcat.dist.lib.dir" value="${hcat.dir}/build/${toString:hcat.dist.lib.dir.fs}"/>
+  <echo message="hcat libs: ${hcat.dist.lib.dir}"/>
+
+  <path id="hbase.jars.path">
+    <fileset dir="${hcat.dist.lib.dir}">
+     <include name="hbase-*.jar" />
+   </fileset>
+  </path>
+  <!-- override this for secure hbase -->
+  <property name="hbase.jars" refid="hbase.jars.path"/>
+
+  <path id="hcat.jars.path">
+    <fileset dir="${hcat.dir}/build/hcatalog">
+     <include name="hcatalog-*.jar" />
+     <exclude name="hcatalog-server-extensions-*.jar" />
+   </fileset>
+  </path>
+  <property name="hcat.jars" refid="hcat.jars.path"/>
+
+  <path id="hcat.client.deps">
+    <pathelement path="${hcat.jars}"/>
+    <pathelement path="${hbase.jars}"/>
+    <fileset dir="${hcat.dist.lib.dir}">
+     <include name="hbase-storage-handler-*.jar" />
+     <include name="zookeeper-*.jar" />
+     <include name="guava-*.jar" /><!-- for hbase storage handler -->
+   </fileset>
+   <!-- hbase-storage-handler only here with default ant target -->
+   <fileset dir="${hcat.dir}/storage-handlers/hbase/build/hbase-storage-handler-0.1.0/lib">
+     <include name="hbase-storage-handler-*.jar" />
+     <include name="zookeeper-*.jar" />
+   </fileset>
+   <fileset dir="${hive.dir}/build/dist/lib/" erroronmissingdir="false">
+     <include name="hive-*.jar" />
+     <include name="libfb303-*.jar" />
+     <include name="guava-*.jar" />
+     <include name="antlr*.jar" />
+   </fileset>
+  </path>
+
+  <pathconvert pathsep="," property="hadoop.libjars" refid="hcat.client.deps"/>
+  <property name="hive.conf.dir" value="${hcat.install.dir}/etc/hcatalog"/>
+
+  <path id="pig.classpath">
+     <path refid="hcat.client.deps" />
+     <pathelement location="${hive.conf.dir}" />
+  </path>
+  <property name="pig.classpath" refid="pig.classpath"/>
+
+  <!-- copy of above w/o antlr -->
+  <path id="pig.additional.jars">
+    <pathelement path="${hcat.jars}"/>
+    <pathelement path="${hbase.jars}"/>
+    <fileset dir="${hcat.dist.lib.dir}">
+     <include name="hbase-storage-handler-*.jar" />
+     <include name="zookeeper-*.jar" />
+     <include name="guava-*.jar" /><!-- for hbase storage handler -->
+   </fileset>
+   <fileset dir="${hive.dir}/build/dist/lib/" erroronmissingdir="false">
+     <include name="hive-*.jar" />
+     <include name="libfb303-*.jar" />
+     <!--include name="antlr*.jar" /-->
+   </fileset>
+   <pathelement location="${hive.conf.dir}" />
+  </path>
+  <property name="pig.additional.jars" refid="pig.additional.jars"/>
+
   </target>
 
   <!-- Prep the test area -->
@@ -176,20 +237,21 @@
   </target>
 
   <target name="test" depends="property-check, udfs, tar, init-test">
+
     <!-- If they have not specified tests to run then null it out -->
      <property name="tests.to.run" value=""/> 
     <echo />
     <exec executable="./test_harness.pl" dir="${test.location}">
       <env key="HARNESS_ROOT" value="."/>
-      <env key="PH_LOCAL" value="."/>
-      <env key="PH_OUT" value="."/>
+      <env key="PH_LOCAL" value="${harness.PH_LOCAL}"/>
+      <env key="PH_OUT" value="${harness.PH_OUT}"/>
       <env key="PH_ROOT" value="."/>
       <env key="HCAT_ROOT" value="${hcat.dir}"/>
       <env key="HCAT_INSTALL_DIR" value="${hcat.install.dir}"/>
       <env key="HIVE_ROOT" value="${hcat.dir}/hive/external/"/>
-      <env key="HCAT_EXTRA_JARS" value="${hcat-deps}/"/>
-      <env key="PIG_CLASSPATH" value="${hcat-deps}/"/>
-      <env key="HCAT_JAR" value="${hcat.jar}/"/>
+      <env key="PIG_CLASSPATH" value="${pig.classpath}/"/>
+      <env key="HADOOP_LIBJARS" value="${hadoop.libjars}"/> 
+      <env key="HCAT_JARS" value="${hcat.jars}"/>
       <env key="HADOOP_HOME" value="${harness.hadoop.home}/"/>
       <env key="PH_OLDPIG" value="${harness.old.pig}"/>
       <env key="PH_CLUSTER" value="${harness.cluster.conf}"/>
@@ -198,8 +260,9 @@
       <env key="HIVE_HOME" value="${basedir}/../../../../hive/external/build/dist"/>
       <env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
       <env key="PIG_HOME" value="${harness.pig.home}"/>
-      <env key="PIG_JAR" value="${harness.pig.jar}"/> <!-- Pig jar without antlr
-->
       <env key="HBASE_CONF_DIR" value="${hbase.conf.dir}"/>
+      <env key="HIVE_CONF_DIR" value="${hive.conf.dir}"/>
+      <env key="PIG_ADDITIONAL_JARS" value="${pig.additional.jars}"/>
       <arg line="${tests.to.run}"/>
       <arg value="${test.location}/tests/pig.conf"/>
       <arg value="${test.location}/tests/hive.conf"/>
@@ -221,14 +284,13 @@
   <target name="deploy-base" depends="property-check, tar, init-test, init-deploy">
     <exec executable="./test_harness.pl" dir="${test.location}">
       <env key="HARNESS_ROOT" value="."/>
-      <env key="PH_LOCAL" value="."/>
-      <env key="PH_OUT" value="."/>
+      <env key="PH_LOCAL" value="${harness.PH_LOCAL}"/>
+      <env key="PH_OUT" value="${harness.PH_OUT}"/>
       <env key="PH_ROOT" value="."/>
       <env key="HADOOP_HOME" value="${harness.hadoop.home}/"/>
       <env key="HIVE_ROOT" value="${hcat.dir}/hive/external/"/>
       <env key="HCAT_ROOT" value="${hcat.dir}"/>
       <env key="HCAT_INSTALL_DIR" value="${hcat.install.dir}"/>
-      <env key="HCAT_EXTRA_JARS" value="${hcat-deps}/"/>
       <env key="PH_OLDPIG" value="${harness.old.pig}"/>
       <env key="PH_CLUSTER" value="${harness.cluster.conf}"/>
       <env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf Thu Apr
12 07:51:04 2012
@@ -17,11 +17,6 @@
 my $me = `whoami`;
 chomp $me;
 
-#get the hive version from the hive build.properties
-my $hive_build_properties = "$ENV{HCAT_ROOT}/hive/external/build.properties";
-my $hive_version = `cat $hive_build_properties | grep '^version=' | cut -d= -f2`;
-chomp $hive_version;
-
 # The contents of this file can be rewritten to fit your installation.
 # Also, you can define the following environment variables and set things up as in the test
setup
 # PH_ROOT     Root directory where test harness is installed
@@ -30,10 +25,12 @@ chomp $hive_version;
 # PH_CLUSTER  Root directory for cluster being used
 # HCAT_ROOT   Root directory for hcat version being used
 
+my $hdfsBase = $ENV{'PH_HDFS_BASE'} || "/user/hcat";
+
 $cfg = {
     #HDFS
-    'inpathbase'     => '/user/hcat/tests/data'
-    , 'outpathbase'    => '/user/hcat/out'
+    'inpathbase'     => "$hdfsBase/tests/data"
+    , 'outpathbase'    => "$hdfsBase/out"
 
    #LOCAL
     , 'localinpathbase'   => "$ENV{PH_LOCAL}/in" 
@@ -48,10 +45,10 @@ $cfg = {
     , 'jythonjar'        => "$ENV{PH_JYTHON_JAR}"
 
     #TESTDB
-    , 'dbuser'         => 'hcattest'
-    , 'dbhost'         => 'localhost'
-    , 'dbpasswd'       => 'hcattest'
-    , 'dbdb'           => 'hcattestdb'
+    , 'dbuser'         => "$ENV{'PH_DBUSER'}" || 'hcattest'
+    , 'dbhost'         => "$ENV{'PH_DBHOST'}" || 'localhost'
+    , 'dbpasswd'       => "$ENV{'PH_DBPASSWD'}" || 'hcattest'
+    , 'dbdb'           => "$ENV{'PH_DBDB'}" || 'hcattestdb'
 
     #COMMON
     , 'metastore.principal' => "$ENV{METASTORE_PRINCIPAL}"   
@@ -62,7 +59,7 @@ $cfg = {
     , 'hcat_data_dir'    => '/user/hcat/tests/data'
     , 'hivehome'          => $ENV{'PH_HIVE_HOME'}
     , 'hcathome'          => $ENV{'HCAT_INSTALL_DIR'}
-    , 'hcatalog.jar' => "$ENV{HCAT_JAR},$ENV{HIVE_ROOT}/build/dist/lib/hive-serde-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-exec-$hive_version.jar,$ENV{PIG_HOME}/pig-0.9.2-withouthadoop.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-metastore-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/libfb303-0.7.0.jar,$ENV{HIVE_ROOT}/build/dist/lib/jdo2-api-2.3-ec.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/hbase-0.92.0.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/zookeeper-3.4.3.jar,$ENV{HCAT_ROOT}/storage-handlers/hbase/build/hbase-storage-handler/hbase-storage-handler-0.1.0.jar,$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar"
+    , 'hcatalog.jar'          => $ENV{'HADOOP_LIBJARS'}
 
     #PIG
     , 'testconfigpath'   => "$ENV{PH_CLUSTER}"
@@ -70,13 +67,11 @@ $cfg = {
     , 'funcjarPath'      => "$ENV{PH_ROOT}/lib/java"
     , 'paramPath'        => "$ENV{PH_ROOT}/paramfiles"
     , 'pigpath'          => "$ENV{PIG_HOME}"
-    , 'pigjar'           => "$ENV{PIG_JAR}" # Pig jar that doesn't have Antlr
     , 'oldpigpath'       => "$ENV{PH_OLDPIG}"
-    , 'additionaljars' => "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.4.0.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar::$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/hbase-0.92.0.jar:$ENV{HCAT_ROOT}/storage-handlers/hbase/build/ivy/lib/hbase-storage-handler/zookeeper-3.4.3.jar:$ENV{HCAT_ROOT}/storage-handlers/hbase/build/hbase-storage-handler/hbase-storage-handler-0.1.0.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog"
+    , 'additionaljars' => "$ENV{PIG_ADDITIONAL_JARS}"
 
     #HADOOP
-    , 'hadoopHome'       => "$ENV{HCAT_ROOT}/lib"
-    , 'hadoop_classpath' =>  "$ENV{HCAT_EXTRA_JARS}"
+    , 'UNUSEDhadoopHome'       => "$ENV{HCAT_ROOT}/lib"
     , 'userhomePath' => "$ENV{HOME}"
     , 'local.bin'     => '/usr/bin'
     , 'logDir'                => "$ENV{PH_OUT}/log" 
@@ -87,6 +82,7 @@ $cfg = {
     , 'hive_bin_location' => "$ENV{HIVE_ROOT}/build/dist/bin" 
     , 'hbaseconfigpath'   => "$ENV{HBASE_CONF_DIR}"
     , 'hivehome' => "$ENV{HIVE_HOME}"
-    , 'hive.additionaljars' =>  "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.4.0.jar"
+    , 'hive.additionaljars' =>  "$ENV{HCAT_JARS}"
+    , 'hive.conf.dir' => "$ENV{HIVE_CONF_DIR}" || "$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog"
 
 };

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/existing_deployer.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/existing_deployer.conf?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/existing_deployer.conf
(original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/existing_deployer.conf
Thu Apr 12 07:51:04 2012
@@ -26,7 +26,7 @@ $cfg = {
 	
 	# hadoop values
 	'hadoopdir'   => $ENV{'PH_CLUSTER'},
-        'hcat_data_dir'  => '/user/hcat/test/data',
+        'hcat_data_dir'  => ("$ENV{'PH_HDFS_BASE'}" || '/user/hcat').'/test/data',
 
 	# db values
 # 	'dbuser' => 'pigtester',

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
(original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
Thu Apr 12 07:51:04 2012
@@ -335,25 +335,18 @@ sub runHadoop
     push(@cmd, split(/ +/,$hadoopcmd));
 
     # Set HADOOP_CLASSPATH environment variable if provided
-    if (defined($testCmd->{'hadoop_classpath'})) {
-        #my $hadoop_classpath = $self->replaceParameters( $testCmd->{'hadoop_classpath'},
$outfile, $testCmd, $log );
-        # TODO This is a total mess.  Half the jars we need are specified in hcatalog.jar,
which is set in the default.conf file, and half are set in hadoop_classpath, which
-        # has to be passed on the command line.  And most of the jars are Hive jars, thus
they don't fit in the list of either hcatalog.jar or hadoop_classpath.  We need to
-        # make sense of this.
-        my $cp = $testCmd->{'hcatalog.jar'} . ":" . $testCmd->{'hadoop_classpath'};
-        $cp =~ s/,/:/g;
-        $cp .= ":" . Util::findPigWithoutHadoopJar($testCmd, $log);
-        # Add in the hcat config file
-        $cp .= ":" . $testCmd->{'hcathome'} . "/etc/hcatalog";
-        $ENV{'HADOOP_CLASSPATH'} = $cp;
-    }
+    my $cp = $testCmd->{'hcatalog.jar'}; 
+    $cp =~ s/,/:/g;
+    # Add in the hcat config file
+    $cp .= ":" . $testCmd->{'hive.conf.dir'};
+    $ENV{'HADOOP_CLASSPATH'} = $cp;
 
     if (defined($testCmd->{'hbaseconfigpath'})) {
         $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconfigpath'}";
     }
 
     if (defined($testCmd->{'metastore.principal'}) && ($testCmd->{'metastore.principal'}
=~ m/\S+/)) {
-        $ENV{'HADOOP_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
+        $ENV{'HADOOP_OPTS'} = "$ENV{'HADOOP_OPTS'} -Dhive.metastore.kerberos.principal="
. $testCmd->{'metastore.principal'};
         $ENV{'HADOOP_CLIENT_OPTS'} = "-Dhive.metastore.kerberos.principal=" . $testCmd->{'metastore.principal'};
     }
 
@@ -367,6 +360,7 @@ sub runHadoop
     open(FH, ">$script") or die "Unable to open file $script to write script, $ERRNO\n";
     print FH join (" ", @cmd) . "\n";
     close(FH);
+
     my @result=`chmod +x $script`;
 
     # Run the command

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm?rev=1325149&r1=1325148&r2=1325149&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm Thu Apr 12
07:51:04 2012
@@ -161,7 +161,7 @@ sub runHiveCmdFromFile($$;$$$$)
 #       push(@cmd, "--hiveconf", "hive.metastore.sasl.enabled=false");
 #   }
 
-    $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hcathome'}/etc/hcatalog/";
+    $ENV{'HIVE_CONF_DIR'} = "$cfg->{'hive.conf.dir'}";
 
     if (defined($cfg->{'hive.additionaljars'})) {
         $ENV{'HIVE_AUX_JARS_PATH'} = $cfg->{'hive.additionaljars'};
@@ -560,13 +560,4 @@ sub getLocaleCmd 
           ."export LC_IDENTIFICATION=\"$locale\"";
 }
 
-sub findPigWithoutHadoopJar($$)
-{
-    my ($cfg, $log) = @_;
-
-    my $jar = `ls $cfg->{'pigpath'}/pig-*withouthadoop.jar`;
-    chomp $jar;
-    return $jar;
-}
-
 1;



Mime
View raw message