incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tof...@apache.org
Subject svn commit: r1235122 - in /incubator/hcatalog/branches/branch-0.3: ./ src/test/e2e/hcatalog/ src/test/e2e/hcatalog/conf/ src/test/e2e/hcatalog/drivers/ src/test/e2e/hcatalog/tests/ storage-drivers/hbase/ivy/
Date Tue, 24 Jan 2012 03:36:03 GMT
Author: toffer
Date: Tue Jan 24 03:36:03 2012
New Revision: 1235122

URL: http://svn.apache.org/viewvc?rev=1235122&view=rev
Log:
HCAT-219. Adding HBase e2e tests (daijy via toffer)

Modified:
    incubator/hcatalog/branches/branch-0.3/CHANGES.txt
    incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/build.xml
    incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/conf/default.conf
    incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
    incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
    incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/Util.pm
    incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/hadoop.conf
    incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/pig.conf
    incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/ivy/libraries.properties

Modified: incubator/hcatalog/branches/branch-0.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/CHANGES.txt?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.3/CHANGES.txt Tue Jan 24 03:36:03 2012
@@ -60,6 +60,8 @@ Release 0.3.0 (unreleased changes)
   HCAT-63. RPM package integration with Hadoop (khorgath via hashutosh)
 
   IMPROVEMENTS
+  HCAT-219. Adding HBase e2e tests (daijy via toffer)
+
   HCAT-223. Include HWI (vikram.dixit via khorgath)
 
   HCAT-179. Make HCatalog compile against Hadoop 0.23 (thw via khorgath)

Modified: incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/build.xml?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/build.xml (original)
+++ incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/build.xml Tue Jan 24 03:36:03
2012
@@ -198,6 +198,7 @@
       <env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
       <env key="PIG_HOME" value="${harness.pig.home}"/>
       <env key="PIG_JAR" value="${harness.pig.jar}"/> <!-- Pig jar without antlr
-->
+      <env key="HBASE_CONF_DIR" value="${hbase.conf.dir}"/>
       <arg line="${tests.to.run}"/>
       <arg value="${test.location}/tests/pig.conf"/>
       <arg value="${test.location}/tests/hive.conf"/>

Modified: incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/conf/default.conf?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/conf/default.conf Tue Jan
24 03:36:03 2012
@@ -85,6 +85,7 @@ $cfg = {
 
     #HIVE
     , 'hive_bin_location' => "$ENV{HIVE_ROOT}/build/dist/bin" 
+    , 'hbaseconfigpath'   => "$ENV{HBASE_CONF_DIR}"
     , 'hivehome' => "$ENV{HIVE_HOME}"
 
 };

Modified: incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
(original)
+++ incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
Tue Jan 24 03:36:03 2012
@@ -343,6 +343,10 @@ sub runHadoop
         $ENV{'HADOOP_CLASSPATH'} = $cp;
     }
 
+    if (defined($testCmd->{'hbaseconfigpath'})) {
+        $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconfigpath'}";
+    }
+
     if (defined($testCmd->{'metastore.principal'}) && ($testCmd->{'metastore.principal'}
=~ m/\S+/)) {
         $ENV{'HADOOP_OPTS'} = "-Dhcat.metastore.principal=" . $testCmd->{'metastore.principal'};
         $ENV{'HADOOP_CLIENT_OPTS'} = "-Dhcat.metastore.principal=" . $testCmd->{'metastore.principal'};
@@ -624,6 +628,7 @@ sub getPigCmd($$$)
     $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'}));
     # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined
     $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local");
#&& (! defined $ENV{'HADOOP_HOME'});
+    $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local"
&& defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'}
ne ""));
 
     # Set it in our current environment.  It will get inherited by the IPC::Run
     # command.

Modified: incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverPig.pm?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
(original)
+++ incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
Tue Jan 24 03:36:03 2012
@@ -375,6 +375,7 @@ sub getPigCmd($$$)
     $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'}));
     # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined
     $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local");
#&& (! defined $ENV{'HADOOP_HOME'});
+    $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local"
&& defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'}
ne ""));
 
     # Set it in our current environment.  It will get inherited by the IPC::Run
     # command.
@@ -396,7 +397,7 @@ sub getPigCmd($$$)
         $opts = $opts . " " . join(" ", @{$testCmd->{'java_params'}});
     }
 
-    $ENV{'PIG_OPTS'} = $opts;
+    $ENV{'PIG_OPTS'} = $ENV{'PIG_OPTS'} . " " . $opts;
 
 	print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
 	print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to
" . $ENV{'PIG_OPTS'} . "\n";

Modified: incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/Util.pm?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/drivers/Util.pm Tue Jan 24
03:36:03 2012
@@ -562,7 +562,7 @@ sub findPigWithoutHadoopJar($$)
 {
     my ($cfg, $log) = @_;
 
-    my $jar = `ls $cfg->{'pigpath'}/pig-*-withouthadoop.jar`;
+    my $jar = `ls $cfg->{'pigpath'}/pig-*withouthadoop.jar`;
     chomp $jar;
     return $jar;
 }

Modified: incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/hadoop.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/hadoop.conf?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/hadoop.conf (original)
+++ incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/hadoop.conf Tue Jan
24 03:36:03 2012
@@ -158,5 +158,21 @@ jar :FUNCPATH:/testudf.jar org.apache.hc
                                 },
                         ],
                 }, # end g
+                {
+                        'name' => 'Hadoop_HBase',
+                        'tests' => [
+                                {
+                                 'num' => 1
+                                ,'hcat_prep'=>q\drop table if exists hadoop_hbase_1;
+create table hadoop_hbase_1(key string, gpa string) STORED BY 'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'
TBLPROPERTIES ('hbase.columns.mapping'=':key,info:gpa');\
+                                ,'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.HBaseReadWrite -libjars :HCAT_JAR: :THRIFTSERVER:
:INPATH:/studenttab10k hadoop_hbase_1 :OUTPATH:
+\,
+                                ,'sql' => q\select name, sum(gpa) from studenttab10k group
by name;\
+                                ,'floatpostprocess' => 1
+                                ,'delimiter' => '	'
+                                },
+                        ],
+                }, # end g
          ]
 }

Modified: incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/pig.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/pig.conf?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/pig.conf (original)
+++ incubator/hcatalog/branches/branch-0.3/src/test/e2e/hcatalog/tests/pig.conf Tue Jan 24
03:36:03 2012
@@ -174,6 +174,41 @@ store b into 'pig_write_3' using org.apa
                                 }
                         ],
                 }, # end g
+                {
+                        'name' => 'Pig_HBase',
+                        'tests' => [
+                                {
+                                 'num' => 1
+                                ,'pig' => q\set hcat.hbase.output.bulkMode 'false'
+e = load 'pig_hbase_1' using org.apache.hcatalog.pig.HCatLoader();
+store e into ':OUTPATH:';\,
+                                ,'result_table' => ['?']
+				,'sql'   => [ 'select name, avg(cast(age as decimal(10,5))), avg(gpa) from studenttab10k
group by name;']
+                                ,'floatpostprocess' => 1
+                                ,'delimiter' => '	'
+                                },
+                                {
+                                 # multiquery
+                                 'num' => 2
+                                ,'hcat_prep'=>q\drop table if exists pig_hbase_2_1;
+create table pig_hbase_2_1(key string, age string, gpa string) STORED BY 'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'
TBLPROPERTIES ('hbase.columns.mapping'=':key,info:age,info:gpa');
+drop table if exists pig_hbase_2_2;
+create table pig_hbase_2_2(key string, age string, gpa string) STORED BY 'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'
TBLPROPERTIES ('hbase.columns.mapping'=':key,info:age,info:gpa');
+\
+                                ,'pig' => q\set hcat.hbase.output.bulkMode 'false'
+a = load ':INPATH:/studenttab10k' as (name:chararray, age:int, gpa:float);
+b = group a by name;
+c = foreach b generate group as name, AVG(a.age) as age, AVG(a.gpa) as gpa;
+d = foreach c generate name as key, (chararray)age, (chararray)gpa as gpa;
+store d into 'pig_hbase_2_1' using org.apache.hcatalog.pig.HCatStorer();
+store d into 'pig_hbase_2_2' using org.apache.hcatalog.pig.HCatStorer();\,
+                                ,'result_table' => ['pig_hbase_2_1','pig_hbase_2_2']
+				,'sql'   => [ 'select name, avg(cast(age as decimal(10,5))), avg(gpa) from studenttab10k
group by name;', 'select name, avg(cast(age as decimal(10,5))), avg(gpa) from studenttab10k
group by name;']
+                                ,'floatpostprocess' => 1
+                                ,'delimiter' => '	'
+                                }
+                        ],
+                }, # end g
 
          ]
 }

Modified: incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/ivy/libraries.properties?rev=1235122&r1=1235121&r2=1235122&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/ivy/libraries.properties
(original)
+++ incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/ivy/libraries.properties
Tue Jan 24 03:36:03 2012
@@ -16,6 +16,6 @@
 junit.version=3.8.1
 ivy.version=2.2.0
 rats-lib.version=0.5.1
-hbase.version=0.90.3
+hbase.version=0.90.5
 zookeeper.version=3.4.0
 thrift.version=0.7.0



Mime
View raw message