incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tof...@apache.org
Subject svn commit: r1235121 - in /incubator/hcatalog/trunk: ./ src/test/e2e/hcatalog/ src/test/e2e/hcatalog/conf/ src/test/e2e/hcatalog/drivers/ src/test/e2e/hcatalog/tests/ src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ storage-drivers/hbase/ivy/
Date Tue, 24 Jan 2012 03:33:45 GMT
Author: toffer
Date: Tue Jan 24 03:33:44 2012
New Revision: 1235121

URL: http://svn.apache.org/viewvc?rev=1235121&view=rev
Log:
HCAT-219. Adding HBase e2e tests (daijy via toffer)

Added:
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/HBaseReadWrite.java
Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf
    incubator/hcatalog/trunk/storage-drivers/hbase/ivy/libraries.properties

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1235121&r1=1235120&r2=1235121&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Tue Jan 24 03:33:44 2012
@@ -60,6 +60,8 @@ Trunk (unreleased changes)
   HCAT-63. RPM package integration with Hadoop (khorgath via hashutosh)
 
   IMPROVEMENTS
+  HCAT-219. Adding HBase e2e tests (daijy via toffer) 
+
   HCAT-226. HBase storage drivers should load hbase resources (hbase-site, hbase-default)
for the user (hashutosh via toffer)
 
   HCAT-223. Include HWI (vikram.dixit via khorgath)

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml?rev=1235121&r1=1235120&r2=1235121&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml Tue Jan 24 03:33:44 2012
@@ -198,6 +198,7 @@
       <env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
       <env key="PIG_HOME" value="${harness.pig.home}"/>
       <env key="PIG_JAR" value="${harness.pig.jar}"/> <!-- Pig jar without antlr
-->
+      <env key="HBASE_CONF_DIR" value="${hbase.conf.dir}"/>
       <arg line="${tests.to.run}"/>
       <arg value="${test.location}/tests/pig.conf"/>
       <arg value="${test.location}/tests/hive.conf"/>

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf?rev=1235121&r1=1235120&r2=1235121&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf Tue Jan 24 03:33:44 2012
@@ -85,6 +85,7 @@ $cfg = {
 
     #HIVE
     , 'hive_bin_location' => "$ENV{HIVE_ROOT}/build/dist/bin" 
+    , 'hbaseconfigpath'   => "$ENV{HBASE_CONF_DIR}"
     , 'hivehome' => "$ENV{HIVE_HOME}"
 
 };

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1235121&r1=1235120&r2=1235121&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm Tue Jan 24
03:33:44 2012
@@ -343,6 +343,10 @@ sub runHadoop
         $ENV{'HADOOP_CLASSPATH'} = $cp;
     }
 
+    if (defined($testCmd->{'hbaseconfigpath'})) {
+        $ENV{'HADOOP_CLASSPATH'} = "$ENV{'HADOOP_CLASSPATH'}:$testCmd->{'hbaseconfigpath'}";
+    }
+
     if (defined($testCmd->{'metastore.principal'}) && ($testCmd->{'metastore.principal'}
=~ m/\S+/)) {
         $ENV{'HADOOP_OPTS'} = "-Dhcat.metastore.principal=" . $testCmd->{'metastore.principal'};
         $ENV{'HADOOP_CLIENT_OPTS'} = "-Dhcat.metastore.principal=" . $testCmd->{'metastore.principal'};
@@ -624,6 +628,7 @@ sub getPigCmd($$$)
     $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'}));
     # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined
     $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local");
#&& (! defined $ENV{'HADOOP_HOME'});
+    $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local"
&& defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'}
ne ""));
 
     # Set it in our current environment.  It will get inherited by the IPC::Run
     # command.

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm?rev=1235121&r1=1235120&r2=1235121&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverPig.pm Tue Jan 24 03:33:44
2012
@@ -375,6 +375,7 @@ sub getPigCmd($$$)
     $pcp .= ":" . $testCmd->{'additionaljars'} if (defined($testCmd->{'additionaljars'}));
     # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined
     $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local");
#&& (! defined $ENV{'HADOOP_HOME'});
+    $pcp .= ":" . $testCmd->{'hbaseconfigpath'} if ($testCmd->{'exectype'} ne "local"
&& defined($testCmd->{'hbaseconfigpath'} && $testCmd->{'hbaseconfigpath'}
ne ""));
 
     # Set it in our current environment.  It will get inherited by the IPC::Run
     # command.
@@ -396,7 +397,7 @@ sub getPigCmd($$$)
         $opts = $opts . " " . join(" ", @{$testCmd->{'java_params'}});
     }
 
-    $ENV{'PIG_OPTS'} = $opts;
+    $ENV{'PIG_OPTS'} = $ENV{'PIG_OPTS'} . " " . $opts;
 
 	print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
 	print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to
" . $ENV{'PIG_OPTS'} . "\n";

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm?rev=1235121&r1=1235120&r2=1235121&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm Tue Jan 24 03:33:44 2012
@@ -562,7 +562,7 @@ sub findPigWithoutHadoopJar($$)
 {
     my ($cfg, $log) = @_;
 
-    my $jar = `ls $cfg->{'pigpath'}/pig-*-withouthadoop.jar`;
+    my $jar = `ls $cfg->{'pigpath'}/pig-*withouthadoop.jar`;
     chomp $jar;
     return $jar;
 }

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf?rev=1235121&r1=1235120&r2=1235121&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf Tue Jan 24 03:33:44 2012
@@ -158,5 +158,21 @@ jar :FUNCPATH:/testudf.jar org.apache.hc
                                 },
                         ],
                 }, # end g
+                {
+                        'name' => 'Hadoop_HBase',
+                        'tests' => [
+                                {
+                                 'num' => 1
+                                ,'hcat_prep'=>q\drop table if exists hadoop_hbase_1;
+create table hadoop_hbase_1(key string, gpa string) STORED BY 'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'
TBLPROPERTIES ('hbase.columns.mapping'=':key,info:gpa');\
+                                ,'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.HBaseReadWrite -libjars :HCAT_JAR: :THRIFTSERVER:
:INPATH:/studenttab10k hadoop_hbase_1 :OUTPATH:
+\,
+                                ,'sql' => q\select name, sum(gpa) from studenttab10k group
by name;\
+                                ,'floatpostprocess' => 1
+                                ,'delimiter' => '	'
+                                },
+                        ],
+                }, # end g
          ]
 }

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf?rev=1235121&r1=1235120&r2=1235121&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf Tue Jan 24 03:33:44 2012
@@ -174,6 +174,41 @@ store b into 'pig_write_3' using org.apa
                                 }
                         ],
                 }, # end g
+                {
+                        'name' => 'Pig_HBase',
+                        'tests' => [
+                                {
+                                 'num' => 1
+                                ,'pig' => q\set hcat.hbase.output.bulkMode 'false'
+e = load 'pig_hbase_1' using org.apache.hcatalog.pig.HCatLoader();
+store e into ':OUTPATH:';\,
+                                ,'result_table' => ['?']
+				,'sql'   => [ 'select name, avg(cast(age as decimal(10,5))), avg(gpa) from studenttab10k
group by name;']
+                                ,'floatpostprocess' => 1
+                                ,'delimiter' => '	'
+                                },
+                                {
+                                 # multiquery
+                                 'num' => 2
+                                ,'hcat_prep'=>q\drop table if exists pig_hbase_2_1;
+create table pig_hbase_2_1(key string, age string, gpa string) STORED BY 'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'
TBLPROPERTIES ('hbase.columns.mapping'=':key,info:age,info:gpa');
+drop table if exists pig_hbase_2_2;
+create table pig_hbase_2_2(key string, age string, gpa string) STORED BY 'org.apache.hcatalog.hbase.HBaseHCatStorageHandler'
TBLPROPERTIES ('hbase.columns.mapping'=':key,info:age,info:gpa');
+\
+                                ,'pig' => q\set hcat.hbase.output.bulkMode 'false'
+a = load ':INPATH:/studenttab10k' as (name:chararray, age:int, gpa:float);
+b = group a by name;
+c = foreach b generate group as name, AVG(a.age) as age, AVG(a.gpa) as gpa;
+d = foreach c generate name as key, (chararray)age, (chararray)gpa as gpa;
+store d into 'pig_hbase_2_1' using org.apache.hcatalog.pig.HCatStorer();
+store d into 'pig_hbase_2_2' using org.apache.hcatalog.pig.HCatStorer();\,
+                                ,'result_table' => ['pig_hbase_2_1','pig_hbase_2_2']
+				,'sql'   => [ 'select name, avg(cast(age as decimal(10,5))), avg(gpa) from studenttab10k
group by name;', 'select name, avg(cast(age as decimal(10,5))), avg(gpa) from studenttab10k
group by name;']
+                                ,'floatpostprocess' => 1
+                                ,'delimiter' => '	'
+                                }
+                        ],
+                }, # end g
 
          ]
 }

Added: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/HBaseReadWrite.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/HBaseReadWrite.java?rev=1235121&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/HBaseReadWrite.java
(added)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/HBaseReadWrite.java
Tue Jan 24 03:33:44 2012
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.utils;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hcatalog.common.HCatConstants;
+import org.apache.hcatalog.data.DefaultHCatRecord;
+import org.apache.hcatalog.data.HCatRecord;
+import org.apache.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.hcatalog.mapreduce.HCatOutputFormat;
+import org.apache.hcatalog.mapreduce.InputJobInfo;
+import org.apache.hcatalog.mapreduce.OutputJobInfo;
+
+/**
+ * This is a map reduce test for testing hcat which goes against the "numbers"
+ * table. It performs a group by on the first column and a SUM operation on the
+ * other columns. This is to simulate a typical operation in a map reduce
+ * program to test that hcat hands the right data to the map reduce program
+ * 
+ * Usage: hadoop jar sumnumbers <serveruri> <output dir> <-libjars hive-hcat
+ * jar> The <tab|ctrla> argument controls the output delimiter The hcat jar
+ * location should be specified as file://<full path to jar>
+ */
+public class HBaseReadWrite extends Configured implements Tool {
+
+    public static class HBaseWriteMap extends
+            Mapper<LongWritable, Text, Text, Text> {
+
+        String name;
+        String age;
+        String gpa;
+
+        @Override
+        protected void map(
+                LongWritable key,
+                Text value,
+                org.apache.hadoop.mapreduce.Mapper<LongWritable, Text, Text, Text>.Context
context)
+                throws IOException, InterruptedException {
+            String line = value.toString();
+            String[] tokens = line.split("\t");
+            name = tokens[0];
+            
+            context.write(new Text(name), value);
+        }
+    }
+    
+
+    public static class HBaseWriteReduce extends
+            Reducer<Text, Text, WritableComparable, HCatRecord> {
+
+        String name;
+        String age;
+        String gpa;
+
+        @Override
+        protected void reduce(Text key, Iterable<Text> values, Context context)
+                throws IOException, InterruptedException {
+            name = key.toString();
+            int count = 0;
+            double sum = 0;
+            for (Text value : values) {
+                String line = value.toString();
+                String[] tokens = line.split("\t");
+                name = tokens[0];
+                age = tokens[1];
+                gpa = tokens[2];
+                
+                count++;
+                sum += Double.parseDouble(gpa.toString());
+            }
+            
+            HCatRecord record = new DefaultHCatRecord(2);
+            record.set(0, name);
+            record.set(1, Double.toString(sum));
+            
+            context.write(null, record);
+        }
+    }
+
+    public static class HBaseReadMap extends
+            Mapper<WritableComparable, HCatRecord, Text, Text> {
+
+        String name;
+        String age;
+        String gpa;
+
+        @Override
+        protected void map(
+                WritableComparable key,
+                HCatRecord value,
+                org.apache.hadoop.mapreduce.Mapper<WritableComparable, HCatRecord, Text,
Text>.Context context)
+                throws IOException, InterruptedException {
+            name = (String) value.get(0);
+            gpa = (String) value.get(1);
+            context.write(new Text(name), new Text(gpa));
+        }
+    }
+    
+
+    public int run(String[] args) throws Exception {
+        Configuration conf = getConf();
+        args = new GenericOptionsParser(conf, args).getRemainingArgs();
+
+        String serverUri = args[0];
+        String inputDir = args[1];
+        String tableName = args[2];
+        String outputDir = args[3];
+        String dbName = null;
+
+        String principalID = System
+                .getProperty(HCatConstants.HCAT_METASTORE_PRINCIPAL);
+        if (principalID != null)
+            conf.set(HCatConstants.HCAT_METASTORE_PRINCIPAL, principalID);
+        conf.set("hcat.hbase.output.bulkMode", "false");
+        Job job = new Job(conf, "HBaseWrite");
+        FileInputFormat.setInputPaths(job, inputDir);
+        
+        job.setInputFormatClass(TextInputFormat.class);
+        job.setOutputFormatClass(HCatOutputFormat.class);
+        job.setJarByClass(HBaseReadWrite.class);
+        job.setMapperClass(HBaseWriteMap.class);
+        job.setMapOutputKeyClass(Text.class);
+        job.setMapOutputValueClass(Text.class);
+        job.setReducerClass(HBaseWriteReduce.class);
+        job.setOutputKeyClass(WritableComparable.class);
+        job.setOutputValueClass(DefaultHCatRecord.class);
+        HCatOutputFormat.setOutput(job, OutputJobInfo.create(dbName,
+                tableName, null, serverUri, principalID));
+        
+        boolean succ = job.waitForCompletion(true);
+        
+        if (!succ) return 1;
+        
+        job = new Job(conf, "HBaseRead");
+        HCatInputFormat.setInput(job, InputJobInfo.create(dbName, tableName,
+                null, serverUri, principalID));
+        
+        job.setInputFormatClass(HCatInputFormat.class);
+        job.setOutputFormatClass(TextOutputFormat.class);
+        job.setJarByClass(HBaseReadWrite.class);
+        job.setMapperClass(HBaseReadMap.class);
+        job.setOutputKeyClass(Text.class);
+        job.setOutputValueClass(Text.class);
+        job.setNumReduceTasks(0);
+        TextOutputFormat.setOutputPath(job, new Path(outputDir));
+        
+        succ = job.waitForCompletion(true);
+        
+        if (!succ) return 2;
+        
+        return 0;
+    }
+
+    public static void main(String[] args) throws Exception {
+        int exitCode = ToolRunner.run(new HBaseReadWrite(), args);
+        System.exit(exitCode);
+    }
+}

Modified: incubator/hcatalog/trunk/storage-drivers/hbase/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-drivers/hbase/ivy/libraries.properties?rev=1235121&r1=1235120&r2=1235121&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-drivers/hbase/ivy/libraries.properties (original)
+++ incubator/hcatalog/trunk/storage-drivers/hbase/ivy/libraries.properties Tue Jan 24 03:33:44
2012
@@ -16,6 +16,6 @@
 junit.version=3.8.1
 ivy.version=2.2.0
 rats-lib.version=0.5.1
-hbase.version=0.90.3
+hbase.version=0.90.5
 zookeeper.version=3.4.0
 thrift.version=0.7.0



Mime
View raw message