incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1293514 - in /incubator/hcatalog/trunk: ./ bin/ src/test/e2e/hcatalog/ src/test/e2e/hcatalog/conf/ src/test/e2e/hcatalog/drivers/ src/test/e2e/hcatalog/tests/ src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/
Date Sat, 25 Feb 2012 01:58:07 GMT
Author: gates
Date: Sat Feb 25 01:58:06 2012
New Revision: 1293514

URL: http://svn.apache.org/viewvc?rev=1293514&view=rev
Log:
HCATALOG-257 e2e harness not working properly after file location change

Added:
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java
Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/bin/hcat-config.sh
    incubator/hcatalog/trunk/build.xml
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Sat Feb 25 01:58:06 2012
@@ -50,6 +50,8 @@ Trunk (unreleased changes)
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-257 e2e harness not working properly after file location change (gates)
+
   HCAT-270 HCatRecord SerDe does not handle List<Map> or List<List> or List<lazy
primitive> correctly (khorgath via gates)
 
   HCAT-248 Set execute.setugi conf variable to true by default (hashutosh via gates)

Modified: incubator/hcatalog/trunk/bin/hcat-config.sh
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/bin/hcat-config.sh?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/bin/hcat-config.sh (original)
+++ incubator/hcatalog/trunk/bin/hcat-config.sh Sat Feb 25 01:58:06 2012
@@ -48,8 +48,8 @@ then
 fi
  
 # Allow alternate conf dir location.
-if [ -e "${HCAT_PREFIX}/conf/hcat-env.sh" ]; then
-  DEFAULT_CONF_DIR=${HCAT_PREFIX}/"conf"
+if [ -e "${HCAT_PREFIX}/etc/hcatalog/hcat-env.sh" ]; then
+  DEFAULT_CONF_DIR=${HCAT_PREFIX}/"etc/hcatalog"
 else
   DEFAULT_CONF_DIR="/etc/hcatalog"
 fi

Modified: incubator/hcatalog/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build.xml?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build.xml (original)
+++ incubator/hcatalog/trunk/build.xml Sat Feb 25 01:58:06 2012
@@ -71,6 +71,7 @@
   <property name="test.output" value="no"/>
   <property name="hive.conf.dir" value="${hive.root}/conf"/>
   <property name="test.warehouse.dir" value="/tmp/hcat_junit_warehouse"/>
+  <property name="test.excludes" value="${test.src.dir}/e2e/**"/>
 
   <!-- e2e test properties -->
   <property name="test.e2e.dir" value="${basedir}/src/test/e2e/hcatalog"/>
@@ -299,7 +300,7 @@
   -->
   <!-- Build HCatalog test files -->
   <target name="compile-test" depends="jar">
-    <javac encoding="${build.encoding}" srcdir="${test.src.dir}" excludes="${excludes}"
+    <javac encoding="${build.encoding}" srcdir="${test.src.dir}" excludes="${test.excludes}"
         includes="**/*.java" destdir="${test.build.classes}" debug="${javac.debug}"
         optimize="${javac.optimize}" target="${javac.version}"
         source="${javac.version}" deprecation="${javac.deprecation}"
@@ -666,6 +667,7 @@
                 <exclude name="ivy/*.jar"/>
                 <include name="lib/**"/>
                 <include name="license/**"/>
+                <include name="shims/**"/>
                 <include name="src/**"/>
                 <include name="storage-drivers/**"/>
                 <include name="*.txt"/>

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml Sat Feb 25 01:58:06 2012
@@ -19,7 +19,7 @@
 
   <property name="hcat.dir" value="${basedir}/../../../../"/>
   <property name="hive.dir" value="${basedir}/../../../../hive/external/"/>
-  <property name="hcat.jar" value="${hcat.dir}/build/hcatalog/hcatalog-0.3.0-dev.jar"/>
+  <property name="hcat.jar" value="${hcat.dir}/build/hcatalog/hcatalog-${hcatalog.version}.jar"/>
   
   <path id="hcat.deps">
     <fileset dir="${hive.dir}/lib/">

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf Sat Feb 25 01:58:06 2012
@@ -62,7 +62,7 @@ $cfg = {
     , 'hcat_data_dir'    => '/user/hcat/tests/data'
     , 'hivehome'          => $ENV{'PH_HIVE_HOME'}
     , 'hcathome'          => $ENV{'HCAT_INSTALL_DIR'}
-    , 'hcatalog.jar' => "$ENV{HCAT_JAR},$ENV{HIVE_ROOT}/build/dist/lib/hive-serde-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-exec-$hive_version.jar,$ENV{PIG_HOME}/pig-withouthadoop.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-metastore-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/libfb303-0.7.0.jar,$ENV{HIVE_ROOT}/build/dist/lib/jdo2-api-2.3-ec.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar,$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar"
+    , 'hcatalog.jar' => "$ENV{HCAT_JAR},$ENV{HIVE_ROOT}/build/dist/lib/hive-serde-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-exec-$hive_version.jar,$ENV{PIG_HOME}/pig-0.9.2-withouthadoop.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-metastore-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/libfb303-0.7.0.jar,$ENV{HIVE_ROOT}/build/dist/lib/jdo2-api-2.3-ec.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar,$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar"
 
     #PIG
     , 'testconfigpath'   => "$ENV{PH_CLUSTER}"
@@ -72,7 +72,7 @@ $cfg = {
     , 'pigpath'          => "$ENV{PIG_HOME}"
     , 'pigjar'           => "$ENV{PIG_JAR}" # Pig jar that doesn't have Antlr
     , 'oldpigpath'       => "$ENV{PH_OLDPIG}"
-    , 'additionaljars' =>  "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.3.0-dev.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar::$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog"
+    , 'additionaljars' =>  "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.4.0.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar::$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog"
 
     #HADOOP
     , 'hadoopHome'       => "$ENV{HCAT_ROOT}/lib"

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm Sat Feb 25
01:58:06 2012
@@ -336,10 +336,15 @@ sub runHadoop
 
     # Set HADOOP_CLASSPATH environment variable if provided
     if (defined($testCmd->{'hadoop_classpath'})) {
-        my $hadoop_classpath = $self->replaceParameters( $testCmd->{'hadoop_classpath'},
$outfile, $testCmd, $log );
-        my $cp = $testCmd->{'hcatalog.jar'};
+        #my $hadoop_classpath = $self->replaceParameters( $testCmd->{'hadoop_classpath'},
$outfile, $testCmd, $log );
+        # TODO This is a total mess.  Half the jars we need are specified in hcatalog.jar,
which is set in the default.conf file, and half are set in hadoop_classpath, which
+        # has to be passed on the command line.  And most of the jars are Hive jars, thus
they don't fit in the list of either hcatalog.jar or hadoop_classpath.  We need to
+        # make sense of this.
+        my $cp = $testCmd->{'hcatalog.jar'} . ":" . $testCmd->{'hadoop_classpath'};
         $cp =~ s/,/:/g;
         $cp .= ":" . Util::findPigWithoutHadoopJar($testCmd, $log);
+        # Add in the hcat config file
+        $cp .= ":" . $testCmd->{'hcathome'} . "/etc/hcatalog";
         $ENV{'HADOOP_CLASSPATH'} = $cp;
     }
 

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf Sat Feb 25 01:58:06 2012
@@ -23,7 +23,7 @@ $cfg = {
 				,'hadoop' => q\
 jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.SimpleRead -libjars :HCAT_JAR: :THRIFTSERVER:
studenttab10k :OUTPATH:
 \,
-                                ,'sql' => q\select name, gpa from studenttab10k;\
+                                ,'sql' => q\select name, age from studenttab10k;\
                                 ,'floatpostprocess' => 1
                                 ,'delimiter' => '	'
 				}, 
@@ -42,15 +42,46 @@ jar :FUNCPATH:/testudf.jar org.apache.hc
                                 {
                                  'num' => 3
                                 ,'hcat_prep'=>q\drop table if exists hadoop_checkin_3;
-create table hadoop_checkin_3 (name string, cnt int) STORED AS TEXTFILE;\
+create table hadoop_checkin_3 (age int, cnt int) STORED AS TEXTFILE;\
                                 ,'hadoop' => q\
 jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.GroupByAge -libjars :HCAT_JAR: :THRIFTSERVER:
studenttab10k hadoop_checkin_3
 \,
                                 ,'result_table' => 'hadoop_checkin_3'
                                 ,'sql' => q\select age, count(*) from studenttab10k group
by age;\
+                                },{
+                                 # Read from a partitioned table
+                                 'num' => 4
+                                ,'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.SimpleRead -libjars :HCAT_JAR: :THRIFTSERVER:
studentparttab30k :OUTPATH:
+\,
+                                ,'sql' => q\select name, age from studentparttab30k;\
                                 ,'floatpostprocess' => 1
                                 ,'delimiter' => '	'
-                                },
+                                },{
+                                 # Write a single partition to a partitioned table
+                                 'num' => 5
+                                ,'hcat_prep'=>q\drop table if exists hadoop_checkin_5;
+create table hadoop_checkin_5 (name string, age int) partitioned by (ds string) STORED AS
TEXTFILE;\
+                                ,'hadoop' => q?
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.WriteTextPartitioned -libjars :HCAT_JAR:
:THRIFTSERVER: studentparttab30k hadoop_checkin_5 ds=\"20110924\"
+?,
+                                ,'result_table' => 'hadoop_checkin_5'
+                                ,'sql' => q\select name, age, ds from studentparttab30k
where ds='20110924';\
+                                ,'floatpostprocess' => 1
+                                ,'delimiter' => '	'
+                                }, {
+                                 # Write a multiple partitions to a partitioned table
+                                 'num' => 6
+                                ,'hcat_prep'=>q\drop table if exists hadoop_checkin_6;
+create table hadoop_checkin_6 (name string, age int) partitioned by (ds string) STORED AS
TEXTFILE;\
+                                ,'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.WriteTextPartitioned -libjars :HCAT_JAR:
:THRIFTSERVER: studentparttab30k hadoop_checkin_6
+\,
+                                ,'result_table' => 'hadoop_checkin_6'
+                                ,'sql' => q\select name, age, ds from studentparttab30k;\
+                                ,'floatpostprocess' => 1
+                                ,'delimiter' => '	'
+                                }
 			],
  		}, # end g
                 {
@@ -122,10 +153,8 @@ create table hadoop_write_2(
             d double,
             m map<string, string>,
             bb array<struct<a: int, b: string>>)
-            STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat'
-            INPUTDRIVER 'org.apache.hcatalog.pig.drivers.LoadFuncBasedInputDriver' OUTPUTDRIVER
'org.apache.hcatalog.pig.drivers.StoreFuncBasedOutputDriver'
-            TBLPROPERTIES ('hcat.pig.loader'='org.apache.pig.builtin.JsonLoader', 'hcat.pig.storer'='org.apache.pig.builtin.JsonStorage',
'hcat.pig.loader.args'=
-'s:chararray, i:int, d:double, m:map[chararray], bb:{t:(a:int, b:chararray)}', 'hcat.pig.args.delimiter'='
');\
+            row format serde 'org/apache/hcatalog/data/JsonSerDe'
+            stored as textfile;\
                                 ,'hadoop' => q\
 jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.WriteJson -libjars :HCAT_JAR: :THRIFTSERVER:
all100kjson hadoop_write_2
 \,

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf Sat Feb 25 01:58:06 2012
@@ -130,7 +130,7 @@ show databases;\
                                 ,'hcat' => q\
 show tables;\,
                                 ,'rc'   => 0
-                                ,'expected_out_regex' => 'studenttab10k'
+                                ,'expected_out_regex' => 'hcat_createtable_1'
                                 },
                         ],
                 }, # end g

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf Sat Feb 25 01:58:06 2012
@@ -77,7 +77,33 @@ store c into ':OUTPATH:';\,
 				,'sql'   => [ 'select * from studenttab10k where age<=40;', 'select * from studenttab10k
where age>40;']
                                 ,'floatpostprocess' => 1
                                 ,'delimiter' => '	'
-				}, 
+				},
+                {
+				 'num' => 6
+                                ,'hcat_prep'=>q\drop table if exists pig_checkin_6;
+create table pig_checkin_6 (name string, age int) partitioned by (ds string) STORED AS TEXTFILE;\
+                                ,'pig' => q\a = load 'studentparttab30k' using org.apache.hcatalog.pig.HCatLoader();
+b = filter a by ds == '20110924';
+c = foreach b generate name, age;
+store c into 'pig_checkin_6' using org.apache.hcatalog.pig.HCatStorer('ds=20110924');\,
+#dump a;\,
+                                ,'result_table' => 'pig_checkin_6',
+				,'sql'   => "select name, age, ds from studentparttab30k where ds='20110924';",
+                                ,'floatpostprocess' => 1
+                                ,'delimiter' => '	'
+				},
+                {
+				 'num' => 7
+                                ,'hcat_prep'=>q\drop table if exists pig_checkin_7;
+create table pig_checkin_7 (name string, age int) partitioned by (ds string) STORED AS TEXTFILE;\
+                                ,'pig' => q\a = load 'studentparttab30k' using org.apache.hcatalog.pig.HCatLoader();
+b = foreach a generate name, age, ds;
+store b into 'pig_checkin_7' using org.apache.hcatalog.pig.HCatStorer();\,
+                                ,'result_table' => 'pig_checkin_7',
+				,'sql'   => "select name, age, ds from studentparttab30k;",
+                                ,'floatpostprocess' => 1
+                                ,'delimiter' => '	'
+				} 
 
 			],
  		}, # end g

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java
(original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java
Sat Feb 25 01:58:06 2012
@@ -55,8 +55,8 @@ public class ReadText extends Configured
   public static class Map
        extends Mapper<WritableComparable, HCatRecord, IntWritable, HCatRecord>{
       
-      int t;
-      int si;
+      byte t;
+      short si;
       int i;
       long b;
       float f;
@@ -68,8 +68,8 @@ public class ReadText extends Configured
           org.apache.hadoop.mapreduce.Mapper<WritableComparable,HCatRecord,
           IntWritable,HCatRecord>.Context context) 
     throws IOException ,InterruptedException {
-        t = (Integer)value.get(0);
-        si = (Integer)value.get(1);
+        t = (Byte)value.get(0);
+        si = (Short)value.get(1);
         i = (Integer)value.get(2);
         b = (Long)value.get(3);
         f = (Float)value.get(4);

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
(original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
Sat Feb 25 01:58:06 2012
@@ -23,7 +23,7 @@ import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapreduce.Job;
@@ -54,7 +54,7 @@ public class SimpleRead extends Configur
     private static final String TAB = "\t";
     
   public static class Map
-       extends Mapper<WritableComparable, HCatRecord, Text, DoubleWritable>{
+       extends Mapper<WritableComparable, HCatRecord, Text, IntWritable>{
       
       String name;
       int age;
@@ -63,12 +63,13 @@ public class SimpleRead extends Configur
     @Override
   protected void map(WritableComparable key, HCatRecord value, 
           org.apache.hadoop.mapreduce.Mapper<WritableComparable,HCatRecord,
-          Text,DoubleWritable>.Context context) 
+          Text,IntWritable>.Context context) 
     throws IOException ,InterruptedException {
         name = (String) value.get(0);
+System.out.println(name);
         age = (Integer) value.get(1);
         gpa = (Double) value.get(2);
-        context.write(new Text(name), new DoubleWritable(gpa));
+        context.write(new Text(name), new IntWritable(age));
 
     }
   }
@@ -95,7 +96,7 @@ public class SimpleRead extends Configur
     job.setJarByClass(SimpleRead.class);
     job.setMapperClass(Map.class);
     job.setOutputKeyClass(Text.class);
-    job.setOutputValueClass(DoubleWritable.class);
+    job.setOutputValueClass(IntWritable.class);
     FileOutputFormat.setOutputPath(job, new Path(outputDir));
     return (job.waitForCompletion(true) ? 0 : 1);
   }

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java?rev=1293514&r1=1293513&r2=1293514&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java
(original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java
Sat Feb 25 01:58:06 2012
@@ -53,8 +53,8 @@ public class WriteText extends Configure
     public static class Map extends
             Mapper<WritableComparable, HCatRecord, WritableComparable, HCatRecord>
{
 
-        int t;
-        int si;
+        byte t;
+        short si;
         int i;
         long b;
         float f;
@@ -67,8 +67,8 @@ public class WriteText extends Configure
                 HCatRecord value,
                 org.apache.hadoop.mapreduce.Mapper<WritableComparable, HCatRecord, WritableComparable,
HCatRecord>.Context context)
                 throws IOException, InterruptedException {
-            t = (Integer)value.get(0);
-            si = (Integer)value.get(1);
+            t = (Byte)value.get(0);
+            si = (Short)value.get(1);
             i = (Integer)value.get(2);
             b = (Long)value.get(3);
             f = (Float)value.get(4);

Added: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java?rev=1293514&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java
(added)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java
Sat Feb 25 01:58:06 2012
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.utils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hcatalog.common.HCatConstants;
+import org.apache.hcatalog.data.DefaultHCatRecord;
+import org.apache.hcatalog.data.HCatRecord;
+import org.apache.hcatalog.data.schema.HCatSchema;
+import org.apache.hcatalog.data.schema.HCatFieldSchema;
+import org.apache.hcatalog.mapreduce.HCatInputFormat;
+import org.apache.hcatalog.mapreduce.HCatOutputFormat;
+import org.apache.hcatalog.mapreduce.InputJobInfo;
+import org.apache.hcatalog.mapreduce.OutputJobInfo;
+
+/**
+ * This is a map reduce test for testing hcat writing to partitioned tables.
+ * table. It performs a group by on the first column and a SUM operation on the
+ * other columns. This is to simulate a typical operation in a map reduce
+ * program to test that hcat hands the right data to the map reduce program
+ * 
+ * Usage: hadoop jar org.apache.hcatalog.utils.HBaseReadWrite -libjars
+ * &lt;hcat_jar&gt; * &lt;serveruri&gt; &lt;input_tablename&gt; &lt;output_tablename&gt;
[filter]
+ * If filter is given it will be provided as the partition to write to.
+ */
+public class WriteTextPartitioned extends Configured implements Tool {
+
+    static String filter = null;
+
+    public static class Map extends
+            Mapper<WritableComparable, HCatRecord, WritableComparable, HCatRecord>
{
+
+        @Override
+        protected void map(
+                WritableComparable key,
+                HCatRecord value,
+                org.apache.hadoop.mapreduce.Mapper<WritableComparable, HCatRecord, WritableComparable,
HCatRecord>.Context context)
+                throws IOException, InterruptedException {
+            String name = (String)value.get(0);
+            int age = (Integer)value.get(1);
+            String ds = (String)value.get(3);
+            
+            HCatRecord record = (filter == null ? new DefaultHCatRecord(3) : new DefaultHCatRecord(2));
+            record.set(0, name);
+            record.set(1, age);
+            if (filter == null) record.set(2, ds);
+            
+            context.write(null, record);
+
+        }
+    }
+
+    public int run(String[] args) throws Exception {
+        Configuration conf = getConf();
+        args = new GenericOptionsParser(conf, args).getRemainingArgs();
+
+        String serverUri = args[0];
+        String inputTableName = args[1];
+        String outputTableName = args[2];
+        if (args.length > 3) filter = args[3];
+        String dbName = null;
+
+        String principalID = System
+                .getProperty(HCatConstants.HCAT_METASTORE_PRINCIPAL);
+        if (principalID != null)
+            conf.set(HCatConstants.HCAT_METASTORE_PRINCIPAL, principalID);
+        Job job = new Job(conf, "WriteTextPartitioned");
+        HCatInputFormat.setInput(job, InputJobInfo.create(dbName,
+                inputTableName, filter, serverUri, principalID));
+        // initialize HCatOutputFormat
+
+        job.setInputFormatClass(HCatInputFormat.class);
+        job.setJarByClass(WriteTextPartitioned.class);
+        job.setMapperClass(Map.class);
+        job.setOutputKeyClass(WritableComparable.class);
+        job.setOutputValueClass(DefaultHCatRecord.class);
+        job.setNumReduceTasks(0);
+
+        java.util.Map<String, String> partitionVals = null;
+        if (filter != null) {
+            String[] s = filter.split("=");
+            String val = s[1].replace('"', ' ').trim();
+            partitionVals = new HashMap<String, String>(1);
+            partitionVals.put(s[0], val);
+        }
+        HCatOutputFormat.setOutput(job, OutputJobInfo.create(dbName,
+                outputTableName, partitionVals));
+        HCatSchema s = HCatInputFormat.getTableSchema(job);
+        // Build the schema for this table, which is slightly different than the
+        // schema for the input table
+        List<HCatFieldSchema> fss = new ArrayList<HCatFieldSchema>(3);
+        fss.add(s.get(0));
+        fss.add(s.get(1));
+        fss.add(s.get(3));
+        HCatOutputFormat.setSchema(job, new HCatSchema(fss));
+        job.setOutputFormatClass(HCatOutputFormat.class);
+        return (job.waitForCompletion(true) ? 0 : 1);
+    }
+
+    public static void main(String[] args) throws Exception {
+        int exitCode = ToolRunner.run(new WriteTextPartitioned(), args);
+        System.exit(exitCode);
+    }
+}



Mime
View raw message