From hcatalog-commits-return-686-apmail-incubator-hcatalog-commits-archive=incubator.apache.org@incubator.apache.org Sat Feb 25 01:58:35 2012 Return-Path: X-Original-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A48E89560 for ; Sat, 25 Feb 2012 01:58:35 +0000 (UTC) Received: (qmail 47514 invoked by uid 500); 25 Feb 2012 01:58:35 -0000 Delivered-To: apmail-incubator-hcatalog-commits-archive@incubator.apache.org Received: (qmail 47464 invoked by uid 500); 25 Feb 2012 01:58:35 -0000 Mailing-List: contact hcatalog-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hcatalog-dev@incubator.apache.org Delivered-To: mailing list hcatalog-commits@incubator.apache.org Received: (qmail 47456 invoked by uid 99); 25 Feb 2012 01:58:35 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 25 Feb 2012 01:58:35 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 25 Feb 2012 01:58:30 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 3A8F72388900; Sat, 25 Feb 2012 01:58:08 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1293514 - in /incubator/hcatalog/trunk: ./ bin/ src/test/e2e/hcatalog/ src/test/e2e/hcatalog/conf/ src/test/e2e/hcatalog/drivers/ src/test/e2e/hcatalog/tests/ src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ Date: Sat, 25 Feb 2012 01:58:07 -0000 To: hcatalog-commits@incubator.apache.org From: gates@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120225015808.3A8F72388900@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: gates Date: Sat Feb 25 01:58:06 2012 New Revision: 1293514 URL: http://svn.apache.org/viewvc?rev=1293514&view=rev Log: HCATALOG-257 e2e harness not working properly after file location change Added: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java Modified: incubator/hcatalog/trunk/CHANGES.txt incubator/hcatalog/trunk/bin/hcat-config.sh incubator/hcatalog/trunk/build.xml incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java Modified: incubator/hcatalog/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/CHANGES.txt (original) +++ incubator/hcatalog/trunk/CHANGES.txt Sat Feb 25 01:58:06 2012 @@ -50,6 +50,8 @@ Trunk (unreleased changes) OPTIMIZATIONS BUG FIXES + HCAT-257 e2e harness not working properly after file location change (gates) + HCAT-270 HCatRecord SerDe does not handle List or List or List correctly (khorgath via gates) HCAT-248 Set execute.setugi conf variable to true by default (hashutosh via gates) Modified: incubator/hcatalog/trunk/bin/hcat-config.sh URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/bin/hcat-config.sh?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/bin/hcat-config.sh (original) +++ incubator/hcatalog/trunk/bin/hcat-config.sh Sat Feb 25 01:58:06 2012 @@ -48,8 +48,8 @@ then fi # Allow alternate conf dir location. -if [ -e "${HCAT_PREFIX}/conf/hcat-env.sh" ]; then - DEFAULT_CONF_DIR=${HCAT_PREFIX}/"conf" +if [ -e "${HCAT_PREFIX}/etc/hcatalog/hcat-env.sh" ]; then + DEFAULT_CONF_DIR=${HCAT_PREFIX}/"etc/hcatalog" else DEFAULT_CONF_DIR="/etc/hcatalog" fi Modified: incubator/hcatalog/trunk/build.xml URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build.xml?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/build.xml (original) +++ incubator/hcatalog/trunk/build.xml Sat Feb 25 01:58:06 2012 @@ -71,6 +71,7 @@ + @@ -299,7 +300,7 @@ --> - + Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml Sat Feb 25 01:58:06 2012 @@ -19,7 +19,7 @@ - + Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf Sat Feb 25 01:58:06 2012 @@ -62,7 +62,7 @@ $cfg = { , 'hcat_data_dir' => '/user/hcat/tests/data' , 'hivehome' => $ENV{'PH_HIVE_HOME'} , 'hcathome' => $ENV{'HCAT_INSTALL_DIR'} - , 'hcatalog.jar' => "$ENV{HCAT_JAR},$ENV{HIVE_ROOT}/build/dist/lib/hive-serde-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-exec-$hive_version.jar,$ENV{PIG_HOME}/pig-withouthadoop.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-metastore-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/libfb303-0.7.0.jar,$ENV{HIVE_ROOT}/build/dist/lib/jdo2-api-2.3-ec.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar,$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar" + , 'hcatalog.jar' => "$ENV{HCAT_JAR},$ENV{HIVE_ROOT}/build/dist/lib/hive-serde-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-exec-$hive_version.jar,$ENV{PIG_HOME}/pig-0.9.2-withouthadoop.jar,$ENV{HIVE_ROOT}/build/dist/lib/hive-metastore-$hive_version.jar,$ENV{HIVE_ROOT}/build/dist/lib/libfb303-0.7.0.jar,$ENV{HIVE_ROOT}/build/dist/lib/jdo2-api-2.3-ec.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar,$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar,$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar" #PIG , 'testconfigpath' => "$ENV{PH_CLUSTER}" @@ -72,7 +72,7 @@ $cfg = { , 'pigpath' => "$ENV{PIG_HOME}" , 'pigjar' => "$ENV{PIG_JAR}" # Pig jar that doesn't have Antlr , 'oldpigpath' => "$ENV{PH_OLDPIG}" - , 'additionaljars' => "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.3.0-dev.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar::$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog" + , 'additionaljars' => "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.4.0.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-$hive_version.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar::$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/hbase-0.90.5.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/ivy/lib/hbase-storage-driver/zookeeper-3.4.0.jar:$ENV{HCAT_ROOT}/storage-drivers/hbase/build/hbase-storage-driver/hbase-storage-driver-0.1.0.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-hbase-handler-$hive_version.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog" #HADOOP , 'hadoopHome' => "$ENV{HCAT_ROOT}/lib" Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm Sat Feb 25 01:58:06 2012 @@ -336,10 +336,15 @@ sub runHadoop # Set HADOOP_CLASSPATH environment variable if provided if (defined($testCmd->{'hadoop_classpath'})) { - my $hadoop_classpath = $self->replaceParameters( $testCmd->{'hadoop_classpath'}, $outfile, $testCmd, $log ); - my $cp = $testCmd->{'hcatalog.jar'}; + #my $hadoop_classpath = $self->replaceParameters( $testCmd->{'hadoop_classpath'}, $outfile, $testCmd, $log ); + # TODO This is a total mess. Half the jars we need are specified in hcatalog.jar, which is set in the default.conf file, and half are set in hadoop_classpath, which + # has to be passed on the command line. And most of the jars are Hive jars, thus they don't fit in the list of either hcatalog.jar or hadoop_classpath. We need to + # make sense of this. + my $cp = $testCmd->{'hcatalog.jar'} . ":" . $testCmd->{'hadoop_classpath'}; $cp =~ s/,/:/g; $cp .= ":" . Util::findPigWithoutHadoopJar($testCmd, $log); + # Add in the hcat config file + $cp .= ":" . $testCmd->{'hcathome'} . "/etc/hcatalog"; $ENV{'HADOOP_CLASSPATH'} = $cp; } Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf Sat Feb 25 01:58:06 2012 @@ -23,7 +23,7 @@ $cfg = { ,'hadoop' => q\ jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.SimpleRead -libjars :HCAT_JAR: :THRIFTSERVER: studenttab10k :OUTPATH: \, - ,'sql' => q\select name, gpa from studenttab10k;\ + ,'sql' => q\select name, age from studenttab10k;\ ,'floatpostprocess' => 1 ,'delimiter' => ' ' }, @@ -42,15 +42,46 @@ jar :FUNCPATH:/testudf.jar org.apache.hc { 'num' => 3 ,'hcat_prep'=>q\drop table if exists hadoop_checkin_3; -create table hadoop_checkin_3 (name string, cnt int) STORED AS TEXTFILE;\ +create table hadoop_checkin_3 (age int, cnt int) STORED AS TEXTFILE;\ ,'hadoop' => q\ jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.GroupByAge -libjars :HCAT_JAR: :THRIFTSERVER: studenttab10k hadoop_checkin_3 \, ,'result_table' => 'hadoop_checkin_3' ,'sql' => q\select age, count(*) from studenttab10k group by age;\ + },{ + # Read from a partitioned table + 'num' => 4 + ,'hadoop' => q\ +jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.SimpleRead -libjars :HCAT_JAR: :THRIFTSERVER: studentparttab30k :OUTPATH: +\, + ,'sql' => q\select name, age from studentparttab30k;\ ,'floatpostprocess' => 1 ,'delimiter' => ' ' - }, + },{ + # Write a single partition to a partitioned table + 'num' => 5 + ,'hcat_prep'=>q\drop table if exists hadoop_checkin_5; +create table hadoop_checkin_5 (name string, age int) partitioned by (ds string) STORED AS TEXTFILE;\ + ,'hadoop' => q? +jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.WriteTextPartitioned -libjars :HCAT_JAR: :THRIFTSERVER: studentparttab30k hadoop_checkin_5 ds=\"20110924\" +?, + ,'result_table' => 'hadoop_checkin_5' + ,'sql' => q\select name, age, ds from studentparttab30k where ds='20110924';\ + ,'floatpostprocess' => 1 + ,'delimiter' => ' ' + }, { + # Write a multiple partitions to a partitioned table + 'num' => 6 + ,'hcat_prep'=>q\drop table if exists hadoop_checkin_6; +create table hadoop_checkin_6 (name string, age int) partitioned by (ds string) STORED AS TEXTFILE;\ + ,'hadoop' => q\ +jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.WriteTextPartitioned -libjars :HCAT_JAR: :THRIFTSERVER: studentparttab30k hadoop_checkin_6 +\, + ,'result_table' => 'hadoop_checkin_6' + ,'sql' => q\select name, age, ds from studentparttab30k;\ + ,'floatpostprocess' => 1 + ,'delimiter' => ' ' + } ], }, # end g { @@ -122,10 +153,8 @@ create table hadoop_write_2( d double, m map, bb array>) - STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat' - INPUTDRIVER 'org.apache.hcatalog.pig.drivers.LoadFuncBasedInputDriver' OUTPUTDRIVER 'org.apache.hcatalog.pig.drivers.StoreFuncBasedOutputDriver' - TBLPROPERTIES ('hcat.pig.loader'='org.apache.pig.builtin.JsonLoader', 'hcat.pig.storer'='org.apache.pig.builtin.JsonStorage', 'hcat.pig.loader.args'= -'s:chararray, i:int, d:double, m:map[chararray], bb:{t:(a:int, b:chararray)}', 'hcat.pig.args.delimiter'=' ');\ + row format serde 'org/apache/hcatalog/data/JsonSerDe' + stored as textfile;\ ,'hadoop' => q\ jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.WriteJson -libjars :HCAT_JAR: :THRIFTSERVER: all100kjson hadoop_write_2 \, Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf Sat Feb 25 01:58:06 2012 @@ -130,7 +130,7 @@ show databases;\ ,'hcat' => q\ show tables;\, ,'rc' => 0 - ,'expected_out_regex' => 'studenttab10k' + ,'expected_out_regex' => 'hcat_createtable_1' }, ], }, # end g Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf Sat Feb 25 01:58:06 2012 @@ -77,7 +77,33 @@ store c into ':OUTPATH:';\, ,'sql' => [ 'select * from studenttab10k where age<=40;', 'select * from studenttab10k where age>40;'] ,'floatpostprocess' => 1 ,'delimiter' => ' ' - }, + }, + { + 'num' => 6 + ,'hcat_prep'=>q\drop table if exists pig_checkin_6; +create table pig_checkin_6 (name string, age int) partitioned by (ds string) STORED AS TEXTFILE;\ + ,'pig' => q\a = load 'studentparttab30k' using org.apache.hcatalog.pig.HCatLoader(); +b = filter a by ds == '20110924'; +c = foreach b generate name, age; +store c into 'pig_checkin_6' using org.apache.hcatalog.pig.HCatStorer('ds=20110924');\, +#dump a;\, + ,'result_table' => 'pig_checkin_6', + ,'sql' => "select name, age, ds from studentparttab30k where ds='20110924';", + ,'floatpostprocess' => 1 + ,'delimiter' => ' ' + }, + { + 'num' => 7 + ,'hcat_prep'=>q\drop table if exists pig_checkin_7; +create table pig_checkin_7 (name string, age int) partitioned by (ds string) STORED AS TEXTFILE;\ + ,'pig' => q\a = load 'studentparttab30k' using org.apache.hcatalog.pig.HCatLoader(); +b = foreach a generate name, age, ds; +store b into 'pig_checkin_7' using org.apache.hcatalog.pig.HCatStorer();\, + ,'result_table' => 'pig_checkin_7', + ,'sql' => "select name, age, ds from studentparttab30k;", + ,'floatpostprocess' => 1 + ,'delimiter' => ' ' + } ], }, # end g Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadText.java Sat Feb 25 01:58:06 2012 @@ -55,8 +55,8 @@ public class ReadText extends Configured public static class Map extends Mapper{ - int t; - int si; + byte t; + short si; int i; long b; float f; @@ -68,8 +68,8 @@ public class ReadText extends Configured org.apache.hadoop.mapreduce.Mapper.Context context) throws IOException ,InterruptedException { - t = (Integer)value.get(0); - si = (Integer)value.get(1); + t = (Byte)value.get(0); + si = (Short)value.get(1); i = (Integer)value.get(2); b = (Long)value.get(3); f = (Float)value.get(4); Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java Sat Feb 25 01:58:06 2012 @@ -23,7 +23,7 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; @@ -54,7 +54,7 @@ public class SimpleRead extends Configur private static final String TAB = "\t"; public static class Map - extends Mapper{ + extends Mapper{ String name; int age; @@ -63,12 +63,13 @@ public class SimpleRead extends Configur @Override protected void map(WritableComparable key, HCatRecord value, org.apache.hadoop.mapreduce.Mapper.Context context) + Text,IntWritable>.Context context) throws IOException ,InterruptedException { name = (String) value.get(0); +System.out.println(name); age = (Integer) value.get(1); gpa = (Double) value.get(2); - context.write(new Text(name), new DoubleWritable(gpa)); + context.write(new Text(name), new IntWritable(age)); } } @@ -95,7 +96,7 @@ public class SimpleRead extends Configur job.setJarByClass(SimpleRead.class); job.setMapperClass(Map.class); job.setOutputKeyClass(Text.class); - job.setOutputValueClass(DoubleWritable.class); + job.setOutputValueClass(IntWritable.class); FileOutputFormat.setOutputPath(job, new Path(outputDir)); return (job.waitForCompletion(true) ? 0 : 1); } Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java?rev=1293514&r1=1293513&r2=1293514&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java (original) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteText.java Sat Feb 25 01:58:06 2012 @@ -53,8 +53,8 @@ public class WriteText extends Configure public static class Map extends Mapper { - int t; - int si; + byte t; + short si; int i; long b; float f; @@ -67,8 +67,8 @@ public class WriteText extends Configure HCatRecord value, org.apache.hadoop.mapreduce.Mapper.Context context) throws IOException, InterruptedException { - t = (Integer)value.get(0); - si = (Integer)value.get(1); + t = (Byte)value.get(0); + si = (Short)value.get(1); i = (Integer)value.get(2); b = (Long)value.get(3); f = (Float)value.get(4); Added: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java?rev=1293514&view=auto ============================================================================== --- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java (added) +++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteTextPartitioned.java Sat Feb 25 01:58:06 2012 @@ -0,0 +1,133 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hcatalog.utils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hcatalog.common.HCatConstants; +import org.apache.hcatalog.data.DefaultHCatRecord; +import org.apache.hcatalog.data.HCatRecord; +import org.apache.hcatalog.data.schema.HCatSchema; +import org.apache.hcatalog.data.schema.HCatFieldSchema; +import org.apache.hcatalog.mapreduce.HCatInputFormat; +import org.apache.hcatalog.mapreduce.HCatOutputFormat; +import org.apache.hcatalog.mapreduce.InputJobInfo; +import org.apache.hcatalog.mapreduce.OutputJobInfo; + +/** + * This is a map reduce test for testing hcat writing to partitioned tables. + * table. It performs a group by on the first column and a SUM operation on the + * other columns. This is to simulate a typical operation in a map reduce + * program to test that hcat hands the right data to the map reduce program + * + * Usage: hadoop jar org.apache.hcatalog.utils.HBaseReadWrite -libjars + * <hcat_jar> * <serveruri> <input_tablename> <output_tablename> [filter] + * If filter is given it will be provided as the partition to write to. + */ +public class WriteTextPartitioned extends Configured implements Tool { + + static String filter = null; + + public static class Map extends + Mapper { + + @Override + protected void map( + WritableComparable key, + HCatRecord value, + org.apache.hadoop.mapreduce.Mapper.Context context) + throws IOException, InterruptedException { + String name = (String)value.get(0); + int age = (Integer)value.get(1); + String ds = (String)value.get(3); + + HCatRecord record = (filter == null ? new DefaultHCatRecord(3) : new DefaultHCatRecord(2)); + record.set(0, name); + record.set(1, age); + if (filter == null) record.set(2, ds); + + context.write(null, record); + + } + } + + public int run(String[] args) throws Exception { + Configuration conf = getConf(); + args = new GenericOptionsParser(conf, args).getRemainingArgs(); + + String serverUri = args[0]; + String inputTableName = args[1]; + String outputTableName = args[2]; + if (args.length > 3) filter = args[3]; + String dbName = null; + + String principalID = System + .getProperty(HCatConstants.HCAT_METASTORE_PRINCIPAL); + if (principalID != null) + conf.set(HCatConstants.HCAT_METASTORE_PRINCIPAL, principalID); + Job job = new Job(conf, "WriteTextPartitioned"); + HCatInputFormat.setInput(job, InputJobInfo.create(dbName, + inputTableName, filter, serverUri, principalID)); + // initialize HCatOutputFormat + + job.setInputFormatClass(HCatInputFormat.class); + job.setJarByClass(WriteTextPartitioned.class); + job.setMapperClass(Map.class); + job.setOutputKeyClass(WritableComparable.class); + job.setOutputValueClass(DefaultHCatRecord.class); + job.setNumReduceTasks(0); + + java.util.Map partitionVals = null; + if (filter != null) { + String[] s = filter.split("="); + String val = s[1].replace('"', ' ').trim(); + partitionVals = new HashMap(1); + partitionVals.put(s[0], val); + } + HCatOutputFormat.setOutput(job, OutputJobInfo.create(dbName, + outputTableName, partitionVals)); + HCatSchema s = HCatInputFormat.getTableSchema(job); + // Build the schema for this table, which is slightly different than the + // schema for the input table + List fss = new ArrayList(3); + fss.add(s.get(0)); + fss.add(s.get(1)); + fss.add(s.get(3)); + HCatOutputFormat.setSchema(job, new HCatSchema(fss)); + job.setOutputFormatClass(HCatOutputFormat.class); + return (job.waitForCompletion(true) ? 0 : 1); + } + + public static void main(String[] args) throws Exception { + int exitCode = ToolRunner.run(new WriteTextPartitioned(), args); + System.exit(exitCode); + } +}