incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1228478 - in /incubator/hcatalog/trunk: ./ src/test/e2e/hcatalog/ src/test/e2e/hcatalog/conf/ src/test/e2e/hcatalog/drivers/ src/test/e2e/hcatalog/tests/ src/test/e2e/hcatalog/udfs/java/ src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/...
Date Fri, 06 Jan 2012 22:46:11 GMT
Author: gates
Date: Fri Jan  6 22:46:10 2012
New Revision: 1228478

URL: http://svn.apache.org/viewvc?rev=1228478&view=rev
Log:
HCATALOG-198 A few issues keep e2e tests from running in an automated environment

Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/build.xml
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hive.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadRC.java
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteRC.java

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Fri Jan  6 22:46:10 2012
@@ -91,6 +91,8 @@ Trunk (unreleased changes)
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-198. A few issues keep e2e tests from running in an automated environment (gates)
+
   HCAT-197. Data generator for e2e assumes hadoop tarball installation (daijy via khorgath)
 
   HCAT-196. Remove dependency on pig.jar for hcatalog server (daijy via khorgath)

Modified: incubator/hcatalog/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build.xml?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build.xml (original)
+++ incubator/hcatalog/trunk/build.xml Fri Jan  6 22:46:10 2012
@@ -72,6 +72,9 @@
   <property name="hive.conf.dir" value="${hive.root}/conf"/>
   <property name="test.warehouse.dir" value="/tmp/hcat_junit_warehouse"/>
 
+  <!-- e2e test properties -->
+  <property name="test.e2e.dir" value="${basedir}/src/test/e2e/hcatalog"/>
+
   <!-- ivy properteis set here -->
   <property name="ivy.repo.dir" value="${user.home}/ivyrepo" />
   <property name="ivy.dir" location="ivy" />
@@ -363,6 +366,7 @@
   <!-- Clean up children -->
   <target name="clean" depends="clean-builds" description="Cleanup all build artifacts">
     <delete dir="${artifacts.dir}" />
+    <ant dir="${test.e2e.dir}" target="clean"/>
   </target>
  
   <!--
@@ -800,4 +804,22 @@
         </copy>
         <delete dir="${package.buildroot}" quiet="true" verbose="false"/>
     </target>
+
+    <!-- ================================================================== -->
+    <!-- End to end tests                                                   -->
+    <!-- ================================================================== -->
+
+    <target name="test-e2e" description="run end-to-end tests">
+        <ant dir="${test.e2e.dir}"/>
+    </target>
+
+    <target name="test-e2e-install" description="deploy end-to-end tests to existing cluster">
+        <ant dir="${test.e2e.dir}" target="install"/>
+    </target>
+
+    <target name="test-e2e-deploy" description="deploy end-to-end tests to existing cluster">
+        <ant dir="${test.e2e.dir}" target="deploy"/>
+    </target>
+
+
 </project>

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml Fri Jan  6 22:46:10 2012
@@ -197,7 +197,7 @@
       <env key="HIVE_HOME" value="${basedir}/../../../../hive/external/build/dist"/>
       <env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
       <env key="PIG_HOME" value="${harness.pig.home}"/>
-      <env key="PIG_ROOT" value="${harness.pig.home}"/>
+      <env key="PIG_JAR" value="${harness.pig.jar}"/> <!-- Pig jar without antlr
-->
       <arg line="${tests.to.run}"/>
       <arg value="${test.location}/tests/pig.conf"/>
       <arg value="${test.location}/tests/hive.conf"/>
@@ -232,6 +232,8 @@
       <env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
       <env key="HIVE_HOME" value="../../../../hive/external"/>
       <env key="PH_METASTORE_THRIFT" value="${harness.metastore.thrift}"/>
+      <env key="PIG_HOME" value="${harness.pig.home}"/>
+      <env key="PIG_JAR" value="${harness.pig.jar}"/> <!-- Pig jar without antlr
-->
       <arg value="-deploycfg"/>
       <arg value="${deploy.conf}"/>
       <arg value="${deploy.opt}"/>

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf Fri Jan  6 22:46:10 2012
@@ -1,19 +1,19 @@
-############################################################################           
-#  Licensed to the Apache Software Foundation (ASF) under one or more                  
-#  contributor license agreements.  See the NOTICE file distributed with               
-#  this work for additional information regarding copyright ownership.                 
-#  The ASF licenses this file to You under the Apache License, Version 2.0             
-#  (the "License"); you may not use this file except in compliance with                
-#  the License.  You may obtain a copy of the License at                               
-#                                                                                      
-#      http://www.apache.org/licenses/LICENSE-2.0                                      
-#                                                                                      
-#  Unless required by applicable law or agreed to in writing, software                 
-#  distributed under the License is distributed on an "AS IS" BASIS,                   
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.            
-#  See the License for the specific language governing permissions and                 
-#  limitations under the License.                                                      
-                                                                                       
+############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 my $me = `whoami`;
 chomp $me;
 
@@ -64,7 +64,8 @@ $cfg = {
     , 'hadoopbin'   => "$ENV{PH_CLUSTER_BIN}"
     , 'funcjarPath'      => "$ENV{PH_ROOT}/lib/java"
     , 'paramPath'        => "$ENV{PH_ROOT}/paramfiles"
-    , 'pigpath'          => "$ENV{PIG_ROOT}"
+    , 'pigpath'          => "$ENV{PIG_HOME}"
+    , 'pigjar'           => "$ENV{PIG_JAR}" # Pig jar that doesn't have Antlr
     , 'oldpigpath'       => "$ENV{PH_OLDPIG}"
     , 'additionaljars' =>  "$ENV{HCAT_ROOT}/build/hcatalog/hcatalog-0.3.0-dev.jar:$ENV{HCAT_ROOT}/hive/external/build/metastore/hive-metastore-0.9.0-SNAPSHOT.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libthrift.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/hive-exec-0.9.0-SNAPSHOT.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/libfb303.jar:$ENV{HCAT_ROOT}/hive/external/build/dist/lib/jdo2-api-2.3-ec.jar:$ENV{'HCAT_INSTALL_DIR'}/etc/hcatalog"
 

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/TestDriverHadoop.pm Fri Jan  6
22:46:10 2012
@@ -339,6 +339,7 @@ sub runHadoop
         my $hadoop_classpath = $self->replaceParameters( $testCmd->{'hadoop_classpath'},
$outfile, $testCmd, $log );
         my $cp = $testCmd->{'hcatalog.jar'};
         $cp =~ s/,/:/g;
+        $cp .= ":" . Util::findPigWithoutHadoopJar($testCmd, $log);
         $ENV{'HADOOP_CLASSPATH'} = $cp;
     }
 

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm Fri Jan  6 22:46:10 2012
@@ -1,21 +1,21 @@
 #!/usr/bin/env perl 
 
-############################################################################           
-#  Licensed to the Apache Software Foundation (ASF) under one or more                  
-#  contributor license agreements.  See the NOTICE file distributed with               
-#  this work for additional information regarding copyright ownership.                 
-#  The ASF licenses this file to You under the Apache License, Version 2.0             
-#  (the "License"); you may not use this file except in compliance with                
-#  the License.  You may obtain a copy of the License at                               
-#                                                                                      
-#      http://www.apache.org/licenses/LICENSE-2.0                                      
-#                                                                                      
-#  Unless required by applicable law or agreed to in writing, software                 
-#  distributed under the License is distributed on an "AS IS" BASIS,                   
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.            
-#  See the License for the specific language governing permissions and                 
-#  limitations under the License.                                                      
-                                                                                       
+############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 
 ###########################################################################
 # Class: Util
@@ -27,7 +27,6 @@
 package Util;
 
 use IPC::Run qw(run);
-use Log::Log4perl qw(:easy);
 
 sub prepareHCat
 {
@@ -237,6 +236,7 @@ sub runHCatCmdFromFile($$;$$$$)
 
     # unset HADOOP_CLASSPATH
     $ENV{'HADOOP_CLASSPATH'} = "";
+    $ENV{'HADOOP_CLASSPATH'} = $cfg->{'pigjar'};
 
     my @cmd;
     if (defined($sql)) {
@@ -309,137 +309,6 @@ sub runHadoopCmd($$$)
         die "Failed running " . join(" ", @cmd) . "\n";
 }
 
-##############################################################################
-#  Sub: localTime
-# 
-#  Returns:
-#  A string with the local time
-
-sub  localTime() {
-
-   my $retval = time();
-   
-   my $local_time = gmtime( $retval);
-
-   return $local_time;
-
-}
-
-##############################################################################
-#  Sub: formatedTime
-#  Returns the time with following format "$mday/$mon/$year $hour:$min:$sec $weekday[$wday]"
-#
-#  Returns:
-#  formated time
-
-sub  formatedTime() {
-
-   my @weekday = ("Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat");
-
-   my $retval = time();
-   
-   my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime(time);
-   $year = $year + 1900; 
-   $mon += 1;
-   return  "$mday/$mon/$year $hour:$min:$sec $weekday[$wday]\n";
-   
-
-}
-
-##############################################################################
-# Sub: execCmd
-# Records the command in the log and prints it to stdout before executing.
-#
-# Paramaters:
-# $log     - The log object 
-# $subName - The name of the subroutine the message originated at
-# $lineNo  - The line number  of the subroutine the message originated at
-# $cmd     - The command string to execute
-# $msg     - A string, the message to print
-# $level   - (optional)The logging level for the message: DEBUG, INFO, WARN, ERROR, FATAL
-#            defaults to DEBUG.
-#
-#  Returns:
-#  An array containing the  output from the executed command.
-#
-
-sub execCmd() {
-
-    my ( $log, $subName, $lineNo, $cmd, $level ) = @_ ;
-
-    my $count = @_;
-    my $thisSubName = (caller(0))[3];
-
-   #Check for errors in arguments
-   if ( $count < 4 ){
-
-       if ( $log ) {
-          $log->msg( $level, $thisSubName, __LINE__ , "Invalid number of arguments, got
$count=( @_ )" );
-
-       } else {
-          print "ERROR: $0 $thisSubName at ".__LINE__."Invalid number of arguments\n";
-    
-       }
-       return 1;
-   }
-
-   #Log command,  execute commdand, return results
-   $level  = "DEBUG" if ( !$level );
-   $log->msg( $level, $subName, $lineNo , "$cmd");
-
-    my @result = `$cmd`;
-
-    $log->msg( $level, $subName, $lineNo , "@cmd") if ( @cmd );
-
-    return @result;
-}
-
-sub getHiveCmd
-{
-    my ( $properties ) = @_;
-
-    my $subName        = (caller(0))[3];
-    my @baseCmd;
-
-    die "$0.$subName: null properties" if (! $properties );
-
-    my $cmd;
-
-    $cmd = $properties->{'hive_bin_location'} . "/hive";
-    if ( ! -x "$cmd" ) {
-      die "\n$0::$subName FATAL: Hive command does not exist: $cmd\n";
-    }
-    push (@baseCmd, $cmd);
-
-#   push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'});
-
-    return @baseCmd;
-}
-
-sub getHCatCmd
-{
-    my ( $properties ) = @_;
-
-    my $subName        = (caller(0))[3];
-    my @baseCmd;
-
-    die "$0.$subName: null properties" if (! $properties );
-
-    my $cmd;
-
-    $cmd = $properties->{'hcat_bin_location'};
-    if ( ! -x "$cmd" ) {
-         print STDERR "\n$0::$subName WARNING: Can't find hcat command: $cmd\n";
-         $cmd = `which hcat`;
-         chomp $cmd;
-         print STDERR "$0::$subName WARNING: Instead using command: $cmd\n";
-    }
-    die "\n$0::$subName FATAL: hcat command does not exist: $cmd\n" if ( ! -x $cmd  ); 
-    $ENV{"hive.metastore.local"} = "false"; 
-    $ENV{"hive.metastore.uris"}  = "thrift://".$properties->{'thriftserver'}; 
-    push (@baseCmd, $cmd);
-    return @baseCmd;
-}
 
 sub show_call_stack {
   my ( $path, $line, $subr );
@@ -651,43 +520,6 @@ print 'not use-pig.pl?????';
 }
 
 
-sub getBasePigSqlCmd 
-{
-
-    my $subName        = (caller(0))[3];
-   
-   Util::getPigCmd( 'testsql', @_ );
-
-}
-
-sub getBasePigCmd 
-{
-
-    my $subName        = (caller(0))[3];
-   
-   Util::getPigCmd( 'testjar', @_ );
-
-}
-
-sub getLatestBasePigCmd 
-{
-
-    my $subName        = (caller(0))[3];
-   
-   Util::getPigCmd( 'latesttestjar', @_ );
-
-}
-
-
-sub getBenchmarkBasePigCmd 
-{
-    my $subName        = (caller(0))[3];
-    my ( $properties ) = @_;
-
-   Util::getPigCmd( 'benchmarkjar', @_ );
-
-}
-
 sub setLocale
 {
    my $locale= shift;
@@ -725,4 +557,5 @@ sub getLocaleCmd 
           ."export LC_MEASUREMENT=\"$locale\";"
           ."export LC_IDENTIFICATION=\"$locale\"";
 }
+
 1;

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hadoop.conf Fri Jan  6 22:46:10 2012
@@ -79,7 +79,7 @@ jar :FUNCPATH:/testudf.jar org.apache.hc
                                 ,'hadoop' => q\
 jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.ReadRC -libjars :HCAT_JAR: :THRIFTSERVER:
all100krc :OUTPATH:
 \,
-                                ,'sql' => q\select * from all100krc;\
+                                ,'sql' => q\select name, age, floor(gpa) + 0.1 from all100krc;\
                                 ,'floatpostprocess' => 1
                                 ,'delimiter' => '	'
                                 },
@@ -89,6 +89,7 @@ jar :FUNCPATH:/testudf.jar org.apache.hc
                         'name' => 'Hadoop_Write',
                         'tests' => [
                                 {
+                                 'ignore' => 1, # Need to checkin HCATALOG-168.
                                  'num' => 1
                                 ,'hcat_prep'=>q\
 drop table if exists hadoop_write_1;
@@ -151,7 +152,7 @@ TBLPROPERTIES (
 jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.WriteRC -libjars :HCAT_JAR: :THRIFTSERVER:
all100krc hadoop_write_3
 \,
                                 ,'result_table' => 'hadoop_write_3'
-                                ,'sql' => q\select * from all100krc;\
+                                ,'sql' => q\select name, age, floor(gpa) + 0.1 from all100krc;\
                                 ,'floatpostprocess' => 1
                                 ,'delimiter' => '	'
                                 },

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hive.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hive.conf?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hive.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hive.conf Fri Jan  6 22:46:10 2012
@@ -94,7 +94,8 @@ $cfg = {
                     'name' => 'Hive_Read',
                     'tests' => [ {
                         'num' => 1,
-                         'sql' => q\select * from all100krc;\,
+                         'sql' => q\select name, age, floor(gpa) from all100krc;\,
+                         'verify_sql' => q\select name, age, truncate(gpa, 0) from all100krc;\,
                          'floatpostprocess' => 1,
                          'delimiter' => '	',
                     } ]
@@ -106,9 +107,9 @@ $cfg = {
                          'sql' => q\
 drop table if exists hive_write_1;
 create table hive_write_1 (name string, age int, gpa double) stored as rcfile;
-insert into TABLE hive_write_1 select * from all100krc;\,
+insert into TABLE hive_write_1 select name, age, floor(gpa) + 0.1 from all100krc;\,
                          'result_table' => 'hive_write_1',
-                         'verify_sql' =>"select name, age, gpa from all100krc;",
+                         'verify_sql' =>"select name, age, floor(gpa) + 0.1 from all100krc;",
                          'floatpostprocess' => 1,
                          'delimiter' => '	',
                     } ]

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/pig.conf Fri Jan  6 22:46:10 2012
@@ -86,6 +86,7 @@ store c into ':OUTPATH:';\,
                         'tests' => [
 
                                 {
+                                 'ignore' => 1, # Need to checkin HCATALOG-168.
                                  'num' => 1
                                 ,'pig' => q\a = load 'all100k' using org.apache.hcatalog.pig.HCatLoader();
 store a into ':OUTPATH:';\,
@@ -105,8 +106,9 @@ store b into ':OUTPATH:';\,
                                 {
                                  'num' => 3
                                 ,'pig' => q\a = load 'all100krc' using org.apache.hcatalog.pig.HCatLoader();
-store a into ':OUTPATH:';\,
-				,'sql' => q\select * from all100krc;\
+b = foreach a generate name, age;
+store b into ':OUTPATH:';\,
+				,'sql' => q\select name, age from all100krc;\
                                 ,'floatpostprocess' => 1
                                 ,'delimiter' => '	'
                                 }
@@ -116,6 +118,7 @@ store a into ':OUTPATH:';\,
                         'name' => 'Pig_Write',
                         'tests' => [
                                 {
+                                 'ignore' => 1, # Need to checkin HCATALOG-168.
                                  'num' => 1
                                 ,'hcat_prep'=>q\drop table if exists pig_write_1;
 create table pig_write_1(t tinyint,si smallint,i int,b bigint,bool boolean,f float,d double,s
string) stored as rcfile;\
@@ -142,8 +145,9 @@ create table pig_write_2(
 \
                                 ,'pig' => q\a = load 'all100kjson' using org.apache.hcatalog.pig.HCatLoader();
 b = foreach a generate s, i, d;
-store b into ':OUTPATH:';\,
+store b into 'pig_write_2' using org.apache.hcatalog.pig.HCatStorer();\,
 				,'sql' => q\select IFNULL(s, ""), IFNULL(i, ""), IFNULL(d, "") from all100kjson;\
+                                ,'result_table' => 'pig_write_2'
                                 ,'floatpostprocess' => 1
                                 ,'delimiter' => '	'
                                 },
@@ -161,8 +165,10 @@ TBLPROPERTIES (
 );
 \
                                 ,'pig' => q\a = load 'all100krc' using org.apache.hcatalog.pig.HCatLoader();
-store a into ':OUTPATH:';\,
-				,'sql' => q\select * from all100krc;\
+b = foreach a generate name, age;
+store b into 'pig_write_3' using org.apache.hcatalog.pig.HCatStorer();\,
+				,'sql' => q\select name, age from all100krc;\
+                                ,'result_table' => 'pig_write_3'
                                 ,'floatpostprocess' => 1
                                 ,'delimiter' => '	'
                                 }

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml Fri Jan  6 22:46:10
2012
@@ -1,23 +1,23 @@
-<!--  Licensed to the Apache Software Foundation (ASF) under one or more             
    
-   contributor license agreements.  See the NOTICE file distributed with               
-   this work for additional information regarding copyright ownership.                 
-   The ASF licenses this file to You under the Apache License, Version 2.0             
-   (the "License"); you may not use this file except in compliance with                
-   the License.  You may obtain a copy of the License at                               
-                                                                                       
-       http://www.apache.org/licenses/LICENSE-2.0                                      
-                                                                                       
-   Unless required by applicable law or agreed to in writing, software                 
-   distributed under the License is distributed on an "AS IS" BASIS,                   
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.            
-   See the License for the specific language governing permissions and                 
+<!--  Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
    limitations under the License.-->
-                                                                                       
+
 <project name="HCatalog-test-utils" default="udf-jar">
 
     <property name="udf.jarfile" value="testudf.jar" />
-    <property name="build.dir" value="${basedir}/build" />
-    <property name="src.dir" value="${basedir}/org/" />
+    <property name="udfs.build.dir" value="${basedir}/build" />
+    <property name="udfs.src.dir" value="${basedir}/org/" />
 
     <path id="udf-classpath">
        <fileset file="../../../../../../build/hcatalog/*.jar" />
@@ -27,17 +27,17 @@
     </path>
 
     <target name="init">
-        <mkdir dir="${build.dir}" />
+        <mkdir dir="${udfs.build.dir}" />
     </target>
 
     <target name="clean">
-        <delete dir="${build.dir}" />
+        <delete dir="${udfs.build.dir}" />
         <delete file="${udf.jarfile}" />
     </target>
 
     <target name="udf-compile" depends="init">
         <echo>*** Compiling UDFs ***</echo>
-        <javac srcdir="${src.dir}" destdir="${build.dir}" debug="on">
+        <javac srcdir="${udfs.src.dir}" destdir="${udfs.build.dir}" debug="on">
             <classpath refid="udf-classpath" />
         </javac>
     </target>

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadRC.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadRC.java?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadRC.java
(original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/ReadRC.java
Fri Jan  6 22:46:10 2012
@@ -68,6 +68,7 @@ public class ReadRC extends Configured i
         name = (String)value.get(0);
         age = (Integer)value.get(1);
         gpa = (Double)value.get(2);
+        gpa = Math.floor(gpa) + 0.1;
         
         HCatRecord record = new DefaultHCatRecord(3);
         record.set(0, name);

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteRC.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteRC.java?rev=1228478&r1=1228477&r2=1228478&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteRC.java
(original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/WriteRC.java
Fri Jan  6 22:46:10 2012
@@ -67,6 +67,8 @@ public class WriteRC extends Configured 
             name = value.get(0)==null?null:(String)value.get(0);
             age = value.get(1)==null?null:(Integer)value.get(1);
             gpa = value.get(2)==null?null:(Double)value.get(2);
+
+            if (gpa != null) gpa = Math.floor(gpa) + 0.1;
             
             HCatRecord record = new DefaultHCatRecord(5);
             record.set(0, name);



Mime
View raw message