incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1296714 - in /incubator/hcatalog/branches/branch-0.4: ./ src/java/org/apache/hcatalog/common/ src/java/org/apache/hcatalog/mapreduce/ src/test/e2e/hcatalog/drivers/ src/test/e2e/hcatalog/tests/ src/test/org/apache/hcatalog/mapreduce/
Date Sat, 03 Mar 2012 22:13:09 GMT
Author: gates
Date: Sat Mar  3 22:13:09 2012
New Revision: 1296714

URL: http://svn.apache.org/viewvc?rev=1296714&view=rev
Log:
HCATALOG-264 Barrier for HCatOutputFormat for Pig/Mapreduce if the table contains feature
only supported in Hive

Modified:
    incubator/hcatalog/branches/branch-0.4/CHANGES.txt
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/common/ErrorType.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tests/pig.conf
    incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java

Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1296714&r1=1296713&r2=1296714&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Sat Mar  3 22:13:09 2012
@@ -21,7 +21,9 @@ Apache HCatalog Change Log
 Release 0.4.0 - Unreleased
 
   INCOMPATIBLE CHANGES
-  HCAT-279 Remove remaining code mentioning isd/osd (diajy via gates)
+  HCAT-264 Barrier for HCatOutputFormat for Pig/Mapreduce if the table contains feature only
supported in Hive (daijy via gates)
+
+  HCAT-279 Remove remaining code mentioning isd/osd (daijy via gates)
 
   HCAT-261 Enable more DDL statements for HCat (daijy via gates)
 

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/common/ErrorType.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/common/ErrorType.java?rev=1296714&r1=1296713&r2=1296714&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/common/ErrorType.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/common/ErrorType.java
Sat Mar  3 22:13:09 2012
@@ -45,6 +45,7 @@ public enum ErrorType {
     ERROR_TOO_MANY_DYNAMIC_PTNS         (2013, "Attempt to create too many dynamic partitions"),
     ERROR_INIT_LOADER                   (2014,  "Error initializing Pig loader"),
     ERROR_INIT_STORER                   (2015,  "Error initializing Pig storer"),
+    ERROR_NOT_SUPPORTED                 (2016,  "Error operation not supported"),
 
     /* Authorization Errors 3000 - 3999 */
     ERROR_ACCESS_CONTROL           (3000, "Permission denied"),

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java?rev=1296714&r1=1296713&r2=1296714&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
Sat Mar  3 22:13:09 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Index;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
@@ -77,7 +78,29 @@ public class HCatOutputFormat extends HC
         hiveConf = HCatUtil.getHiveConf(null, conf);
         client = HCatUtil.createHiveClient(hiveConf);
         Table table = client.getTable(outputJobInfo.getDatabaseName(), outputJobInfo.getTableName());
-
+        
+        List<String> indexList = client.listIndexNames(outputJobInfo.getDatabaseName(),
outputJobInfo.getTableName(), Short.MAX_VALUE);
+        
+        for (String indexName : indexList) {
+            Index index = client.getIndex(outputJobInfo.getDatabaseName(), outputJobInfo.getTableName(),
indexName);
+            if (!index.isDeferredRebuild()) {
+                throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a table
with an automatic index from Pig/Mapreduce is not supported");
+            }
+        }
+        StorageDescriptor sd = table.getSd();
+        
+        if (sd.isCompressed()) {
+            throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a compressed
partition from Pig/Mapreduce is not supported");
+        }
+        
+        if (sd.getBucketCols()!=null && !sd.getBucketCols().isEmpty()) {
+            throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a partition
with bucket definition from Pig/Mapreduce is not supported");
+        }
+        
+        if (sd.getSortCols()!=null && !sd.getSortCols().isEmpty()) {
+            throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a partition
with sorted column definition from Pig/Mapreduce is not supported");
+        }
+        
         if (table.getPartitionKeysSize() == 0 ){
           if ((outputJobInfo.getPartitionValues() != null) && (!outputJobInfo.getPartitionValues().isEmpty())){
             // attempt made to save partition values in non-partitioned table - throw error.

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverPig.pm?rev=1296714&r1=1296713&r2=1296714&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
(original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/TestDriverPig.pm
Sat Mar  3 22:13:09 2012
@@ -192,7 +192,7 @@ sub runTest
        # run a another Pig script to dump the results of the table.
        my $result;
        if (defined($testCmd->{'result_table'})) {
-           $result = $self->runPig( $testCmd, $log, 0);
+           $result = $self->runPig( $testCmd, $log, 0, 1);
            my @results = ();
            my @outputs = ();
            if (ref($testCmd->{'result_table'}) ne 'ARRAY') {
@@ -214,7 +214,7 @@ sub runTest
                    $tableName = $results[$i];
 	           $modifiedTestCmd{'num'} = $testCmd->{'num'} . "_" . $i . "_benchmark";
                    $modifiedTestCmd{'pig'} = "a = load '$tableName' using org.apache.hcatalog.pig.HCatLoader();
store a into ':OUTPATH:';";
-                   my $r = $self->runPig(\%modifiedTestCmd, $log, 1);
+                   my $r = $self->runPig(\%modifiedTestCmd, $log, 1, 1);
 	           $outputs[$i] = $r->{'output'};
                } else {
                    $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_"
. $testCmd->{'num'} . ".out/$id";
@@ -233,7 +233,7 @@ sub runTest
            }
        }
        else {
-           $result = $self->runPig( $testCmd, $log, 1);
+           $result = $self->runPig( $testCmd, $log, 1, 1);
        }
        return $result;
     } elsif(  $testCmd->{'script'} ){
@@ -447,7 +447,7 @@ sub dumpPigTable
 
 sub runPig
 {
-    my ($self, $testCmd, $log, $copyResults) = @_;
+    my ($self, $testCmd, $log, $copyResults, $noFailOnFail) = @_;
     my $subName  = (caller(0))[3];
 
     my %result;
@@ -488,8 +488,13 @@ sub runPig
     print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n";
     print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n";
 
-    IPC::Run::run(\@cmd, \undef, $log, $log) or
+    my $runrc = IPC::Run::run(\@cmd, \undef, $log, $log);
+
+    if (defined($noFailOnFail) && $noFailOnFail) {
+    } else {
         die "Failed running $pigfile\n";
+    }
+
     $result{'rc'} = $? >> 8;
 
 

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tests/pig.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tests/pig.conf?rev=1296714&r1=1296713&r2=1296714&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tests/pig.conf (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tests/pig.conf Sat Mar  3
22:13:09 2012
@@ -280,6 +280,28 @@ store a into ':OUTPATH:';\,
                                 }
                         ],
                 }, # end g
-
+                {
+                        'name' => 'Pig_HCat_Barrier',
+                        'tests' => [
+                                {
+                                 'num' => 1
+                                ,'hcat_prep'=>q\drop table if exists pig_hcat_barrier_1;
+create table pig_hcat_barrier_1 (name string, age int, gpa double) partitioned by (b string)
CLUSTERED BY (name) INTO 1 BUCKETS STORED AS TEXTFILE;\
+                                ,'pig' => q\
+a = load ':INPATH:/studenttab10k' as (name:chararray, age:int, gpa:double);
+store a into 'pig_hcat_barrier_1' using org.apache.hcatalog.pig.HCatStorer('b=1'); \,
+                                ,'expected_err_regex' => 'not supported'
+                                },
+                                {
+                                 'num' => 2
+                                ,'hcat_prep'=>q\drop table if exists pig_hcat_barrier_2;
+create table pig_hcat_barrier_2 (name string, age int, gpa double) partitioned by (b string)
CLUSTERED BY (name) SORTED BY (name) INTO 1 BUCKETS STORED AS TEXTFILE;\
+                                ,'pig' => q\
+a = load ':INPATH:/studenttab10k' as (name:chararray, age:int, gpa:double);
+store a into 'pig_hcat_barrier_2' using org.apache.hcatalog.pig.HCatStorer('b=1'); \,
+                                ,'expected_err_regex' => 'not supported'
+                                },
+                        ],
+                }, # end g
          ]
 }

Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java?rev=1296714&r1=1296713&r2=1296714&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java
(original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java
Sat Mar  3 22:13:09 2012
@@ -109,8 +109,8 @@ public class TestHCatOutputFormat extend
     sd.setOutputFormat(RCFileOutputFormat.class.getName());
     sd.setParameters(new HashMap<String, String>());
     sd.getParameters().put("test_param_1", "Use this for comments etc");
-    sd.setBucketCols(new ArrayList<String>(2));
-    sd.getBucketCols().add("name");
+    //sd.setBucketCols(new ArrayList<String>(2));
+    //sd.getBucketCols().add("name");
     sd.setSerdeInfo(new SerDeInfo());
     sd.getSerdeInfo().setName(tbl.getTableName());
     sd.getSerdeInfo().setParameters(new HashMap<String, String>());



Mime
View raw message