hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject svn commit: r1616642 - in /hive/trunk/hcatalog: core/src/main/java/org/apache/hive/hcatalog/mapreduce/SpecialCases.java hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
Date Thu, 07 Aug 2014 23:57:14 GMT
Author: khorgath
Date: Thu Aug  7 23:57:14 2014
New Revision: 1616642

URL: http://svn.apache.org/r1616642
Log:
HIVE-7072 : HCatLoader only loads first region of hbase table (Sushanth Sowmyan, reviewed
by Daniel Dai)

Modified:
    hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/SpecialCases.java
    hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java

Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/SpecialCases.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/SpecialCases.java?rev=1616642&r1=1616641&r2=1616642&view=diff
==============================================================================
--- hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/SpecialCases.java
(original)
+++ hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/SpecialCases.java
Thu Aug  7 23:57:14 2014
@@ -20,6 +20,7 @@ package org.apache.hive.hcatalog.mapredu
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
 import org.apache.hadoop.hive.ql.io.orc.OrcFile;
@@ -37,6 +38,7 @@ import java.util.Map;
  * class that allows us to still be as generic as possible
  * in the main codeflow path, and call attention to the special
  * cases here.
+ *
  * Note : For all methods introduced here, please document why
  * the special case is necessary, providing a jira number if
  * possible.
@@ -50,6 +52,11 @@ public class SpecialCases {
    * instantiating a storage handler to write. We set any parameters
    * we want to be visible to the job in jobProperties, and this will
    * be available to the job via jobconf at run time.
+   *
+   * This is mostly intended to be used by StorageHandlers that wrap
+   * File-based OutputFormats such as FosterStorageHandler that wraps
+   * RCFile, ORC, etc.
+   *
    * @param jobProperties : map to write to
    * @param jobInfo : information about this output job to read from
    * @param ofclass : the output format in use
@@ -78,5 +85,26 @@ public class SpecialCases {
     }
   }
 
+  /**
+   * Method to do any storage-handler specific special casing while instantiating a
+   * HCatLoader
+   *
+   * @param conf : configuration to write to
+   * @param tableInfo : the table definition being used
+   */
+  public static void addSpecialCasesParametersForHCatLoader(
+      Configuration conf, HCatTableInfo tableInfo) {
+    if ((tableInfo == null) || (tableInfo.getStorerInfo() == null)){
+      return;
+    }
+    String shClass = tableInfo.getStorerInfo().getStorageHandlerClass();
+    if ((shClass != null) && shClass.equals("org.apache.hadoop.hive.hbase.HBaseStorageHandler")){
+      // NOTE: The reason we use a string name of the hive hbase handler here is
+      // because we do not want to introduce a compile-dependency on the hive-hbase-handler
+      // module from within hive-hcatalog.
+      // This parameter was added due to the requirement in HIVE-7072
+      conf.set("pig.noSplitCombination", "true");
+    }
+  }
 
 }

Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java?rev=1616642&r1=1616641&r2=1616642&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
(original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatLoader.java
Thu Aug  7 23:57:14 2014
@@ -43,6 +43,7 @@ import org.apache.hive.hcatalog.data.Pai
 import org.apache.hive.hcatalog.data.schema.HCatSchema;
 import org.apache.hive.hcatalog.mapreduce.HCatInputFormat;
 import org.apache.hive.hcatalog.mapreduce.InputJobInfo;
+import org.apache.hive.hcatalog.mapreduce.SpecialCases;
 import org.apache.pig.Expression;
 import org.apache.pig.Expression.BinaryExpression;
 import org.apache.pig.PigException;
@@ -125,6 +126,12 @@ public class HCatLoader extends HCatBase
       Job clone = new Job(job.getConfiguration());
       HCatInputFormat.setInput(job, dbName, tableName, getPartitionFilterString());
 
+      InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(
+          job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO));
+
+      SpecialCases.addSpecialCasesParametersForHCatLoader(job.getConfiguration(),
+          inputJobInfo.getTableInfo());
+
       // We will store all the new /changed properties in the job in the
       // udf context, so the the HCatInputFormat.setInput method need not
       //be called many times.



Mime
View raw message