hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nzh...@apache.org
Subject svn commit: r947910 - /hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
Date Tue, 25 May 2010 04:24:13 GMT
Author: nzhang
Date: Tue May 25 04:24:12 2010
New Revision: 947910

URL: http://svn.apache.org/viewvc?rev=947910&view=rev
Log:
HIVE-1366. inputFileFormat error if the merge job takes a different input file format than
the default output file format (Namit Jain via Ning Zhang)

Modified:
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java?rev=947910&r1=947909&r2=947910&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
(original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
Tue May 25 04:24:12 2010
@@ -72,7 +72,7 @@ public class CombineHiveInputFormat<K ex
    */
   public static class CombineHiveInputSplit implements InputSplitShim {
 
-    static String inputFormatClassName = null;
+    String inputFormatClassName;
     InputSplitShim inputSplitShim;
 
     public CombineHiveInputSplit() throws IOException {
@@ -87,18 +87,17 @@ public class CombineHiveInputFormat<K ex
     public CombineHiveInputSplit(JobConf job, InputSplitShim inputSplitShim)
         throws IOException {
       this.inputSplitShim = inputSplitShim;
-
-      // get the inputFormatClassName only once because it will be the same
-      // for all input files within the same input split.
-      if (job != null && inputFormatClassName == null) {
+      if (job != null) {
         Map<String, PartitionDesc> pathToPartitionInfo = Utilities
             .getMapRedWork(job).getPathToPartitionInfo();
 
         // extract all the inputFormatClass names for each chunk in the
         // CombinedSplit.
         Path[] ipaths = inputSplitShim.getPaths();
-        PartitionDesc part = getPartitionDescFromPath(pathToPartitionInfo, ipaths[0]);
-        inputFormatClassName = part.getInputFileFormatClass().getName();
+        if (ipaths.length > 0) {
+          PartitionDesc part = getPartitionDescFromPath(pathToPartitionInfo, ipaths[0]);
+          inputFormatClassName = part.getInputFileFormatClass().getName();
+        }
       }
     }
 
@@ -336,13 +335,13 @@ public class CombineHiveInputFormat<K ex
     String dirPath = dir.toUri().getPath();
     PartitionDesc part = pathToPartitionInfo.get(dir.toString());
     if (part == null) {
-      LOG.warn("exact match not found, try ripping input path's theme and authority");
+      //      LOG.warn("exact match not found, try ripping input path's theme and authority");
       part = pathToPartitionInfo.get(dirPath);
     }
     if (part == null) {
 
-      LOG.warn("still does not found just the path part: " + dirPath + " in pathToPartitionInfo."
-          + " Will try prefix matching");
+      //      LOG.warn("still does not found just the path part: " + dirPath + " in pathToPartitionInfo."
+      //          + " Will try prefix matching");
       for (Map.Entry<String, PartitionDesc> entry : pathToPartitionInfo.entrySet())
{
         String keyPath = entry.getKey();
         String dirStr = dir.toString();



Mime
View raw message