hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r947104 - in /hadoop/hive/trunk: CHANGES.txt ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
Date Fri, 21 May 2010 18:08:31 GMT
Author: namit
Date: Fri May 21 18:08:31 2010
New Revision: 947104

URL: http://svn.apache.org/viewvc?rev=947104&view=rev
Log:
HIVE-1357. optimize CombineHiveInputFormat to cache inputFormat
(Ning Zhang via namit)


Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=947104&r1=947103&r2=947104&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri May 21 18:08:31 2010
@@ -450,6 +450,9 @@ Trunk -  Unreleased
     HIVE-1354. partition level properties honored if it exists
     (namit via He Yongqiang)
 
+    HIVE-1357. optimize CombineHiveInputFormat to cache inputFormat
+    (Ning Zhang via namit)
+
 Release 0.5.0 -  Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java?rev=947104&r1=947103&r2=947104&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
(original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
Fri May 21 18:08:31 2010
@@ -72,7 +72,7 @@ public class CombineHiveInputFormat<K ex
    */
   public static class CombineHiveInputSplit implements InputSplitShim {
 
-    String inputFormatClassName;
+    static String inputFormatClassName = null;
     InputSplitShim inputSplitShim;
 
     public CombineHiveInputSplit() throws IOException {
@@ -87,25 +87,18 @@ public class CombineHiveInputFormat<K ex
     public CombineHiveInputSplit(JobConf job, InputSplitShim inputSplitShim)
         throws IOException {
       this.inputSplitShim = inputSplitShim;
-      if (job != null) {
+
+      // get the inputFormatClassName only once because it will be the same
+      // for all input files within the same input split.
+      if (job != null && inputFormatClassName == null) {
         Map<String, PartitionDesc> pathToPartitionInfo = Utilities
             .getMapRedWork(job).getPathToPartitionInfo();
 
         // extract all the inputFormatClass names for each chunk in the
         // CombinedSplit.
         Path[] ipaths = inputSplitShim.getPaths();
-        for (int i = 0; i < ipaths.length; i++) {
-          PartitionDesc part = getPartitionDescFromPath(pathToPartitionInfo, ipaths[i]);
-
-          // create a new InputFormat instance if this is the first time to see
-          // this class
-          if (i == 0) {
-            inputFormatClassName = part.getInputFileFormatClass().getName();
-          } else {
-            assert inputFormatClassName.equals(part.getInputFileFormatClass()
-                .getName());
-          }
-        }
+        PartitionDesc part = getPartitionDescFromPath(pathToPartitionInfo, ipaths[0]);
+        inputFormatClassName = part.getInputFileFormatClass().getName();
       }
     }
 



Mime
View raw message