hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r1084964 - /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
Date Thu, 24 Mar 2011 14:09:50 GMT
Author: namit
Date: Thu Mar 24 14:09:49 2011
New Revision: 1084964

URL: http://svn.apache.org/viewvc?rev=1084964&view=rev
Log:
HIVE-1966 mapjoin operator should not load hashtable for each new
          inputfile if the hashtable to be loaded is already there
          (Yongqiang He via namit)


Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1084964&r1=1084963&r2=1084964&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Thu Mar 24
14:09:49 2011
@@ -64,6 +64,7 @@ public class MapJoinOperator extends Abs
   transient int metadataKeyTag;
   transient int[] metadataValueTag;
   transient int maxMapJoinSize;
+  transient boolean hashTblInitedOnce;
   private int bigTableAlias;
 
   public MapJoinOperator() {
@@ -103,7 +104,7 @@ public class MapJoinOperator extends Abs
       rowContainerMap.put(Byte.valueOf((byte) pos), rowContainer);
     }
 
-
+    hashTblInitedOnce = false;
   }
 
   @Override
@@ -144,10 +145,17 @@ public class MapJoinOperator extends Abs
   }
 
   private void loadHashTable() throws HiveException {
+
+    if (!this.getExecContext().getLocalWork().getInputFileChangeSensitive()) {
+      if (hashTblInitedOnce) {
+        return;
+      } else {
+        hashTblInitedOnce = true;
+      }
+    }
+    
     boolean localMode = HiveConf.getVar(hconf, HiveConf.ConfVars.HADOOPJT).equals("local");
     String baseDir = null;
-    HashMapWrapper<AbstractMapJoinKey, MapJoinObjectValue> hashtable;
-    Byte pos;
 
     String currentInputFile = HiveConf.getVar(hconf, HiveConf.ConfVars.HADOOPMAPFILENAME);
     LOG.info("******* Load from HashTable File: input : " + currentInputFile);
@@ -181,8 +189,8 @@ public class MapJoinOperator extends Abs
       }
       for (Map.Entry<Byte, HashMapWrapper<AbstractMapJoinKey, MapJoinObjectValue>>
entry : mapJoinTables
           .entrySet()) {
-        pos = entry.getKey();
-        hashtable = entry.getValue();
+        Byte pos = entry.getKey();
+        HashMapWrapper<AbstractMapJoinKey, MapJoinObjectValue> hashtable = entry.getValue();
         String filePath = Utilities.generatePath(baseDir, pos, currentFileName);
         Path path = new Path(filePath);
         LOG.info("\tLoad back 1 hashtable file from tmp file uri:" + path.toString());
@@ -293,7 +301,7 @@ public class MapJoinOperator extends Abs
   public void closeOp(boolean abort) throws HiveException {
 
     if (mapJoinTables != null) {
-      for (HashMapWrapper hashTable : mapJoinTables.values()) {
+      for (HashMapWrapper<?, ?> hashTable : mapJoinTables.values()) {
         hashTable.close();
       }
     }



Mime
View raw message