hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject svn commit: r1577259 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec: ./ mr/ persistence/ tez/ vector/
Date Thu, 13 Mar 2014 17:32:53 GMT
Author: sershe
Date: Thu Mar 13 17:32:52 2014
New Revision: 1577259

URL: http://svn.apache.org/r1577259
Log:
HIVE-6618 : assertion when getting reference key from loader with byte-array mapjoin key (Sergey
Shelukhin, reviewed by Gunther Hagleitner)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java?rev=1577259&r1=1577258&r2=1577259&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java Thu Mar 13
17:32:52 2014
@@ -35,6 +35,4 @@ public interface HashTableLoader {
 
   void load(MapJoinTableContainer[] mapJoinTables, MapJoinTableContainerSerDe[] mapJoinTableSerdes)
       throws HiveException;
-
-  MapJoinKey getKeyType();
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1577259&r1=1577258&r2=1577259&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Thu Mar 13
17:32:52 2014
@@ -170,11 +170,25 @@ public class MapJoinOperator extends Abs
 
   protected transient final Output outputForMapJoinKey = new Output();
   protected MapJoinKey computeMapJoinKey(Object row, byte alias) throws HiveException {
-    MapJoinKey refKey = (key == null ? loader.getKeyType() : key);
+    MapJoinKey refKey = getRefKey(key, alias);
     return MapJoinKey.readFromRow(outputForMapJoinKey,
         refKey, row, joinKeys[alias], joinKeysObjectInspectors[alias], key == refKey);
   }
 
+  protected MapJoinKey getRefKey(MapJoinKey prevKey, byte alias) {
+    if (prevKey != null) return prevKey;
+    // We assume that since we are joining on the same key, all tables would have either
+    // optimized or non-optimized key; hence, we can pass any key in any table as reference.
+    // We do it so that MJKB could determine whether it can use optimized keys.
+    for (byte pos = 0; pos < order.length; pos++) {
+      if (pos == alias) continue;
+      MapJoinKey refKey = mapJoinTables[pos].getAnyKey();
+      if (refKey != null) return refKey;
+    }
+    return null; // All join tables have 0 keys, doesn't matter what we generate.
+  }
+
+
   @Override
   public void processOp(Object row, int tag) throws HiveException {
     try {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java?rev=1577259&r1=1577258&r2=1577259&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HashTableLoader.java Thu Mar
13 17:32:52 2014
@@ -155,9 +155,4 @@ public class HashTableLoader implements 
 
     Arrays.fill(tables, null);
   }
-
-  @Override
-  public MapJoinKey getKeyType() {
-    return new MapJoinKeyObject(); // always use Object-array keys
-  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java?rev=1577259&r1=1577258&r2=1577259&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
Thu Mar 13 17:32:52 2014
@@ -102,4 +102,9 @@ public class HashMapWrapper extends Abst
     metaData.put(LOAD_NAME, String.valueOf(loadFactor));
     return metaData;
   }
+
+  @Override
+  public MapJoinKey getAnyKey() {
+    return mHash.isEmpty() ? null : mHash.keySet().iterator().next();
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java?rev=1577259&r1=1577258&r2=1577259&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java
Thu Mar 13 17:32:52 2014
@@ -35,4 +35,6 @@ public interface MapJoinTableContainer {
 
   public void clear();
 
+  public MapJoinKey getAnyKey();
+
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java?rev=1577259&r1=1577258&r2=1577259&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java Thu Mar
13 17:32:52 2014
@@ -124,12 +124,4 @@ public class HashTableLoader implements 
       lastKey = new MapJoinKeyObject(); // No rows in tables, the key type doesn't matter.
     }
   }
-
-  @Override
-  public MapJoinKey getKeyType() {
-    if (lastKey == null) {
-      throw new AssertionError("Should be called after loading tables");
-    }
-    return lastKey;
-  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java?rev=1577259&r1=1577258&r2=1577259&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
Thu Mar 13 17:32:52 2014
@@ -159,7 +159,7 @@ public class VectorMapJoinOperator exten
 
       @Override
       public MapJoinKey evaluate(VectorHashKeyWrapper kw) throws HiveException {
-        MapJoinKey refKey = (key == null ? loader.getKeyType() : key);
+        MapJoinKey refKey = getRefKey(key, alias);
         key = MapJoinKey.readFromVector(
             output, refKey, kw, keyOutputWriters, keyWrapperBatch, refKey == key);
         return key;



Mime
View raw message