hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject hive git commit: HIVE-12208: Vectorized JOIN NPE on dynamically partitioned hash-join + map-join (Gunther Hagleitner, reviewed by Matt McCline)
Date Wed, 11 Nov 2015 21:45:52 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 e9c85d021 -> 691538b49


HIVE-12208: Vectorized JOIN NPE on dynamically partitioned hash-join + map-join (Gunther Hagleitner,
reviewed by Matt McCline)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/691538b4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/691538b4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/691538b4

Branch: refs/heads/branch-1
Commit: 691538b49f9acc913a5336890bd532db23d93c37
Parents: e9c85d0
Author: Gunther Hagleitner <gunther@apache.org>
Authored: Wed Nov 11 13:41:26 2015 -0800
Committer: Gunther Hagleitner <gunther@apache.org>
Committed: Wed Nov 11 13:42:46 2015 -0800

----------------------------------------------------------------------
 .../hadoop/hive/ql/exec/MapJoinOperator.java    |  2 +-
 .../mapjoin/VectorMapJoinCommonOperator.java    | 24 +++++---------------
 2 files changed, 7 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/691538b4/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
index 1b9d7ef..aa648b0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
@@ -187,7 +187,7 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc>
implem
 
   @SuppressWarnings("unchecked")
   @Override
-  protected final void completeInitializationOp(Object[] os) throws HiveException {
+  protected void completeInitializationOp(Object[] os) throws HiveException {
     if (os.length != 0) {
       Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> pair =
           (Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]>) os[0];

http://git-wip-us.apache.org/repos/asf/hive/blob/691538b4/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
index 420ea91..94e6c67 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
@@ -612,10 +612,9 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator
implem
   }
 
   @Override
-  protected Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> loadHashTable(
-      ExecMapperContext mapContext, MapredContext mrContext) throws HiveException {
-
-    Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> pair;
+  protected void completeInitializationOp(Object[] os) throws HiveException {
+    // setup mapJoinTables and serdes
+    super.completeInitializationOp(os);
 
     VectorMapJoinDesc vectorDesc = conf.getVectorDesc();
     HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType();
@@ -623,36 +622,25 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator
implem
     switch (vectorDesc.hashTableImplementationType()) {
     case OPTIMIZED:
       {
-        // Using Tez's HashTableLoader, create either a MapJoinBytesTableContainer or
-        // HybridHashTableContainer.
-        pair = super.loadHashTable(mapContext, mrContext);
-
         // Create our vector map join optimized hash table variation *above* the
         // map join table container.
-        MapJoinTableContainer[] mapJoinTableContainers = pair.getLeft();
         vectorMapJoinHashTable = VectorMapJoinOptimizedCreateHashTable.createHashTable(conf,
-                mapJoinTableContainers[posSingleVectorMapJoinSmallTable]);
+                mapJoinTables[posSingleVectorMapJoinSmallTable]);
       }
       break;
 
     case FAST:
       {
-        // Use our VectorMapJoinFastHashTableLoader to create a VectorMapJoinTableContainer.
-        pair = super.loadHashTable(mapContext, mrContext);
-
         // Get our vector map join fast hash table variation from the
         // vector map join table container.
-        MapJoinTableContainer[] mapJoinTableContainers = pair.getLeft();
         VectorMapJoinTableContainer vectorMapJoinTableContainer =
-                (VectorMapJoinTableContainer) mapJoinTableContainers[posSingleVectorMapJoinSmallTable];
+                (VectorMapJoinTableContainer) mapJoinTables[posSingleVectorMapJoinSmallTable];
         vectorMapJoinHashTable = vectorMapJoinTableContainer.vectorMapJoinHashTable();
       }
       break;
     default:
       throw new RuntimeException("Unknown vector map join hash table implementation type
" + hashTableImplementationType.name());
     }
-
-    return pair;
   }
 
   /*
@@ -773,4 +761,4 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator
implem
   public VectorizationContext getOuputVectorizationContext() {
     return vOutContext;
   }
-}
\ No newline at end of file
+}


Mime
View raw message