hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [11/17] hive git commit: HIVE-9824 : LLAP: Native Vectorization of Map Join (Matt McCline, reviewed by Sergey Shelukhin, Vikram Dixit K)
Date Wed, 22 Apr 2015 22:51:01 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java
new file mode 100644
index 0000000..3789275
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
+import org.apache.hadoop.hive.ql.exec.tez.HashTableLoader;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashTable;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinTableContainer;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc;
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableImplementationType;
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableKeyType;
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableKind;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.tez.runtime.library.api.KeyValueReader;
+
+/**
+ * HashTableLoader for Tez constructs the hashtable from records read from
+ * a broadcast edge.
+ */
+public class VectorMapJoinFastTableContainer implements VectorMapJoinTableContainer {
+
+  private static final Log LOG = LogFactory.getLog(HashTableLoader.class.getName());
+
+  private MapJoinDesc desc;
+  private Configuration hconf;
+
+  private float keyCountAdj;
+  private int threshold;
+  private float loadFactor;
+  private int wbSize;
+  private long keyCount;
+  private long memUsage;
+
+
+  private VectorMapJoinFastHashTable VectorMapJoinFastHashTable;
+
+  public VectorMapJoinFastTableContainer(MapJoinDesc desc, Configuration hconf,
+      long keyCount, long memUsage) throws SerDeException {
+
+    this.desc = desc;
+    this.hconf = hconf;
+
+    keyCountAdj = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT);
+    threshold = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD);
+    loadFactor = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR);
+    wbSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE);
+
+    this.keyCount = keyCount;
+    this.memUsage = memUsage;
+
+    // LOG.info("VectorMapJoinFastTableContainer load keyCountAdj " + keyCountAdj);
+    // LOG.info("VectorMapJoinFastTableContainer load threshold " + threshold);
+    // LOG.info("VectorMapJoinFastTableContainer load loadFactor " + loadFactor);
+    // LOG.info("VectorMapJoinFastTableContainer load wbSize " + wbSize);
+    // LOG.info("VectorMapJoinFastTableContainer load memUsage " + memUsage);
+
+    int newThreshold = HashMapWrapper.calculateTableSize(
+        keyCountAdj, threshold, loadFactor, keyCount);
+
+    // LOG.info("VectorMapJoinFastTableContainer load newThreshold " + newThreshold);
+
+    VectorMapJoinFastHashTable = createHashTable(newThreshold);
+  }
+
+  @Override
+  public VectorMapJoinHashTable vectorMapJoinHashTable() {
+    return (VectorMapJoinHashTable) VectorMapJoinFastHashTable;
+  }
+
+  private VectorMapJoinFastHashTable createHashTable(int newThreshold) {
+
+    boolean isOuterJoin = !desc.isNoOuterJoin();
+    VectorMapJoinDesc vectorDesc = desc.getVectorDesc();
+    HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType();
+    HashTableKind hashTableKind = vectorDesc.hashTableKind();
+    HashTableKeyType hashTableKeyType = vectorDesc.hashTableKeyType();
+    boolean minMaxEnabled = vectorDesc.minMaxEnabled();
+
+    int writeBufferSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEWBSIZE);
+
+    VectorMapJoinFastHashTable hashTable = null;
+
+    switch (hashTableKeyType) {
+    case BOOLEAN:
+    case BYTE:
+    case SHORT:
+    case INT:
+    case LONG:
+      switch (hashTableKind) {
+      case HASH_MAP:
+        hashTable = new VectorMapJoinFastLongHashMap(
+                minMaxEnabled, isOuterJoin, hashTableKeyType,
+                newThreshold, loadFactor, writeBufferSize, memUsage);
+        break;
+      case HASH_MULTISET:
+        hashTable = new VectorMapJoinFastLongHashMultiSet(
+                minMaxEnabled, isOuterJoin, hashTableKeyType,
+                newThreshold, loadFactor, writeBufferSize, memUsage);
+        break;
+      case HASH_SET:
+        hashTable = new VectorMapJoinFastLongHashSet(
+                minMaxEnabled, isOuterJoin, hashTableKeyType,
+                newThreshold, loadFactor, writeBufferSize, memUsage);
+        break;
+      }
+      break;
+
+    case STRING:
+      switch (hashTableKind) {
+      case HASH_MAP:
+        hashTable = new VectorMapJoinFastStringHashMap(
+                isOuterJoin,
+                newThreshold, loadFactor, writeBufferSize, memUsage);
+        break;
+      case HASH_MULTISET:
+        hashTable = new VectorMapJoinFastStringHashMultiSet(
+                isOuterJoin,
+                newThreshold, loadFactor, writeBufferSize, memUsage);
+        break;
+      case HASH_SET:
+        hashTable = new VectorMapJoinFastStringHashSet(
+                isOuterJoin,
+                newThreshold, loadFactor, writeBufferSize, memUsage);
+        break;
+      }
+      break;
+
+    case MULTI_KEY:
+      switch (hashTableKind) {
+      case HASH_MAP:
+        hashTable = new VectorMapJoinFastMultiKeyHashMap(
+            isOuterJoin,
+            newThreshold, loadFactor, writeBufferSize, memUsage);
+        break;
+      case HASH_MULTISET:
+        hashTable = new VectorMapJoinFastMultiKeyHashMultiSet(
+                isOuterJoin,
+                newThreshold, loadFactor, writeBufferSize, memUsage);
+        break;
+      case HASH_SET:
+        hashTable = new VectorMapJoinFastMultiKeyHashSet(
+                isOuterJoin,
+                newThreshold, loadFactor, writeBufferSize, memUsage);
+        break;
+      }
+      break;
+    }
+
+    return hashTable;
+  }
+
+  @Override
+  public MapJoinKey putRow(MapJoinObjectSerDeContext keyContext,
+      Writable currentKey, MapJoinObjectSerDeContext valueContext,
+      Writable currentValue) throws SerDeException, HiveException, IOException {
+
+    // We are not using the key and value contexts, nor do we support a MapJoinKey.
+    VectorMapJoinFastHashTable.putRow((BytesWritable) currentKey, (BytesWritable) currentValue);
+    return null;
+  }
+
+  @Override
+  public void seal() {
+    // Do nothing
+  }
+
+  @Override
+  public ReusableGetAdaptor createGetter(MapJoinKey keyTypeFromLoader) {
+    throw new RuntimeException("Not applicable");
+  }
+
+  @Override
+  public void clear() {
+    throw new RuntimeException("Not applicable");
+  }
+
+  @Override
+  public MapJoinKey getAnyKey() {
+    throw new RuntimeException("Not applicable");
+  }
+
+  @Override
+  public void dumpMetrics() {
+    // TODO
+  }
+
+  @Override
+  public boolean hasSpill() {
+    return false;
+  }
+
+  /*
+  @Override
+  public com.esotericsoftware.kryo.io.Output getHybridBigTableSpillOutput(int partitionId) {
+    throw new RuntimeException("Not applicable");
+  }
+  */
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java
new file mode 100644
index 0000000..caa705c
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java
@@ -0,0 +1,557 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMapResult;
+import org.apache.hadoop.hive.serde2.WriteBuffers;
+import org.apache.hadoop.hive.serde2.WriteBuffers.ByteSegmentRef;
+import org.apache.hadoop.hive.serde2.WriteBuffers.Position;;
+
+
+// Supports random access.
+
+public class VectorMapJoinFastValueStore {
+
+  private static final Log LOG = LogFactory.getLog(VectorMapJoinFastValueStore.class.getName());
+
+  private WriteBuffers writeBuffers;
+
+
+  /**
+   * A store for "lists" of arbitrary length values in memory.
+   *
+   * The memory is a "infinite" byte array or WriteBuffers object.
+   *
+   * We give the client a 64-bit (long) value reference to keep that has the offset within
+   * the "infinite" byte array of the last value inserted in a "list".
+   *
+   * We optimize the common case when "list"s are 1 element and values are short and store the
+   * value length in the value reference word.
+   *
+   * We also support keeping a count (up to a limit or cap) so help with join result generation
+   * algorithms.
+   *
+   * If the last value is big, the big length will be encoded as an integer at the beginning
+   * of the value followed by the big value bytes.
+   *
+   * Due to optimizing by keeping the last value's length in the value reference, when we have
+   * more than one value, a new value will need to keep the small value length of the next
+   * value.
+   *
+   * So, values after the first value have 4 parts: a relative offset word with flags, an
+   * optional length if the current value is big, an optional next value length if it is small,
+   * and the value bytes.
+   *
+   * Cases:
+   *  1) One element, small:
+   *
+   *    Value Reference -------------
+   *                                 |
+   *                                 |
+   *                                 v
+   *                                 {Small Value Bytes}
+   *
+   *  2) One element, big:
+   *
+   *   Value Reference --------------
+   *                                 |
+   *                                 |
+   *                                 v
+   *                                 {Big Value Len} {Big Value Bytes}
+   *
+   *  1) Multiple elements:
+   *
+   *    Value Reference ----------------
+   *                                    |
+   *                                    |    //  Last value added.
+   *                                    |
+   *                                    v
+   *                                    {Rel Offset Word} [Big Value Len] [Next Value Small Len] {Value Bytes}
+   *                                             |            optional        optional
+   *                                             |
+   *                                             |
+   *                                --- . . . ---
+   *                               |
+   *                               |      // 0 or more
+   *                               |
+   *                               v
+   *                              {Rel Offset Word} [Big Value Len] [Next Value Small Len] {Value Bytes}
+   *                                         |          optional        optional
+   *                                         |
+   *                                         |
+   *                     --------------------
+   *                    |
+   *                    |
+   *                    v
+   *                   [Big Value Length] {Value Bytes}
+   *                       optional
+   *
+   *                   // First value added without Relative Offset Word, etc.
+   */
+
+  public WriteBuffers writeBuffers() {
+    return writeBuffers;
+  }
+
+  public static class HashMapResult extends VectorMapJoinHashMapResult {
+
+    private VectorMapJoinFastValueStore valueStore;
+
+    private boolean hasRows;
+    private long valueRefWord;
+    private boolean isSingleRow;
+    private int cappedCount;
+
+    private boolean haveReadCurrent;
+    private int readIndex;
+    private boolean isEof;
+
+    private boolean isNextEof;
+    private boolean isNextLast;
+    long nextAbsoluteValueOffset;
+    boolean isNextValueLengthSmall;
+    int nextSmallValueLength;
+
+    private ByteSegmentRef byteSegmentRef;
+    private Position readPos;
+
+    public HashMapResult() {
+      super();
+      valueRefWord = -1;
+      hasRows = false;
+      byteSegmentRef = new ByteSegmentRef();
+      readPos = new Position();
+    }
+
+    public void set(VectorMapJoinFastValueStore valueStore, long valueRefWord) {
+      // LOG.info("VectorMapJoinFastValueStore set valueRefWord " + Long.toHexString(valueRefWord));
+
+      this.valueStore = valueStore;
+      this.valueRefWord = valueRefWord;
+
+      hasRows = true;
+      isSingleRow = ((valueRefWord & IsLastFlag.flagOnMask) != 0);
+      cappedCount =
+          (int) ((valueRefWord & CappedCount.bitMask) >> CappedCount.bitShift);
+      // Position to beginning.
+      haveReadCurrent = false;
+      readIndex = 0;
+      isEof = false;
+    }
+
+    @Override
+    public boolean hasRows() {
+      return hasRows;
+    }
+
+    @Override
+    public boolean isSingleRow() {
+      if (!hasRows) {
+        return false;
+      }
+
+      return isSingleRow;
+    }
+
+    @Override
+    public boolean isCappedCountAvailable() {
+      return true;
+    }
+
+    @Override
+    public int cappedCount() {
+      if (!hasRows) {
+        return 0;
+      }
+
+      return cappedCount;
+    }
+
+    @Override
+    public ByteSegmentRef first() {
+      if (!hasRows) {
+        return null;
+      }
+
+      // Position to beginning.
+      haveReadCurrent = false;
+      readIndex = 0;
+      isEof = false;
+
+      return internalRead();
+    }
+
+    @Override
+    public ByteSegmentRef next() {
+      if (!hasRows) {
+        return null;
+      }
+
+      return internalRead();
+    }
+
+
+    public ByteSegmentRef internalRead() {
+
+      long absoluteValueOffset;
+
+      int valueLength;
+
+      if (readIndex == 0) {
+        /*
+         * Extract information from reference word from slot table.
+         */
+        absoluteValueOffset =
+            (valueRefWord & AbsoluteValueOffset.bitMask);
+
+        // Position before the last written value.
+        valueStore.writeBuffers.setReadPoint(absoluteValueOffset, readPos);
+
+        if (isSingleRow) {
+          isNextEof = true;
+
+          valueLength =
+              (int) ((valueRefWord & SmallValueLength.bitMask) >> SmallValueLength.bitShift);
+          boolean isValueLengthSmall = (valueLength != SmallValueLength.allBitsOn);
+          if (!isValueLengthSmall) {
+            // And, if current value is big we must read it.
+            valueLength = valueStore.writeBuffers.readVInt(readPos);
+          }
+        } else {
+          isNextEof = false;
+
+          // 2nd and beyond records have a relative offset word at the beginning.
+          long relativeOffsetWord = valueStore.writeBuffers.readVLong(readPos);
+
+          long relativeOffset =
+              (relativeOffsetWord & NextRelativeValueOffset.bitMask) >> NextRelativeValueOffset.bitShift;
+
+          nextAbsoluteValueOffset = absoluteValueOffset - relativeOffset;
+
+          isNextLast = ((relativeOffsetWord & IsNextValueLastFlag.flagOnMask) != 0);
+          isNextValueLengthSmall =
+              ((relativeOffsetWord & IsNextValueLengthSmallFlag.flagOnMask) != 0);
+        }
+
+        valueLength =
+            (int) ((valueRefWord & SmallValueLength.bitMask) >> SmallValueLength.bitShift);
+        boolean isValueLengthSmall = (valueLength != SmallValueLength.allBitsOn);
+        if (!isValueLengthSmall) {
+          // And, if current value is big we must read it.
+          valueLength = valueStore.writeBuffers.readVInt(readPos);
+        }
+
+        // 2nd and beyond have the next value's small length in the current record.
+        if (isNextValueLengthSmall) {
+          nextSmallValueLength = valueStore.writeBuffers.readVInt(readPos);
+        } else {
+          nextSmallValueLength = -1;
+        }
+
+      } else {
+        if (isNextEof) {
+          return null;
+        }
+
+        absoluteValueOffset =  nextAbsoluteValueOffset;
+
+        // Position before the last written value.
+        valueStore.writeBuffers.setReadPoint(absoluteValueOffset, readPos);
+
+        if (isNextLast) {
+          isNextEof = true;
+
+          if (isNextValueLengthSmall) {
+            valueLength = nextSmallValueLength;
+          } else {
+            valueLength = (int) valueStore.writeBuffers.readVLong(readPos);
+          }
+        } else {
+          isNextEof = false;
+
+          // 2nd and beyond records have a relative offset word at the beginning.
+          long relativeOffsetWord = valueStore.writeBuffers.readVLong(readPos);
+
+          // Read current value's big length now, if necessary.
+          if (isNextValueLengthSmall) {
+            valueLength = nextSmallValueLength;
+          } else {
+            valueLength = (int) valueStore.writeBuffers.readVLong(readPos);
+          }
+
+          long relativeOffset =
+              (relativeOffsetWord & NextRelativeValueOffset.bitMask) >> NextRelativeValueOffset.bitShift;
+
+          nextAbsoluteValueOffset = absoluteValueOffset - relativeOffset;
+
+          isNextLast = ((relativeOffsetWord & IsNextValueLastFlag.flagOnMask) != 0);
+          isNextValueLengthSmall =
+              ((relativeOffsetWord & IsNextValueLengthSmallFlag.flagOnMask) != 0);
+          if (isNextValueLengthSmall) {
+            // TODO: Write readVInt
+            nextSmallValueLength = (int) valueStore.writeBuffers.readVLong(readPos);
+          } else {
+            nextSmallValueLength = -1;
+          }
+        }
+      }
+
+      // Our reading is positioned to the value.
+      valueStore.writeBuffers.getByteSegmentRefToCurrent(byteSegmentRef, valueLength, readPos);
+
+      readIndex++;
+      return byteSegmentRef;
+    }
+
+    @Override
+    public boolean isEof() {
+      if (!hasRows) {
+        return true;
+      }
+      return isEof;
+    }
+
+    @Override
+    public void forget() {
+    }
+
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder();
+      sb.append("(" + super.toString() + ", ");
+      sb.append("cappedCount " + cappedCount() + ")");
+      return sb.toString();
+    }
+  }
+
+  /**
+   * Bit-length fields within a 64-bit (long) value reference.
+   *
+   * Lowest field: An absolute byte offset the value in the WriteBuffers.
+   *
+   * 2nd field: For short values, the length of the value.  Otherwise, a special constant
+   * indicating a big value whose length is stored with the value.
+   *
+   * 3rd field: A value count, up to a limit (a cap).  Have a count helps the join result
+   * algorithms determine which optimization to use for M x N result cross products.
+   * A special constant indicates if the value count is >= the cap.
+   *
+   * Last field: an bit indicating whether there is only one value.
+   */
+
+  // Lowest field.
+  private final class AbsoluteValueOffset {
+    private static final int bitLength = 40;
+    private static final long allBitsOn = (1L << bitLength) - 1;
+    private static final long bitMask = allBitsOn;
+
+    // Make it a power of 2.
+    private static final long maxSize = 1L << (bitLength - 2);
+  }
+
+  private final class SmallValueLength {
+    private static final int bitLength = 10;
+    private static final int allBitsOn = (1 << bitLength) - 1;
+    private static final int threshold = allBitsOn;  // Lower this for big value testing.
+    private static final int bitShift = AbsoluteValueOffset.bitLength;
+    private static final long bitMask = ((long) allBitsOn) << bitShift;
+    private static final long allBitsOnBitShifted = ((long) allBitsOn) << bitShift;
+  }
+
+  private final class CappedCount {
+    private static final int bitLength = 10;
+    private static final int allBitsOn = (1 << bitLength) - 1;
+    private static final int limit = allBitsOn;
+    private static final int bitShift =  SmallValueLength.bitShift + SmallValueLength.bitLength;
+    private static final long bitMask = ((long) allBitsOn) << bitShift;
+  }
+
+  private final class IsLastFlag {
+    private static final int bitShift = CappedCount.bitShift + CappedCount.bitLength;;
+    private static final long flagOnMask = 1L << bitShift;
+  }
+
+  // This bit should not be on for valid value references.  We use -1 for a no value marker.
+  private final class IsInvalidFlag {
+    private static final int bitShift = 63;
+    private static final long flagOnMask = 1L << bitShift;
+  }
+
+  /**
+   * Relative Offset Word stored at the beginning of all but the last value that has a
+   * relative offset and 2 flags.
+   *
+   * We put the flags at the low end of the word so the variable length integer will
+   * encode smaller.
+   *
+   * First bit is a flag indicating if the next value (not the current value) has a small length.
+   * When the first value is added and it has a small length, that length is stored in the
+   * value reference and not with the value.  So, when we have multiple values, we need a way to
+   * know to keep the next value's small length with the current value.
+   *
+   * Second bit is a flag indicating if the next value (not the current value) is the last value.
+   *
+   * The relative offset *backwards* to the next value.
+   */
+
+  private final class IsNextValueLengthSmallFlag {
+    private static final int bitLength = 1;
+    private static final long flagOnMask = 1L;
+  }
+
+  private final class IsNextValueLastFlag {
+    private static final int bitLength = 1;
+    private static final int bitShift = IsNextValueLengthSmallFlag.bitLength;
+    private static final long flagOnMask = 1L << bitShift;
+  }
+
+  private final class NextRelativeValueOffset {
+    private static final int bitLength = 40;
+    private static final long allBitsOn = (1L << bitLength) - 1;
+    private static final int bitShift = IsNextValueLastFlag.bitShift + IsNextValueLastFlag.bitLength;
+    private static final long bitMask = allBitsOn << bitShift;
+  }
+
+  public long addFirst(byte[] valueBytes, int valueStart, int valueLength) {
+
+    // First value is written without: next relative offset, next value length, is next value last
+    // flag, is next value length small flag, etc.
+
+    /*
+     * We build up the Value Reference Word we will return that will be kept by the caller.
+     */
+
+    long valueRefWord = IsLastFlag.flagOnMask;
+
+    valueRefWord |= ((long) 1 << CappedCount.bitShift);
+
+    long newAbsoluteOffset;
+    if (valueLength < SmallValueLength.threshold) {
+
+      // Small case: Just write the value bytes only.
+
+      if (valueLength == 0) {
+        // We don't write a first empty value.
+        // Get an offset to reduce the relative offset later if there are more than 1 value.
+        newAbsoluteOffset = writeBuffers.getWritePoint();
+      } else {
+        newAbsoluteOffset = writeBuffers.getWritePoint();
+        writeBuffers.write(valueBytes, valueStart, valueLength);
+      }
+
+      // The caller remembers the small value length.
+      valueRefWord |= ((long) valueLength) << SmallValueLength.bitShift;
+    } else {
+
+      // Big case: write the length as a VInt and then the value bytes.
+
+      newAbsoluteOffset = writeBuffers.getWritePoint();
+
+      writeBuffers.writeVInt(valueLength);
+      writeBuffers.write(valueBytes, valueStart, valueLength);
+
+      // Use magic length value to indicate big.
+      valueRefWord |= SmallValueLength.allBitsOnBitShifted;
+    }
+
+    // LOG.info("VectorMapJoinFastValueStore addFirst valueLength " + valueLength + " newAbsoluteOffset " + newAbsoluteOffset + " valueRefWord " + Long.toHexString(valueRefWord));
+
+    // The lower bits are the absolute value offset.
+    valueRefWord |= newAbsoluteOffset;
+
+    return valueRefWord;
+  }
+
+  public long addMore(long oldValueRef, byte[] valueBytes, int valueStart, int valueLength) {
+
+    if ((oldValueRef & IsInvalidFlag.flagOnMask) != 0) {
+      throw new RuntimeException("Invalid optimized hash table reference");
+    }
+    /*
+     * Extract information about the old value.
+     */
+    long oldAbsoluteValueOffset =
+        (oldValueRef & AbsoluteValueOffset.bitMask);
+    int oldSmallValueLength =
+        (int) ((oldValueRef & SmallValueLength.bitMask) >> SmallValueLength.bitShift);
+    boolean isOldValueLengthSmall = (oldSmallValueLength != SmallValueLength.allBitsOn);
+    int oldCappedCount =
+        (int) ((oldValueRef & CappedCount.bitMask) >> CappedCount.bitShift);
+    boolean isOldValueLast =
+        ((oldValueRef & IsLastFlag.flagOnMask) != 0);
+
+    // LOG.info("VectorMapJoinFastValueStore addMore isOldValueLast " + isOldValueLast + " oldSmallValueLength " + oldSmallValueLength + " oldAbsoluteValueOffset " + oldAbsoluteValueOffset + " oldValueRef " + Long.toHexString(oldValueRef));
+
+    /*
+     * Write information about the old value (which becomes our next) at the beginning
+     * of our new value.
+     */
+    long newAbsoluteOffset = writeBuffers.getWritePoint();
+
+    long relativeOffsetWord = 0;
+    if (isOldValueLengthSmall) {
+      relativeOffsetWord |= IsNextValueLengthSmallFlag.flagOnMask;
+    }
+    if (isOldValueLast) {
+      relativeOffsetWord |= IsNextValueLastFlag.flagOnMask;
+    }
+    int newCappedCount = oldCappedCount;
+    if (newCappedCount < CappedCount.limit) {
+      newCappedCount++;
+    }
+    long relativeOffset = newAbsoluteOffset - oldAbsoluteValueOffset;
+    relativeOffsetWord |= (relativeOffset << NextRelativeValueOffset.bitShift);
+
+    writeBuffers.writeVLong(relativeOffsetWord);
+
+    // When the next value is small it was not recorded with the old (i.e. next) value and we
+    // have to remember it.
+    if (isOldValueLengthSmall) {
+      writeBuffers.writeVInt(oldSmallValueLength);
+    }
+
+    // Now, we have written all information about the next value, work on the *new* value.
+
+    long newValueRef = ((long) newCappedCount) << CappedCount.bitShift;
+    boolean isNewValueSmall = (valueLength < SmallValueLength.threshold);
+    if (!isNewValueSmall) {
+      // Use magic value to indicating we are writing the big value length.
+      newValueRef |= ((long) SmallValueLength.allBitsOn << SmallValueLength.bitShift);
+      writeBuffers.writeVInt(valueLength);
+    } else {
+      // Caller must remember small value length.
+      newValueRef |= ((long) valueLength) << SmallValueLength.bitShift;
+    }
+    writeBuffers.write(valueBytes, valueStart, valueLength);
+
+    // The lower bits are the absolute value offset.
+    newValueRef |=  newAbsoluteOffset;
+
+    // LOG.info("VectorMapJoinFastValueStore addMore valueLength " + valueLength + " newAbsoluteOffset " + newAbsoluteOffset + " newValueRef " + Long.toHexString(newValueRef));
+
+    return newValueRef;
+  }
+
+  public VectorMapJoinFastValueStore(int writeBuffersSize) {
+    writeBuffers = new WriteBuffers(writeBuffersSize, AbsoluteValueOffset.maxSize);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashMap.java
new file mode 100644
index 0000000..512db1b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashMap.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+
+/*
+ * The interface for a single byte array key hash map lookup method.
+ */
+public interface VectorMapJoinBytesHashMap
+        extends VectorMapJoinBytesHashTable, VectorMapJoinHashMap {
+
+  /*
+   * Lookup a byte array key in the hash map.
+   *
+   * @param keyBytes
+   *         A byte array containing the key within a range.
+   * @param keyStart
+   *         The offset the beginning of the key.
+   * @param keyLength
+   *         The length of the key.
+   * @param hashMapResult
+   *         The object to receive small table value(s) information on a MATCH.
+   *         Or, for SPILL, it has information on where to spill the big table row.
+   *
+   * @return
+   *         Whether the lookup was a match, no match, or spill (the partition with the key
+   *         is currently spilled).
+   */
+  JoinUtil.JoinResult lookup(byte[] keyBytes, int keyStart, int keyLength,
+          VectorMapJoinHashMapResult hashMapResult) throws IOException;
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashMultiSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashMultiSet.java
new file mode 100644
index 0000000..196403d
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashMultiSet.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+
+/*
+ * The interface for a single byte array key hash multi-set contains method.
+ */
+public interface VectorMapJoinBytesHashMultiSet
+        extends VectorMapJoinBytesHashTable, VectorMapJoinHashMultiSet {
+
+  /*
+   * Lookup an byte array key in the hash multi-set.
+   *
+   * @param keyBytes
+   *         A byte array containing the key within a range.
+   * @param keyStart
+   *         The offset the beginning of the key.
+   * @param keyLength
+   *         The length of the key.
+   * @param hashMultiSetResult
+   *         The object to receive small table value(s) information on a MATCH.
+   *         Or, for SPILL, it has information on where to spill the big table row.
+   *
+   * @return
+   *         Whether the lookup was a match, no match, or spilled (the partition with the key
+   *         is currently spilled).
+   */
+  JoinUtil.JoinResult contains(byte[] keyBytes, int keyStart, int keyLength,
+          VectorMapJoinHashMultiSetResult hashMultiSetResult) throws IOException;
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashSet.java
new file mode 100644
index 0000000..a0c93e5
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashSet.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+
+/*
+ * The interface for a single byte array key hash multi-set contains method.
+ */
+public interface VectorMapJoinBytesHashSet
+        extends VectorMapJoinBytesHashTable, VectorMapJoinHashSet {
+
+  /*
+   * Lookup an byte array key in the hash set.
+   *
+   * @param keyBytes
+   *         A byte array containing the key within a range.
+   * @param keyStart
+   *         The offset the beginning of the key.
+   * @param keyLength
+   *         The length of the key.
+   * @param hashSetResult
+   *         The object to receive small table value(s) information on a MATCH.
+   *         Or, for SPILL, it has information on where to spill the big table row.
+   *
+   * @return
+   *         Whether the lookup was a match, no match, or spilled (the partition with the key
+   *         is currently spilled).
+   */
+  JoinUtil.JoinResult contains(byte[] keyBytes, int keyStart, int keyLength,
+          VectorMapJoinHashSetResult hashSetResult) throws IOException;
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashTable.java
new file mode 100644
index 0000000..7494e1d
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinBytesHashTable.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+/*
+ * Interface for a vector map join hash table (which could be a hash map, hash multi-set, or
+ * hash set) for a single byte array key.
+ */
+public interface VectorMapJoinBytesHashTable extends VectorMapJoinHashTable {
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMap.java
new file mode 100644
index 0000000..7abe2c8
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMap.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+/*
+ * The root interface for a vector map join hash map.
+ */
+public interface VectorMapJoinHashMap extends VectorMapJoinHashTable {
+
+  /*
+   * @return A new hash map result implementation specific object.
+   *
+   * The object can be used to access the values when there is a match, or
+   * access spill information when the partition with the key is currently spilled.
+   */
+  VectorMapJoinHashMapResult createHashMapResult();
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMapResult.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMapResult.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMapResult.java
new file mode 100644
index 0000000..fa6dedb
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMapResult.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+import org.apache.hadoop.hive.serde2.WriteBuffers.ByteSegmentRef;
+
+/*
+ * Abstract class for a hash map result.  For reading the values, one-by-one.
+ */
+public abstract class VectorMapJoinHashMapResult extends VectorMapJoinHashTableResult {
+
+  /**
+   * @return Whether there are any rows (i.e. true for match).
+   */
+  public abstract boolean hasRows();
+
+  /**
+   * @return Whether there is 1 value row.
+   */
+  public abstract boolean isSingleRow();
+
+  /**
+   * @return Whether there is a capped count available from cappedCount.
+   */
+  public abstract boolean isCappedCountAvailable();
+
+  /**
+   * @return The count of values, up to a arbitrary cap limit.  When available, the capped
+   *         count can be used to make decisions on how to optimally generate join results.
+   */
+  public abstract int cappedCount();
+
+  /**
+   * @return A reference to the first value, or null if there are no values.
+   */
+  public abstract ByteSegmentRef first();
+
+  /**
+   * @return The next value, or null if there are no more values to be read.
+   */
+  public abstract ByteSegmentRef next();
+
+  /**
+   * @return Whether reading is at the end.
+   */
+  public abstract boolean isEof();
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMultiSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMultiSet.java
new file mode 100644
index 0000000..210597d
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMultiSet.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+public interface VectorMapJoinHashMultiSet extends VectorMapJoinHashTable {
+
+  /*
+   * @return A new hash multi-set result implementation specific object.
+   *
+   * The object can be used to access the *count* of values when the key is contained in the
+   * multi-set, or access spill information when the partition with the key is currently spilled.
+   */
+  VectorMapJoinHashMultiSetResult createHashMultiSetResult();
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMultiSetResult.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMultiSetResult.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMultiSetResult.java
new file mode 100644
index 0000000..0728f78
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashMultiSetResult.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+/*
+ * Abstract class for a hash multi-set result.
+ */
+public abstract class VectorMapJoinHashMultiSetResult extends VectorMapJoinHashTableResult {
+
+  protected long count;
+
+  /*
+   * @return The multi-set count for the lookup key.
+   */
+  public long count() {
+   return count;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashSet.java
new file mode 100644
index 0000000..a26f997
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashSet.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+/*
+ * The root interface for a vector map join hash set.
+ */
+public interface VectorMapJoinHashSet extends VectorMapJoinHashTable {
+
+  /*
+   * @return A new hash set result implementation specific object.
+   *
+   * The object can be used to access access spill information when the partition with the key
+   * is currently spilled.
+   */
+  VectorMapJoinHashSetResult createHashSetResult();
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashSetResult.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashSetResult.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashSetResult.java
new file mode 100644
index 0000000..467c4c1
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashSetResult.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+/*
+ * Abstract class for a hash set result.
+ */
+public abstract class VectorMapJoinHashSetResult extends VectorMapJoinHashTableResult {
+
+  // Nothing currently available for hash sets.
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java
new file mode 100644
index 0000000..7e219ec
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.io.BytesWritable;
+
+/*
+ * Root interface for a vector map join hash table (which could be a hash map, hash multi-set, or
+ * hash set).
+ */
+public interface VectorMapJoinHashTable {
+
+
+  /*
+   * @param currentKey
+   *          The current key.
+   * @param currentValue
+   *          The current value.
+   */
+  void putRow(BytesWritable currentKey, BytesWritable currentValue)
+      throws SerDeException, HiveException, IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTableResult.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTableResult.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTableResult.java
new file mode 100644
index 0000000..ce598e3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTableResult.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+
+/*
+ * Root abstract class for a hash table result.
+ */
+public abstract class VectorMapJoinHashTableResult {
+
+  private JoinUtil.JoinResult joinResult;
+
+  private int spillPartitionId;
+
+  public VectorMapJoinHashTableResult() {
+    joinResult = JoinUtil.JoinResult.NOMATCH;
+    spillPartitionId = -1;
+  }
+
+  /**
+   * @return The join result from the most recent hash map match, or hash multi-set / set contains
+   *         call.
+   */
+  public JoinUtil.JoinResult joinResult() {
+    return joinResult;
+  }
+
+  /**
+   * Set the current join result.
+   * @param joinResult
+   *               The new join result.
+   */
+  public void setJoinResult(JoinUtil.JoinResult joinResult) {
+    this.joinResult = joinResult;
+  }
+
+  /**
+   * Forget about the most recent hash table lookup or contains call.
+   */
+  public void forget() {
+    joinResult = JoinUtil.JoinResult.NOMATCH;
+  }
+
+  /**
+   * Set the spill partition id.
+   */
+  public void setSpillPartitionId(int spillPartitionId) {
+    this.spillPartitionId = spillPartitionId;
+  }
+
+  /**
+   * @return The Hybrid Grace spill partition id.
+   */
+  public int spillPartitionId() {
+    return spillPartitionId;
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("joinResult " + joinResult.name());
+    return sb.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashMap.java
new file mode 100644
index 0000000..f180d02
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashMap.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+
+/*
+ * The interface for a single long key hash map lookup method.
+ */
+public interface VectorMapJoinLongHashMap
+          extends VectorMapJoinLongHashTable, VectorMapJoinHashMap {
+
+  /*
+   * Lookup an long in the hash map.
+   *
+   * @param key
+   *         The long key.
+   * @param hashMapResult
+   *         The object to receive small table value(s) information on a MATCH.
+   *         Or, for SPILL, it has information on where to spill the big table row.
+   *
+   * @return
+   *         Whether the lookup was a match, no match, or spilled (the partition with the key
+   *         is currently spilled).
+   */
+  JoinUtil.JoinResult lookup(long key, VectorMapJoinHashMapResult hashMapResult) throws IOException;
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashMultiSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashMultiSet.java
new file mode 100644
index 0000000..7477584
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashMultiSet.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+
+/*
+ * The interface for a single long key hash multi-set contains method.
+ */
+public interface VectorMapJoinLongHashMultiSet
+          extends VectorMapJoinLongHashTable, VectorMapJoinHashMultiSet {
+
+  /*
+   * Lookup an long in the hash multi-set.
+   *
+   * @param key
+   *         The long key.
+   * @param hashMultiSetResult
+   *         The object to receive small table value(s) information on a MATCH.
+   *         Or, for SPILL, it has information on where to spill the big table row.
+   *
+   * @return
+   *         Whether the lookup was a match, no match, or spilled (the partition with the key
+   *         is currently spilled).
+   */
+  JoinUtil.JoinResult contains(long key, VectorMapJoinHashMultiSetResult hashMultiSetResult) throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashSet.java
new file mode 100644
index 0000000..8c28bff
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashSet.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+
+/*
+ * The interface adds the single long key hash multi-set contains method.
+ */
+public interface VectorMapJoinLongHashSet
+          extends VectorMapJoinLongHashTable, VectorMapJoinHashSet {
+
+  /*
+   * Lookup an long in the hash set.
+   *
+   * @param key
+   *         The long key.
+   * @param hashSetResult
+   *         The object to receive small table value(s) information on a MATCH.
+   *         Or, for SPILL, it has information on where to spill the big table row.
+   *
+   * @return
+   *         Whether the lookup was a match, no match, or spilled (the partition with the key
+   *         is currently spilled).
+   */
+  JoinUtil.JoinResult contains(long key, VectorMapJoinHashSetResult hashSetResult) throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashTable.java
new file mode 100644
index 0000000..046a403
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinLongHashTable.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+/*
+ * Interface for a vector map join hash table (which could be a hash map, hash multi-set, or
+ * hash set) for a single long.
+ */
+public interface VectorMapJoinLongHashTable extends VectorMapJoinHashTable {
+
+  boolean useMinMax();
+  long min();
+  long max();
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinTableContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinTableContainer.java
new file mode 100644
index 0000000..09631e4
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinTableContainer.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable;
+
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
+
+public interface VectorMapJoinTableContainer extends MapJoinTableContainer {
+
+  VectorMapJoinHashTable vectorMapJoinHashTable();
+
+  // com.esotericsoftware.kryo.io.Output getHybridBigTableSpillOutput(int partitionId);
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedCreateHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedCreateHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedCreateHashTable.java
new file mode 100644
index 0000000..5442834
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedCreateHashTable.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.optimized;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.ReusableGetAdaptor;
+import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc;
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableKeyType;
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableKind;
+
+/**
+ */
+public class VectorMapJoinOptimizedCreateHashTable {
+
+  private static final Log LOG = LogFactory.getLog(VectorMapJoinOptimizedCreateHashTable.class.getName());
+
+  public static VectorMapJoinOptimizedHashTable createHashTable(MapJoinDesc desc,
+          MapJoinTableContainer mapJoinTableContainer) {
+
+    MapJoinKey refKey = mapJoinTableContainer.getAnyKey();
+    ReusableGetAdaptor hashMapRowGetter = mapJoinTableContainer.createGetter(refKey);
+
+    boolean isOuterJoin = !desc.isNoOuterJoin();
+    VectorMapJoinDesc vectorDesc = desc.getVectorDesc();
+    HashTableKind hashTableKind = vectorDesc.hashTableKind();
+    HashTableKeyType hashTableKeyType = vectorDesc.hashTableKeyType();
+    boolean minMaxEnabled = vectorDesc.minMaxEnabled();
+
+    VectorMapJoinOptimizedHashTable hashTable = null;
+
+    switch (hashTableKeyType) {
+    case BOOLEAN:
+    case BYTE:
+    case SHORT:
+    case INT:
+    case LONG:
+      switch (hashTableKind) {
+      case HASH_MAP:
+        hashTable = new VectorMapJoinOptimizedLongHashMap(
+                  minMaxEnabled, isOuterJoin, hashTableKeyType,
+                  mapJoinTableContainer, hashMapRowGetter);
+        break;
+      case HASH_MULTISET:
+        hashTable = new VectorMapJoinOptimizedLongHashMultiSet(
+                  minMaxEnabled, isOuterJoin, hashTableKeyType,
+                  mapJoinTableContainer, hashMapRowGetter);
+        break;
+      case HASH_SET:
+        hashTable = new VectorMapJoinOptimizedLongHashSet(
+                  minMaxEnabled, isOuterJoin, hashTableKeyType,
+                  mapJoinTableContainer, hashMapRowGetter);
+        break;
+      }
+      break;
+
+    case STRING:
+      switch (hashTableKind) {
+      case HASH_MAP:
+        hashTable = new VectorMapJoinOptimizedStringHashMap(
+                  isOuterJoin,
+                  mapJoinTableContainer, hashMapRowGetter);
+        break;
+      case HASH_MULTISET:
+        hashTable = new VectorMapJoinOptimizedStringHashMultiSet(
+                  isOuterJoin,
+                  mapJoinTableContainer, hashMapRowGetter);
+        break;
+      case HASH_SET:
+        hashTable = new VectorMapJoinOptimizedStringHashSet(
+                  isOuterJoin,
+                  mapJoinTableContainer, hashMapRowGetter);
+        break;
+      }
+      break;
+
+    case MULTI_KEY:
+      switch (hashTableKind) {
+      case HASH_MAP:
+        hashTable = new VectorMapJoinOptimizedMultiKeyHashMap(
+                  isOuterJoin,
+                  mapJoinTableContainer, hashMapRowGetter);
+        break;
+      case HASH_MULTISET:
+        hashTable = new VectorMapJoinOptimizedMultiKeyHashMultiSet(
+                  isOuterJoin,
+                  mapJoinTableContainer, hashMapRowGetter);
+        break;
+      case HASH_SET:
+        hashTable = new VectorMapJoinOptimizedMultiKeyHashSet(
+                  isOuterJoin,
+                  mapJoinTableContainer, hashMapRowGetter);
+        break;
+      }
+      break;
+    }
+    return hashTable;
+  }
+
+  /*
+  @Override
+  public com.esotericsoftware.kryo.io.Output getHybridBigTableSpillOutput(int partitionId) {
+
+    HybridHashTableContainer ht = (HybridHashTableContainer) mapJoinTableContainer;
+
+    HashPartition hp = ht.getHashPartitions()[partitionId];
+
+    return hp.getMatchfileOutput();
+  }
+  */
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashMap.java
new file mode 100644
index 0000000..e56c821
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashMap.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.optimized;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+import org.apache.hadoop.hive.ql.exec.persistence.BytesBytesMultiHashMap;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.ReusableGetAdaptor;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinBytesHashMap;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMapResult;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashTableResult;
+import org.apache.hadoop.hive.serde2.WriteBuffers.ByteSegmentRef;
+
+public class VectorMapJoinOptimizedHashMap
+          extends VectorMapJoinOptimizedHashTable
+          implements VectorMapJoinBytesHashMap {
+
+  @Override
+  public VectorMapJoinHashMapResult createHashMapResult() {
+    return new HashMapResult();
+  }
+
+  public static class HashMapResult extends VectorMapJoinHashMapResult {
+
+    private BytesBytesMultiHashMap.Result bytesBytesMultiHashMapResult;
+
+    public HashMapResult() {
+      super();
+      bytesBytesMultiHashMapResult = new BytesBytesMultiHashMap.Result();
+    }
+
+    public BytesBytesMultiHashMap.Result bytesBytesMultiHashMapResult() {
+      return bytesBytesMultiHashMapResult;
+    }
+
+    @Override
+    public boolean hasRows() {
+      return (joinResult() == JoinUtil.JoinResult.MATCH);
+    }
+
+    @Override
+    public boolean isSingleRow() {
+      if (joinResult() != JoinUtil.JoinResult.MATCH) {
+        throw new RuntimeException("HashMapResult is not a match");
+      }
+      return bytesBytesMultiHashMapResult.isSingleRow();
+    }
+
+    @Override
+    public boolean isCappedCountAvailable() {
+      return false;
+    }
+
+    @Override
+    public int cappedCount() {
+      return 0;
+    }
+
+    @Override
+    public ByteSegmentRef first() {
+      if (joinResult() != JoinUtil.JoinResult.MATCH) {
+        throw new RuntimeException("HashMapResult is not a match");
+      }
+      return bytesBytesMultiHashMapResult.first();
+    }
+
+    @Override
+    public ByteSegmentRef next() {
+      return bytesBytesMultiHashMapResult.next();
+    }
+
+    @Override
+    public boolean isEof() {
+      return bytesBytesMultiHashMapResult.isEof();
+    }
+
+    @Override
+    public void forget() {
+      bytesBytesMultiHashMapResult.forget();
+      super.forget();
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder();
+      sb.append("(" + super.toString() + ", ");
+      sb.append("isSingleRow " + (joinResult() == JoinUtil.JoinResult.MATCH ? isSingleRow() : "<none>") + ")");
+      return sb.toString();
+    }
+ }
+
+  @Override
+  public JoinUtil.JoinResult lookup(byte[] keyBytes, int keyOffset, int keyLength,
+     VectorMapJoinHashMapResult hashMapResult) throws IOException {
+
+    HashMapResult implementationHashMapResult = (HashMapResult) hashMapResult;
+
+    JoinUtil.JoinResult joinResult =
+        doLookup(keyBytes, keyOffset, keyLength,
+            implementationHashMapResult.bytesBytesMultiHashMapResult(),
+            (VectorMapJoinHashTableResult) hashMapResult);
+
+    return joinResult;
+  }
+
+  public VectorMapJoinOptimizedHashMap(
+      MapJoinTableContainer originalTableContainer, ReusableGetAdaptor hashMapRowGetter) {
+    super(originalTableContainer, hashMapRowGetter);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashMultiSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashMultiSet.java
new file mode 100644
index 0000000..34de7e1
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashMultiSet.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.optimized;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+import org.apache.hadoop.hive.ql.exec.persistence.BytesBytesMultiHashMap;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.ReusableGetAdaptor;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinBytesHashMultiSet;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMultiSetResult;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashTableResult;
+import org.apache.hadoop.hive.serde2.WriteBuffers.ByteSegmentRef;
+
+public class VectorMapJoinOptimizedHashMultiSet
+        extends VectorMapJoinOptimizedHashTable
+        implements VectorMapJoinBytesHashMultiSet {
+
+  @Override
+  public VectorMapJoinHashMultiSetResult createHashMultiSetResult() {
+    return new HashMultiSetResult();
+  }
+
+  public static class HashMultiSetResult extends VectorMapJoinHashMultiSetResult {
+
+    private BytesBytesMultiHashMap.Result bytesBytesMultiHashMapResult;
+
+    private boolean haveCount;
+
+    public HashMultiSetResult() {
+      super();
+      bytesBytesMultiHashMapResult = new BytesBytesMultiHashMap.Result();
+    }
+
+    public BytesBytesMultiHashMap.Result bytesBytesMultiHashMapResult() {
+      return bytesBytesMultiHashMapResult;
+    }
+
+    /*
+     * @return The multi-set count for the lookup key.
+     */
+    @Override
+    public long count() {
+      if (!haveCount) {
+        if (bytesBytesMultiHashMapResult.isSingleRow()) {
+          count = 1;
+        } else {
+          count = 0;
+          ByteSegmentRef byteSegmentRef = bytesBytesMultiHashMapResult.first();
+          while (byteSegmentRef != null) {
+            count++;
+            byteSegmentRef = bytesBytesMultiHashMapResult.next();
+          }
+        }
+        haveCount = true;
+      }
+      return count;
+    }
+
+    @Override
+    public void forget() {
+      haveCount = false;
+      bytesBytesMultiHashMapResult.forget();
+      super.forget();
+    }
+  }
+
+  @Override
+  public JoinUtil.JoinResult contains(byte[] keyBytes, int keyOffset, int keyLength,
+          VectorMapJoinHashMultiSetResult hashMultiSetResult) throws IOException {
+
+    HashMultiSetResult implementationHashMultiSetResult = (HashMultiSetResult) hashMultiSetResult;
+
+    JoinUtil.JoinResult joinResult =
+        doLookup(keyBytes, keyOffset, keyLength,
+            implementationHashMultiSetResult.bytesBytesMultiHashMapResult(),
+            (VectorMapJoinHashTableResult) hashMultiSetResult);
+
+    return joinResult;
+  }
+
+  public VectorMapJoinOptimizedHashMultiSet(
+      MapJoinTableContainer originalTableContainer, ReusableGetAdaptor hashMapRowGetter) {
+    super(originalTableContainer, hashMapRowGetter);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java
new file mode 100644
index 0000000..93a89d7
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.optimized;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+import org.apache.hadoop.hive.ql.exec.persistence.BytesBytesMultiHashMap;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.ReusableGetAdaptor;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinBytesHashSet;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashSetResult;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashTableResult;
+
+public class VectorMapJoinOptimizedHashSet
+        extends VectorMapJoinOptimizedHashTable
+        implements VectorMapJoinBytesHashSet {
+
+  @Override
+  public VectorMapJoinHashSetResult createHashSetResult() {
+    return new HashSetResult();
+  }
+
+  public static class HashSetResult extends VectorMapJoinHashSetResult {
+
+    private BytesBytesMultiHashMap.Result bytesBytesMultiHashMapResult;
+
+    public HashSetResult() {
+      super();
+      bytesBytesMultiHashMapResult = new BytesBytesMultiHashMap.Result();
+    }
+
+    public BytesBytesMultiHashMap.Result bytesBytesMultiHashMapResult() {
+      return bytesBytesMultiHashMapResult;
+    }
+
+    @Override
+    public void forget() {
+      bytesBytesMultiHashMapResult.forget();
+      super.forget();
+    }
+  }
+
+  @Override
+  public JoinUtil.JoinResult contains(byte[] keyBytes, int keyOffset, int keyLength,
+          VectorMapJoinHashSetResult hashSetResult) throws IOException {
+
+    HashSetResult implementationHashSetResult = (HashSetResult) hashSetResult;
+
+    JoinUtil.JoinResult joinResult =
+        doLookup(keyBytes, keyOffset, keyLength,
+            implementationHashSetResult.bytesBytesMultiHashMapResult(),
+            (VectorMapJoinHashTableResult) hashSetResult);
+
+    return joinResult;
+  }
+
+  public VectorMapJoinOptimizedHashSet(
+      MapJoinTableContainer originalTableContainer, ReusableGetAdaptor hashMapRowGetter) {
+    super(originalTableContainer, hashMapRowGetter);
+  }
+}
\ No newline at end of file


Mime
View raw message