hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [12/17] hive git commit: HIVE-9824 : LLAP: Native Vectorization of Map Join (Matt McCline, reviewed by Sergey Shelukhin, Vikram Dixit K)
Date Wed, 22 Apr 2015 22:51:02 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java
new file mode 100644
index 0000000..9f20fdc
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinBytesHashSet;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashSetResult;
+import org.apache.hadoop.io.BytesWritable;
+
+/*
+ * An single byte array value hash multi-set optimized for vector map join.
+ */
+public abstract class VectorMapJoinFastBytesHashSet
+        extends VectorMapJoinFastBytesHashTable
+        implements VectorMapJoinBytesHashSet {
+
+  private static final Log LOG = LogFactory.getLog(VectorMapJoinFastBytesHashSet.class);
+
+  @Override
+  public VectorMapJoinHashSetResult createHashSetResult() {
+    return new VectorMapJoinFastHashSet.HashSetResult();
+  }
+
+  @Override
+  public void assignSlot(int slot, byte[] keyBytes, int keyStart, int keyLength,
+          long hashCode, boolean isNewKey, BytesWritable currentValue) {
+
+    int tripleIndex = 3 * slot;
+    if (isNewKey) {
+      // First entry.
+      slotTriples[tripleIndex] = keyStore.add(keyBytes, keyStart, keyLength);
+      slotTriples[tripleIndex + 1] = hashCode;
+      slotTriples[tripleIndex + 2] = 1;    // Existence
+      keysAssigned++;
+    }
+  }
+
+  @Override
+  public JoinUtil.JoinResult contains(byte[] keyBytes, int keyStart, int keyLength,
+          VectorMapJoinHashSetResult hashSetResult) {
+
+    VectorMapJoinFastHashSet.HashSetResult optimizedHashSetResult =
+        (VectorMapJoinFastHashSet.HashSetResult) hashSetResult;
+
+    optimizedHashSetResult.forget();
+
+    long hashCode = VectorMapJoinFastBytesHashUtil.hashKey(keyBytes, keyStart, keyLength);
+    long existance = findReadSlot(keyBytes, keyStart, keyLength, hashCode);
+    JoinUtil.JoinResult joinResult;
+    if (existance == -1) {
+      joinResult = JoinUtil.JoinResult.NOMATCH;
+    } else {
+      joinResult = JoinUtil.JoinResult.MATCH;
+    }
+
+    optimizedHashSetResult.setJoinResult(joinResult);
+
+    return joinResult;
+  }
+
+  public VectorMapJoinFastBytesHashSet(
+      int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+
+    keyStore = new VectorMapJoinFastKeyStore(writeBuffersSize);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
new file mode 100644
index 0000000..594a77f
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinBytesHashTable;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.BytesWritable;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/*
+ * An single byte array value hash map optimized for vector map join.
+ */
+public abstract class VectorMapJoinFastBytesHashTable
+        extends VectorMapJoinFastHashTable
+        implements VectorMapJoinBytesHashTable {
+
+  private static final Log LOG = LogFactory.getLog(VectorMapJoinFastBytesHashTable.class);
+
+  protected VectorMapJoinFastKeyStore keyStore;
+
+  private BytesWritable testKeyBytesWritable;
+  private BytesWritable testValueBytesWritable;
+
+  @Override
+  public void putRow(BytesWritable currentKey, BytesWritable currentValue) throws HiveException, IOException {
+    // No deserialization of key(s) here -- just get reference to bytes.
+    byte[] keyBytes = currentKey.getBytes();
+    int keyLength = currentKey.getLength();
+    add(keyBytes, 0, keyLength, currentValue);
+  }
+
+  @VisibleForTesting
+  public void putRow(byte[] currentKey, byte[] currentValue) throws HiveException, IOException {
+    if (testKeyBytesWritable == null) {
+      testKeyBytesWritable = new BytesWritable();
+      testValueBytesWritable = new BytesWritable();
+    }
+    testKeyBytesWritable.set(currentKey, 0, currentKey.length);
+    testValueBytesWritable.set(currentValue, 0, currentValue.length);
+    putRow(testKeyBytesWritable, testValueBytesWritable);
+  }
+
+  protected abstract void assignSlot(int slot, byte[] keyBytes, int keyStart, int keyLength,
+          long hashCode, boolean isNewKey, BytesWritable currentValue);
+
+  public void add(byte[] keyBytes, int keyStart, int keyLength, BytesWritable currentValue) {
+
+    if (resizeThreshold <= keysAssigned) {
+      expandAndRehash();
+    }
+
+    long hashCode = VectorMapJoinFastBytesHashUtil.hashKey(keyBytes, keyStart, keyLength);
+    int intHashCode = (int) hashCode;
+    int slot = (intHashCode & logicalHashBucketMask);
+    long probeSlot = slot;
+    int i = 0;
+    boolean isNewKey;
+    while (true) {
+      int tripleIndex = 3 * slot;
+      if (slotTriples[tripleIndex] == 0) {
+        // LOG.info("VectorMapJoinFastBytesHashMap findWriteSlot slot " + slot + " tripleIndex " + tripleIndex + " empty");
+        isNewKey = true;;
+        break;
+      }
+      if (hashCode == slotTriples[tripleIndex + 1] &&
+          keyStore.equalKey(slotTriples[tripleIndex], keyBytes, keyStart, keyLength)) {
+        // LOG.info("VectorMapJoinFastBytesHashMap findWriteSlot slot " + slot + " tripleIndex " + tripleIndex + " existing");
+        isNewKey = false;
+        break;
+      }
+      // TODO
+      ++metricPutConflict;
+      // Some other key (collision) - keep probing.
+      probeSlot += (++i);
+      slot = (int) (probeSlot & logicalHashBucketMask);
+    }
+
+    if (largestNumberOfSteps < i) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Probed " + i + " slots (the longest so far) to find space");
+      }
+      largestNumberOfSteps = i;
+      // debugDumpKeyProbe(keyOffset, keyLength, hashCode, slot);
+    }
+
+    assignSlot(slot, keyBytes, keyStart, keyLength, hashCode, isNewKey, currentValue);
+
+    if (isNewKey) {
+      keysAssigned++;
+    }
+  }
+
+  private void expandAndRehash() {
+
+    int newLogicalHashBucketCount = logicalHashBucketCount * 2;
+    int newLogicalHashBucketMask = newLogicalHashBucketCount - 1;
+    int newMetricPutConflict = 0;
+    int newLargestNumberOfSteps = 0;
+
+    int newSlotTripleArraySize = newLogicalHashBucketCount * 3;
+    long[] newSlotTriples = new long[newSlotTripleArraySize];
+
+    for (int slot = 0; slot < logicalHashBucketCount; slot++) {
+      int tripleIndex = slot * 3;
+      long keyRef = slotTriples[tripleIndex];
+      if (keyRef != 0) {
+        long hashCode = slotTriples[tripleIndex + 1];
+        long valueRef = slotTriples[tripleIndex + 2];
+
+        // Copy to new slot table.
+        int intHashCode = (int) hashCode;
+        int newSlot = intHashCode & newLogicalHashBucketMask;
+        long newProbeSlot = newSlot;
+        int newTripleIndex;
+        int i = 0;
+        while (true) {
+          newTripleIndex = newSlot * 3;
+          long newKeyRef = newSlotTriples[newTripleIndex];
+          if (newKeyRef == 0) {
+            break;
+          }
+          ++newMetricPutConflict;
+          // Some other key (collision) - keep probing.
+          newProbeSlot += (++i);
+          newSlot = (int)(newProbeSlot & newLogicalHashBucketMask);
+        }
+
+        if (newLargestNumberOfSteps < i) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Probed " + i + " slots (the longest so far) to find space");
+          }
+          newLargestNumberOfSteps = i;
+          // debugDumpKeyProbe(keyOffset, keyLength, hashCode, slot);
+        }
+
+        // Use old value reference word.
+        // LOG.info("VectorMapJoinFastLongHashTable expandAndRehash key " + tableKey + " slot " + newSlot + " newPairIndex " + newPairIndex + " empty slot (i = " + i + ")");
+
+        newSlotTriples[newTripleIndex] = keyRef;
+        newSlotTriples[newTripleIndex + 1] = hashCode;
+        newSlotTriples[newTripleIndex + 2] = valueRef;
+      }
+    }
+
+    slotTriples = newSlotTriples;
+    logicalHashBucketCount = newLogicalHashBucketCount;
+    logicalHashBucketMask = newLogicalHashBucketMask;
+    metricPutConflict = newMetricPutConflict;
+    largestNumberOfSteps = newLargestNumberOfSteps;
+    resizeThreshold = (int)(logicalHashBucketCount * loadFactor);
+    metricExpands++;
+    // LOG.info("VectorMapJoinFastLongHashTable expandAndRehash new logicalHashBucketCount " + logicalHashBucketCount + " resizeThreshold " + resizeThreshold + " metricExpands " + metricExpands);
+  }
+
+  protected long findReadSlot(byte[] keyBytes, int keyStart, int keyLength, long hashCode) {
+
+    int intHashCode = (int) hashCode;
+    int slot = (intHashCode & logicalHashBucketMask);
+    long probeSlot = slot;
+    int i = 0;
+    while (true) {
+      int tripleIndex = slot * 3;
+      // LOG.info("VectorMapJoinFastBytesHashMap findReadSlot slot keyRefWord " + Long.toHexString(slotTriples[tripleIndex]) + " hashCode " + Long.toHexString(hashCode) + " entry hashCode " + Long.toHexString(slotTriples[tripleIndex + 1]) + " valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
+      if (slotTriples[tripleIndex] != 0 && hashCode == slotTriples[tripleIndex + 1]) {
+        // Finally, verify the key bytes match.
+
+        if (keyStore.equalKey(slotTriples[tripleIndex], keyBytes, keyStart, keyLength)) {
+          return slotTriples[tripleIndex + 2];
+        }
+      }
+      // Some other key (collision) - keep probing.
+      probeSlot += (++i);
+      if (i > largestNumberOfSteps) {
+        // We know we never went that far when we were inserting.
+        return -1;
+      }
+      slot = (int)(probeSlot & logicalHashBucketMask);
+    }
+  }
+
+  /*
+   * The hash table slots.  For a bytes key hash table, each slot is 3 longs and the array is
+   * 3X sized.
+   *
+   * The slot triple is 1) a non-zero reference word to the key bytes, 2) the key hash code, and
+   * 3) a non-zero reference word to the first value bytes.
+   */
+  protected long[] slotTriples;
+
+  private void allocateBucketArray() {
+    int slotTripleArraySize = 3 * logicalHashBucketCount;
+    slotTriples = new long[slotTripleArraySize];
+  }
+
+  public VectorMapJoinFastBytesHashTable(
+        int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+    allocateBucketArray();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashUtil.java
new file mode 100644
index 0000000..28f7357
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashUtil.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import org.apache.hadoop.hive.serde2.WriteBuffers;
+
+public class VectorMapJoinFastBytesHashUtil {
+
+  public static long hashKey(byte[] bytes, int start, int length) {
+    return WriteBuffers.murmurHash(bytes, start, length);
+  }
+
+  public static String displayBytes(byte[] bytes, int start, int length) {
+    StringBuilder sb = new StringBuilder();
+    for (int i = start; i < start + length; i++) {
+      char ch = (char) bytes[i];
+      if (ch < ' ' || ch > '~') {
+        sb.append(String.format("\\%03d", (int) (bytes[i] & 0xff)));
+      } else {
+        sb.append(ch);
+      }
+    }
+    return sb.toString();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java
new file mode 100644
index 0000000..b37247c
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMap.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMap;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMapResult;
+
+public abstract class VectorMapJoinFastHashMap
+          extends VectorMapJoinFastHashTable
+          implements VectorMapJoinHashMap {
+
+  @Override
+  public VectorMapJoinHashMapResult createHashMapResult() {
+    return new VectorMapJoinFastValueStore.HashMapResult();
+  }
+
+  public VectorMapJoinFastHashMap(
+        boolean isOuterJoin,
+        int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java
new file mode 100644
index 0000000..5569f6e
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashMultiSet.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMultiSet;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMultiSetResult;
+
+public abstract class VectorMapJoinFastHashMultiSet
+        extends VectorMapJoinFastHashTable implements VectorMapJoinHashMultiSet {
+
+  @Override
+  public VectorMapJoinHashMultiSetResult createHashMultiSetResult() {
+    return new HashMultiSetResult();
+  }
+
+  public static class HashMultiSetResult extends VectorMapJoinHashMultiSetResult {
+
+    HashMultiSetResult() {
+      super();
+    }
+
+    public void set(long count) {
+      this.count = count;
+    }
+  }
+
+  public VectorMapJoinFastHashMultiSet(
+        boolean isOuterJoin,
+        int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java
new file mode 100644
index 0000000..0738df3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashSet.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashSet;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashSetResult;
+
+public abstract class VectorMapJoinFastHashSet
+        extends VectorMapJoinFastHashTable implements VectorMapJoinHashSet {
+
+  @Override
+  public VectorMapJoinHashSetResult createHashSetResult() {
+    return new HashSetResult();
+  }
+
+  public static class HashSetResult extends VectorMapJoinHashSetResult {
+
+    HashSetResult() {
+      super();
+    }
+  }
+
+  public VectorMapJoinFastHashSet(
+        boolean isOuterJoin,
+        int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java
new file mode 100644
index 0000000..33e34fa
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashTable;
+
+public abstract class VectorMapJoinFastHashTable implements VectorMapJoinHashTable {
+  public static final Log LOG = LogFactory.getLog(VectorMapJoinFastHashTable.class);
+
+  protected int logicalHashBucketCount;
+  protected int logicalHashBucketMask;
+
+  protected float loadFactor;
+  protected int writeBuffersSize;
+  protected long memUsage;
+
+  protected int metricPutConflict;
+  protected int largestNumberOfSteps;
+  protected int keysAssigned;
+  protected int resizeThreshold;
+  protected int metricExpands;
+
+  private static void validateCapacity(long capacity) {
+    if (Long.bitCount(capacity) != 1) {
+      throw new AssertionError("Capacity must be a power of two");
+    }
+    if (capacity <= 0) {
+      throw new AssertionError("Invalid capacity " + capacity);
+    }
+  }
+
+  private static int nextHighestPowerOfTwo(int v) {
+    return Integer.highestOneBit(v) << 1;
+  }
+
+  public VectorMapJoinFastHashTable(
+        int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+
+    initialCapacity = (Long.bitCount(initialCapacity) == 1)
+        ? initialCapacity : nextHighestPowerOfTwo(initialCapacity);
+
+    validateCapacity(initialCapacity);
+
+    logicalHashBucketCount = initialCapacity;
+    logicalHashBucketMask = logicalHashBucketCount - 1;
+    resizeThreshold = (int)(logicalHashBucketCount * loadFactor);
+
+    this.loadFactor = loadFactor;
+    this.writeBuffersSize = writeBuffersSize;
+    this.memUsage = memUsage;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
new file mode 100644
index 0000000..92b5d40
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
+import org.apache.hadoop.hive.ql.exec.MapredContext;
+import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
+import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
+import org.apache.hadoop.hive.ql.exec.tez.TezContext;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.tez.runtime.api.Input;
+import org.apache.tez.runtime.api.LogicalInput;
+import org.apache.tez.runtime.library.api.KeyValueReader;
+
+/**
+ * HashTableLoader for Tez constructs the hashtable from records read from
+ * a broadcast edge.
+ */
+public class VectorMapJoinFastHashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTableLoader {
+
+  private static final Log LOG = LogFactory.getLog(VectorMapJoinFastHashTableLoader.class.getName());
+
+  private Configuration hconf;
+  protected MapJoinDesc desc;
+  private TezContext tezContext;
+
+  @Override
+  public void init(ExecMapperContext context, MapredContext mrContext,
+      Configuration hconf, MapJoinOperator joinOp) {
+    this.tezContext = (TezContext) mrContext;
+    this.hconf = hconf;
+    this.desc = joinOp.getConf();
+  }
+
+  @Override
+  public void load(MapJoinTableContainer[] mapJoinTables,
+      MapJoinTableContainerSerDe[] mapJoinTableSerdes, long memUsage)
+      throws HiveException {
+
+    Map<Integer, String> parentToInput = desc.getParentToInput();
+    Map<Integer, Long> parentKeyCounts = desc.getParentKeyCounts();
+
+    for (int pos = 0; pos < mapJoinTables.length; pos++) {
+      if (pos == desc.getPosBigTable()) {
+        continue;
+      }
+
+      String inputName = parentToInput.get(pos);
+      LogicalInput input = tezContext.getInput(inputName);
+
+      try {
+        input.start();
+        tezContext.getTezProcessorContext().waitForAnyInputReady(
+            Collections.<Input> singletonList(input));
+      } catch (Exception e) {
+        throw new HiveException(e);
+      }
+
+      try {
+        KeyValueReader kvReader = (KeyValueReader) input.getReader();
+
+        Long keyCountObj = parentKeyCounts.get(pos);
+        long keyCount = (keyCountObj == null) ? -1 : keyCountObj.longValue();
+
+        VectorMapJoinFastTableContainer vectorMapJoinFastTableContainer =
+                new VectorMapJoinFastTableContainer(desc, hconf, keyCount, memUsage);
+
+        while (kvReader.next()) {
+          vectorMapJoinFastTableContainer.putRow(
+              null, (BytesWritable) kvReader.getCurrentKey(),
+              null, (BytesWritable) kvReader.getCurrentValue());
+        }
+
+        vectorMapJoinFastTableContainer.seal();
+        mapJoinTables[pos] = (MapJoinTableContainer) vectorMapJoinFastTableContainer;
+
+      } catch (IOException e) {
+        throw new HiveException(e);
+      } catch (SerDeException e) {
+        throw new HiveException(e);
+      } catch (Exception e) {
+        throw new HiveException(e);
+      }
+    }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastIntHashUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastIntHashUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastIntHashUtil.java
new file mode 100644
index 0000000..a818cb2
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastIntHashUtil.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+public class VectorMapJoinFastIntHashUtil {
+
+  public static int hashKey(int key) {
+    key = ~key + (key << 15); // key = (key << 15) - key - 1;
+    key = key ^ (key >>> 12);
+    key = key + (key << 2);
+    key = key ^ (key >>> 4);
+    key = key * 2057; // key = (key + (key << 3)) + (key << 11);
+    key = key ^ (key >>> 16);
+    return key;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java
new file mode 100644
index 0000000..f2f42ee
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.serde2.WriteBuffers;
+
+// Optimized for sequential key lookup.
+
+public class VectorMapJoinFastKeyStore {
+
+  private static final Log LOG = LogFactory.getLog(VectorMapJoinFastKeyStore.class.getName());
+
+  private WriteBuffers writeBuffers;
+
+  private WriteBuffers.ByteSegmentRef byteSegmentRef;
+  private WriteBuffers.Position readPos;
+
+  /**
+   * A store for arbitrary length keys in memory.
+   *
+   * The memory is a "infinite" byte array or WriteBuffers object.
+   *
+   * We give the client a 64-bit (long) key reference to keep that has the offset within
+   * the "infinite" byte array of the key.
+   *
+   * We optimize the common case when keys are short and store the key length in the key reference
+   * word.
+   *
+   * If the key is big, the big length will be encoded as an integer at the beginning of the key
+   * followed by the big key bytes.
+   */
+
+  /**
+   * Bit-length fields within a 64-bit (long) key reference.
+   *
+   * Lowest field: An absolute byte offset the the key in the WriteBuffers.
+   *
+   * Next field: For short keys, the length of the key.  Otherwise, a special constant
+   * indicating a big key whose length is stored with the key.
+   *
+   * Last field: an always on bit to insure the key reference non-zero when the offset and
+   * length are zero.
+   */
+
+  /*
+   * The absolute offset to the beginning of the key within the WriteBuffers.
+   */
+  private final class AbsoluteKeyOffset {
+    private static final int bitLength = 40;
+    private static final long allBitsOn = (((long) 1) << bitLength) - 1;
+    private static final long bitMask = allBitsOn;
+
+    // Make it a power of 2 by backing down (i.e. the -2).
+    private static final long maxSize = ((long) 1) << (bitLength - 2);
+  }
+
+  /*
+   * The small key length.
+   *
+   * If the key is big (i.e. length >= allBitsOn), then the key length is stored in the
+   * WriteBuffers.
+   */
+  private final class SmallKeyLength {
+    private static final int bitLength = 20;
+    private static final int allBitsOn = (1 << bitLength) - 1;
+    private static final int threshold = allBitsOn;  // Lower this for big key testing.
+    private static final int bitShift = AbsoluteKeyOffset.bitLength;
+    private static final long bitMask = ((long) allBitsOn) << bitShift;
+    private static final long allBitsOnBitShifted = ((long) allBitsOn) << bitShift;
+  }
+
+  /*
+   * An always on bit to insure the key reference non-zero.
+   */
+  private final class IsNonZeroFlag {
+    private static final int bitShift = SmallKeyLength.bitShift + SmallKeyLength.bitLength;;
+    private static final long flagOnMask = ((long) 1) << bitShift;
+  }
+
+  public long add(byte[] keyBytes, int keyStart, int keyLength) {
+    boolean isKeyLengthBig = (keyLength >= SmallKeyLength.threshold);
+
+    long absoluteKeyOffset = writeBuffers.getWritePoint();
+    if (isKeyLengthBig) {
+      writeBuffers.writeVInt(keyLength);
+    }
+    writeBuffers.write(keyBytes, keyStart, keyLength);
+
+    long keyRefWord = IsNonZeroFlag.flagOnMask;
+    if (isKeyLengthBig) {
+      keyRefWord |= SmallKeyLength.allBitsOnBitShifted;
+    } else {
+      keyRefWord |= ((long) keyLength) << SmallKeyLength.bitShift;
+    }
+    keyRefWord |= absoluteKeyOffset;
+
+    // LOG.info("VectorMapJoinFastKeyStore add keyLength " + keyLength + " absoluteKeyOffset " + absoluteKeyOffset + " keyRefWord " + Long.toHexString(keyRefWord));
+    return keyRefWord;
+  }
+
+  public boolean equalKey(long keyRefWord, byte[] keyBytes, int keyStart, int keyLength) {
+
+    int storedKeyLengthLength =
+        (int) ((keyRefWord & SmallKeyLength.bitMask) >> SmallKeyLength.bitShift);
+    boolean isKeyLengthSmall = (storedKeyLengthLength != SmallKeyLength.allBitsOn);
+
+    // LOG.info("VectorMapJoinFastKeyStore equalKey keyLength " + keyLength + " isKeyLengthSmall " + isKeyLengthSmall + " storedKeyLengthLength " + storedKeyLengthLength + " keyRefWord " + Long.toHexString(keyRefWord));
+
+    if (isKeyLengthSmall && storedKeyLengthLength != keyLength) {
+      return false;
+    }
+    long absoluteKeyOffset =
+        (keyRefWord & AbsoluteKeyOffset.bitMask);
+
+    writeBuffers.setReadPoint(absoluteKeyOffset, readPos);
+    if (!isKeyLengthSmall) {
+      // Read big value length we wrote with the value.
+      storedKeyLengthLength = writeBuffers.readVInt(readPos);
+      if (storedKeyLengthLength != keyLength) {
+        // LOG.info("VectorMapJoinFastKeyStore equalKey no match big length");
+        return false;
+      }
+    }
+
+    // Our reading is positioned to the key.
+    writeBuffers.getByteSegmentRefToCurrent(byteSegmentRef, keyLength, readPos);
+
+    byte[] currentBytes = byteSegmentRef.getBytes();
+    int currentStart = (int) byteSegmentRef.getOffset();
+
+    for (int i = 0; i < keyLength; i++) {
+      if (currentBytes[currentStart + i] != keyBytes[keyStart + i]) {
+        // LOG.info("VectorMapJoinFastKeyStore equalKey no match on bytes");
+        return false;
+      }
+    }
+    // LOG.info("VectorMapJoinFastKeyStore equalKey match on bytes");
+    return true;
+  }
+
+  public VectorMapJoinFastKeyStore(int writeBuffersSize) {
+    writeBuffers = new WriteBuffers(writeBuffersSize, AbsoluteKeyOffset.maxSize);
+
+    byteSegmentRef = new WriteBuffers.ByteSegmentRef();
+    readPos = new WriteBuffers.Position();
+  }
+
+  public VectorMapJoinFastKeyStore(WriteBuffers writeBuffers) {
+    // TODO: Check if maximum size compatible with AbsoluteKeyOffset.maxSize.
+    this.writeBuffers = writeBuffers;
+
+    byteSegmentRef = new WriteBuffers.ByteSegmentRef();
+    readPos = new WriteBuffers.Position();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java
new file mode 100644
index 0000000..3a0b380
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMapResult;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinLongHashMap;
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableKeyType;
+import org.apache.hadoop.io.BytesWritable;
+
+/*
+ * An single long value map optimized for vector map join.
+ */
+public class VectorMapJoinFastLongHashMap
+             extends VectorMapJoinFastLongHashTable
+             implements VectorMapJoinLongHashMap {
+
+  public static final Log LOG = LogFactory.getLog(VectorMapJoinFastLongHashMap.class);
+
+  protected VectorMapJoinFastValueStore valueStore;
+
+  @Override
+  public VectorMapJoinHashMapResult createHashMapResult() {
+    return new VectorMapJoinFastValueStore.HashMapResult();
+  }
+
+  @Override
+  public void assignSlot(int slot, long key, boolean isNewKey, BytesWritable currentValue) {
+
+    byte[] valueBytes = currentValue.getBytes();
+    int valueLength = currentValue.getLength();
+
+    int pairIndex = 2 * slot;
+    if (isNewKey) {
+      // First entry.
+      slotPairs[pairIndex] = valueStore.addFirst(valueBytes, 0, valueLength);
+      slotPairs[pairIndex + 1] = key;
+    } else {
+      // Add another value.
+      slotPairs[pairIndex] = valueStore.addMore(slotPairs[pairIndex], valueBytes, 0, valueLength);
+    }
+  }
+
+  @Override
+  public JoinUtil.JoinResult lookup(long key, VectorMapJoinHashMapResult hashMapResult) {
+
+    VectorMapJoinFastValueStore.HashMapResult optimizedHashMapResult =
+        (VectorMapJoinFastValueStore.HashMapResult) hashMapResult;
+
+    optimizedHashMapResult.forget();
+
+    long hashCode = VectorMapJoinFastLongHashUtil.hashKey(key);
+    // LOG.info("VectorMapJoinFastLongHashMap lookup " + key + " hashCode " + hashCode);
+    long valueRef = findReadSlot(key, hashCode);
+    JoinUtil.JoinResult joinResult;
+    if (valueRef == -1) {
+      joinResult = JoinUtil.JoinResult.NOMATCH;
+    } else {
+      optimizedHashMapResult.set(valueStore, valueRef);
+
+      joinResult = JoinUtil.JoinResult.MATCH;
+    }
+
+    optimizedHashMapResult.setJoinResult(joinResult);
+
+    return joinResult;
+  }
+
+  public VectorMapJoinFastLongHashMap(
+      boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType,
+      int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(minMaxEnabled, isOuterJoin, hashTableKeyType,
+        initialCapacity, loadFactor, writeBuffersSize, memUsage);
+    valueStore = new VectorMapJoinFastValueStore(writeBuffersSize);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java
new file mode 100644
index 0000000..f9763e3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMultiSetResult;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinLongHashMultiSet;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableKeyType;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.io.BytesWritable;
+
+/*
+ * An single long value multi-set optimized for vector map join.
+ */
+public class VectorMapJoinFastLongHashMultiSet
+             extends VectorMapJoinFastLongHashTable
+             implements VectorMapJoinLongHashMultiSet {
+
+  public static final Log LOG = LogFactory.getLog(VectorMapJoinFastLongHashMultiSet.class);
+
+  @Override
+  public VectorMapJoinHashMultiSetResult createHashMultiSetResult() {
+    return new VectorMapJoinFastHashMultiSet.HashMultiSetResult();
+  }
+
+  @Override
+  public void assignSlot(int slot, long key, boolean isNewKey, BytesWritable currentValue) {
+
+    int pairIndex = 2 * slot;
+    if (isNewKey) {
+      // First entry.
+      slotPairs[pairIndex] = 1;    // Count.
+      slotPairs[pairIndex + 1] = key;
+    } else {
+      // Add another value.
+      slotPairs[pairIndex]++;
+    }
+  }
+
+
+  @Override
+  public JoinUtil.JoinResult contains(long key, VectorMapJoinHashMultiSetResult hashMultiSetResult) {
+
+    VectorMapJoinFastHashMultiSet.HashMultiSetResult optimizedHashMultiSetResult =
+        (VectorMapJoinFastHashMultiSet.HashMultiSetResult) hashMultiSetResult;
+
+    optimizedHashMultiSetResult.forget();
+
+    long hashCode = VectorMapJoinFastLongHashUtil.hashKey(key);
+    long count = findReadSlot(key, hashCode);
+    JoinUtil.JoinResult joinResult;
+    if (count == -1) {
+      joinResult = JoinUtil.JoinResult.NOMATCH;
+    } else {
+      optimizedHashMultiSetResult.set(count);
+      joinResult = JoinUtil.JoinResult.MATCH;
+    }
+
+    optimizedHashMultiSetResult.setJoinResult(joinResult);
+
+    return joinResult;
+  }
+
+  public VectorMapJoinFastLongHashMultiSet(
+      boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType,
+      int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(minMaxEnabled, isOuterJoin, hashTableKeyType,
+        initialCapacity, loadFactor, writeBuffersSize, memUsage);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java
new file mode 100644
index 0000000..cd23949
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+import org.apache.hadoop.hive.ql.exec.JoinUtil.JoinResult;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashSetResult;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinLongHashSet;
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableKeyType;
+import org.apache.hadoop.io.BytesWritable;
+
+/*
+ * An single long value multi-set optimized for vector map join.
+ */
+public class VectorMapJoinFastLongHashSet
+             extends VectorMapJoinFastLongHashTable
+             implements VectorMapJoinLongHashSet {
+
+  public static final Log LOG = LogFactory.getLog(VectorMapJoinFastLongHashSet.class);
+
+  @Override
+  public VectorMapJoinHashSetResult createHashSetResult() {
+    return new VectorMapJoinFastHashSet.HashSetResult();
+  }
+
+  @Override
+  public void assignSlot(int slot, long key, boolean isNewKey, BytesWritable currentValue) {
+
+    int pairIndex = 2 * slot;
+    if (isNewKey) {
+      // First entry.
+      slotPairs[pairIndex] = 1;    // Existence.
+      slotPairs[pairIndex + 1] = key;
+    }
+  }
+
+  @Override
+  public JoinResult contains(long key, VectorMapJoinHashSetResult hashSetResult) {
+
+    VectorMapJoinFastHashSet.HashSetResult optimizedHashSetResult =
+        (VectorMapJoinFastHashSet.HashSetResult) hashSetResult;
+
+    optimizedHashSetResult.forget();
+
+    long hashCode = VectorMapJoinFastLongHashUtil.hashKey(key);
+    long existance = findReadSlot(key, hashCode);
+    JoinUtil.JoinResult joinResult;
+    if (existance == -1) {
+      joinResult = JoinUtil.JoinResult.NOMATCH;
+    } else {
+      joinResult = JoinUtil.JoinResult.MATCH;
+    }
+
+    optimizedHashSetResult.setJoinResult(joinResult);
+
+    return joinResult;
+
+  }
+
+  public VectorMapJoinFastLongHashSet(
+      boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType,
+      int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(minMaxEnabled, isOuterJoin, hashTableKeyType,
+        initialCapacity, loadFactor, writeBuffersSize, memUsage);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java
new file mode 100644
index 0000000..b448e1f
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java
@@ -0,0 +1,284 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.JoinUtil;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinHashMap;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinLongHashMap;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.hashtable.VectorMapJoinLongHashTable;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableKeyType;
+import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.tez.runtime.library.api.KeyValueReader;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/*
+ * An single long value map optimized for vector map join.
+ */
+public abstract class VectorMapJoinFastLongHashTable
+             extends VectorMapJoinFastHashTable
+             implements VectorMapJoinLongHashTable {
+
+  public static final Log LOG = LogFactory.getLog(VectorMapJoinFastLongHashTable.class);
+
+  private HashTableKeyType hashTableKeyType;
+
+  private boolean isOuterJoin;
+
+  private BinarySortableDeserializeRead keyBinarySortableDeserializeRead;
+
+  private boolean useMinMax;
+  private long min;
+  private long max;
+
+  private BytesWritable testValueBytesWritable;
+
+  @Override
+  public boolean useMinMax() {
+    return useMinMax;
+  }
+
+  @Override
+  public long min() {
+    return min;
+  }
+
+  @Override
+  public long max() {
+    return max;
+  }
+
+  @Override
+  public void putRow(BytesWritable currentKey, BytesWritable currentValue) throws HiveException, IOException {
+    byte[] keyBytes = currentKey.getBytes();
+    int keyLength = currentKey.getLength();
+    keyBinarySortableDeserializeRead.set(keyBytes, 0, keyLength);
+    if (keyBinarySortableDeserializeRead.readCheckNull()) {
+      if (isOuterJoin) {
+        return;
+      } else {
+        // For inner join, we expect all NULL values to have been filtered out before now.
+        throw new HiveException("Unexpected NULL in map join small table");
+      }
+    }
+
+    long key = VectorMapJoinFastLongHashUtil.deserializeLongKey(
+                            keyBinarySortableDeserializeRead, hashTableKeyType);
+
+    add(key, currentValue);
+  }
+
+
+  @VisibleForTesting
+  public void putRow(long currentKey, byte[] currentValue) throws HiveException, IOException {
+    if (testValueBytesWritable == null) {
+      testValueBytesWritable = new BytesWritable();
+    }
+    testValueBytesWritable.set(currentValue, 0, currentValue.length);
+    add(currentKey, testValueBytesWritable);
+  }
+
+
+  protected abstract void assignSlot(int slot, long key, boolean isNewKey, BytesWritable currentValue);
+
+  public void add(long key, BytesWritable currentValue) {
+
+    if (resizeThreshold <= keysAssigned) {
+      expandAndRehash();
+    }
+
+    long hashCode = VectorMapJoinFastLongHashUtil.hashKey(key);
+    int intHashCode = (int) hashCode;
+    int slot = (intHashCode & logicalHashBucketMask);
+    long probeSlot = slot;
+    int i = 0;
+    boolean isNewKey;
+    while (true) {
+      int pairIndex = 2 * slot;
+      long valueRef = slotPairs[pairIndex];
+      if (valueRef == 0) {
+        // LOG.info("VectorMapJoinFastLongHashTable add key " + key + " slot " + slot + " pairIndex " + pairIndex + " empty slot (i = " + i + ")");
+        isNewKey = true;
+        break;
+      }
+      long tableKey = slotPairs[pairIndex + 1];
+      if (key == tableKey) {
+        // LOG.info("VectorMapJoinFastLongHashTable add key " + key + " slot " + slot + " pairIndex " + pairIndex + " found key (i = " + i + ")");
+        isNewKey = false;
+        break;
+      }
+      ++metricPutConflict;
+      // Some other key (collision) - keep probing.
+      probeSlot += (++i);
+      slot = (int)(probeSlot & logicalHashBucketMask);
+    }
+
+    if (largestNumberOfSteps < i) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Probed " + i + " slots (the longest so far) to find space");
+      }
+      largestNumberOfSteps = i;
+      // debugDumpKeyProbe(keyOffset, keyLength, hashCode, slot);
+    }
+
+    // LOG.info("VectorMapJoinFastLongHashTable add slot " + slot + " hashCode " + Long.toHexString(hashCode));
+
+    assignSlot(slot, key, isNewKey, currentValue);
+
+    if (isNewKey) {
+      keysAssigned++;
+      if (useMinMax) {
+        if (key < min) {
+          min = key;
+        }
+        if (key > max) {
+          max = key;
+        }
+      }
+    }
+  }
+
+  private void expandAndRehash() {
+
+    int newLogicalHashBucketCount = logicalHashBucketCount * 2;
+    int newLogicalHashBucketMask = newLogicalHashBucketCount - 1;
+    int newMetricPutConflict = 0;
+    int newLargestNumberOfSteps = 0;
+
+    int newSlotPairArraySize = newLogicalHashBucketCount * 2;
+    long[] newSlotPairs = new long[newSlotPairArraySize];
+
+    for (int slot = 0; slot < logicalHashBucketCount; slot++) {
+      int pairIndex = slot * 2;
+      long valueRef = slotPairs[pairIndex];
+      if (valueRef != 0) {
+        long tableKey = slotPairs[pairIndex + 1];
+
+        // Copy to new slot table.
+        long hashCode = VectorMapJoinFastLongHashUtil.hashKey(tableKey);
+        int intHashCode = (int) hashCode;
+        int newSlot = intHashCode & newLogicalHashBucketMask;
+        long newProbeSlot = newSlot;
+        int newPairIndex;
+        int i = 0;
+        while (true) {
+          newPairIndex = newSlot * 2;
+          long newValueRef = newSlotPairs[newPairIndex];
+          if (newValueRef == 0) {
+            break;
+          }
+          ++newMetricPutConflict;
+          // Some other key (collision) - keep probing.
+          newProbeSlot += (++i);
+          newSlot = (int)(newProbeSlot & newLogicalHashBucketMask);
+        }
+
+        if (newLargestNumberOfSteps < i) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Probed " + i + " slots (the longest so far) to find space");
+          }
+          newLargestNumberOfSteps = i;
+          // debugDumpKeyProbe(keyOffset, keyLength, hashCode, slot);
+        }
+
+        // Use old value reference word.
+        // LOG.info("VectorMapJoinFastLongHashTable expandAndRehash key " + tableKey + " slot " + newSlot + " newPairIndex " + newPairIndex + " empty slot (i = " + i + ")");
+
+        newSlotPairs[newPairIndex] = valueRef;
+        newSlotPairs[newPairIndex + 1] = tableKey;
+      }
+    }
+
+    slotPairs = newSlotPairs;
+    logicalHashBucketCount = newLogicalHashBucketCount;
+    logicalHashBucketMask = newLogicalHashBucketMask;
+    metricPutConflict = newMetricPutConflict;
+    largestNumberOfSteps = newLargestNumberOfSteps;
+    resizeThreshold = (int)(logicalHashBucketCount * loadFactor);
+    metricExpands++;
+    // LOG.info("VectorMapJoinFastLongHashTable expandAndRehash new logicalHashBucketCount " + logicalHashBucketCount + " resizeThreshold " + resizeThreshold + " metricExpands " + metricExpands);
+  }
+
+  protected long findReadSlot(long key, long hashCode) {
+
+    int intHashCode = (int) hashCode;
+    int slot = intHashCode & logicalHashBucketMask;
+
+    long probeSlot = slot;
+    int i = 0;
+    while (true) {
+      int pairIndex = 2 * slot;
+      long valueRef = slotPairs[pairIndex];
+      if (valueRef == 0) {
+        // Given that we do not delete, an empty slot means no match.
+        // LOG.info("VectorMapJoinFastLongHashTable findReadSlot key " + key + " slot " + slot + " pairIndex " + pairIndex + " empty slot (i = " + i + ")");
+        return -1;
+      }
+      long tableKey = slotPairs[pairIndex + 1];
+      if (key == tableKey) {
+        // LOG.info("VectorMapJoinFastLongHashTable findReadSlot key " + key + " slot " + slot + " pairIndex " + pairIndex + " found key (i = " + i + ")");
+        return slotPairs[pairIndex];
+      }
+      // Some other key (collision) - keep probing.
+      probeSlot += (++i);
+      if (i > largestNumberOfSteps) {
+        // LOG.info("VectorMapJoinFastLongHashTable findReadSlot returning not found");
+        // We know we never went that far when we were inserting.
+        // LOG.info("VectorMapJoinFastLongHashTable findReadSlot key " + key + " slot " + slot + " pairIndex " + pairIndex + " largestNumberOfSteps " + largestNumberOfSteps + " (i = " + i + ")");
+        return -1;
+      }
+      slot = (int)(probeSlot & logicalHashBucketMask);
+    }
+  }
+
+  /*
+   * The hash table slots.  For a long key hash table, each slot is 2 longs and the array is
+   * 2X sized.
+   *
+   * The slot pair is 1) a non-zero reference word to the first value bytes and 2) the long value.
+   */
+  protected long[] slotPairs;
+
+  private void allocateBucketArray() {
+    int slotPairArraySize = 2 * logicalHashBucketCount;
+    slotPairs = new long[slotPairArraySize];
+  }
+
+  public VectorMapJoinFastLongHashTable(
+        boolean minMaxEnabled, boolean isOuterJoin, HashTableKeyType hashTableKeyType,
+        int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+    this.isOuterJoin = isOuterJoin;
+    this.hashTableKeyType = hashTableKeyType;
+    PrimitiveTypeInfo[] primitiveTypeInfos = { TypeInfoFactory.longTypeInfo };
+    keyBinarySortableDeserializeRead = new BinarySortableDeserializeRead(primitiveTypeInfos);
+    allocateBucketArray();
+    useMinMax = minMaxEnabled;
+    min = Long.MAX_VALUE;
+    max = Long.MIN_VALUE;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashUtil.java
new file mode 100644
index 0000000..298ca61
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashUtil.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableKeyType;
+import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead;
+
+public class VectorMapJoinFastLongHashUtil {
+
+  public static long hashKey(long key) {
+    key = (~key) + (key << 21); // key = (key << 21) - key - 1;
+    key = key ^ (key >>> 24);
+    key = (key + (key << 3)) + (key << 8); // key * 265
+    key = key ^ (key >>> 14);
+    key = (key + (key << 2)) + (key << 4); // key * 21
+    key = key ^ (key >>> 28);
+    key = key + (key << 31);
+    return key;
+  }
+
+  public static long deserializeLongKey(BinarySortableDeserializeRead keyBinarySortableDeserializeRead,
+      HashTableKeyType hashTableKeyType) throws IOException {
+    long key = 0;
+    switch (hashTableKeyType) {
+    case BOOLEAN:
+      key = (keyBinarySortableDeserializeRead.readBoolean() ? 1 : 0);
+      break;
+    case BYTE:
+      key = (long) keyBinarySortableDeserializeRead.readByte();
+      break;
+    case SHORT:
+      key = (long) keyBinarySortableDeserializeRead.readShort();
+      break;
+    case INT:
+      key = (long) keyBinarySortableDeserializeRead.readInt();
+      break;
+    case LONG:
+      key = keyBinarySortableDeserializeRead.readLong();
+      break;
+    default:
+      throw new RuntimeException("Unexpected hash table key type " + hashTableKeyType.name());
+    }
+    return key;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java
new file mode 100644
index 0000000..b962475
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/*
+ * An multi-key value hash map optimized for vector map join.
+ */
+public class VectorMapJoinFastMultiKeyHashMap
+        extends VectorMapJoinFastBytesHashMap {
+
+  @VisibleForTesting
+  public VectorMapJoinFastMultiKeyHashMap(int initialCapacity, float loadFactor, int wbSize) {
+    this(false, initialCapacity, loadFactor, wbSize, -1);
+  }
+
+  public VectorMapJoinFastMultiKeyHashMap(
+        boolean isOuterJoin,
+        int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java
new file mode 100644
index 0000000..71a62fe
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+/*
+ * An multi-key value hash multi-set optimized for vector map join.
+ */
+public class VectorMapJoinFastMultiKeyHashMultiSet
+        extends VectorMapJoinFastBytesHashMultiSet {
+
+  public VectorMapJoinFastMultiKeyHashMultiSet(
+        boolean isOuterJoin,
+        int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java
new file mode 100644
index 0000000..dad3b32
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+/*
+ * An multi-key value hash set optimized for vector map join.
+ */
+public class VectorMapJoinFastMultiKeyHashSet
+        extends VectorMapJoinFastBytesHashSet {
+
+  public VectorMapJoinFastMultiKeyHashSet(
+        boolean isOuterJoin,
+        int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringCommon.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringCommon.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringCommon.java
new file mode 100644
index 0000000..5c7792f
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringCommon.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead;
+import org.apache.hadoop.hive.serde2.fast.DeserializeRead.ReadStringResults;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.BytesWritable;
+
+/*
+ * An single byte array value hash map optimized for vector map join.
+ */
+public class VectorMapJoinFastStringCommon {
+
+  private boolean isOuterJoin;
+
+  private BinarySortableDeserializeRead keyBinarySortableDeserializeRead;
+
+  private ReadStringResults readStringResults;
+
+  public void adaptPutRow(VectorMapJoinFastBytesHashTable hashTable,
+          BytesWritable currentKey, BytesWritable currentValue) throws HiveException, IOException {
+
+    byte[] keyBytes = currentKey.getBytes();
+    int keyLength = currentKey.getLength();
+    keyBinarySortableDeserializeRead.set(keyBytes, 0, keyLength);
+    if (keyBinarySortableDeserializeRead.readCheckNull()) {
+      if (isOuterJoin) {
+        return;
+      } else {
+        // For inner join, we expect all NULL values to have been filtered out before now.
+        throw new HiveException("Unexpected NULL in map join small table");
+      }
+    }
+    keyBinarySortableDeserializeRead.readString(readStringResults);
+
+    hashTable.add(readStringResults.bytes, readStringResults.start, readStringResults.length,
+        currentValue);
+  }
+
+  public VectorMapJoinFastStringCommon(boolean isOuterJoin) {
+    this.isOuterJoin = isOuterJoin;
+    PrimitiveTypeInfo[] primitiveTypeInfos = { TypeInfoFactory.stringTypeInfo };
+    keyBinarySortableDeserializeRead = new BinarySortableDeserializeRead(primitiveTypeInfos);
+    readStringResults = keyBinarySortableDeserializeRead.createReadStringResults();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java
new file mode 100644
index 0000000..c80ea89
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.BytesWritable;
+
+/*
+ * An single byte array value hash map optimized for vector map join.
+ */
+public class VectorMapJoinFastStringHashMap extends VectorMapJoinFastBytesHashMap {
+
+  private VectorMapJoinFastStringCommon stringCommon;
+
+  @Override
+  public void putRow(BytesWritable currentKey, BytesWritable currentValue) throws HiveException, IOException {
+    stringCommon.adaptPutRow(this, currentKey, currentValue);
+  }
+
+  public VectorMapJoinFastStringHashMap(
+      boolean isOuterJoin,
+      int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+    stringCommon = new VectorMapJoinFastStringCommon(isOuterJoin);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java
new file mode 100644
index 0000000..4933b16
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.BytesWritable;
+
+/*
+ * An single byte array value hash map optimized for vector map join.
+ */
+public class VectorMapJoinFastStringHashMultiSet extends VectorMapJoinFastBytesHashMultiSet {
+
+  private VectorMapJoinFastStringCommon stringCommon;
+
+  @Override
+  public void putRow(BytesWritable currentKey, BytesWritable currentValue) throws HiveException, IOException {
+    stringCommon.adaptPutRow(this, currentKey, currentValue);
+  }
+
+  public VectorMapJoinFastStringHashMultiSet(
+      boolean isOuterJoin,
+      int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+    stringCommon = new VectorMapJoinFastStringCommon(isOuterJoin);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/d47995d9/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java
new file mode 100644
index 0000000..ae8d943
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.mapjoin.fast;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.BytesWritable;
+
+/*
+ * An single byte array value hash map optimized for vector map join.
+ */
+public class VectorMapJoinFastStringHashSet extends VectorMapJoinFastBytesHashSet {
+
+  private VectorMapJoinFastStringCommon stringCommon;
+
+  @Override
+  public void putRow(BytesWritable currentKey, BytesWritable currentValue) throws HiveException, IOException {
+    stringCommon.adaptPutRow(this, currentKey, currentValue);
+  }
+
+  public VectorMapJoinFastStringHashSet(
+      boolean isOuterJoin,
+      int initialCapacity, float loadFactor, int writeBuffersSize, long memUsage) {
+    super(initialCapacity, loadFactor, writeBuffersSize, memUsage);
+    stringCommon = new VectorMapJoinFastStringCommon(isOuterJoin);
+  }
+}
\ No newline at end of file


Mime
View raw message