hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject svn commit: r1652314 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpo...
Date Thu, 15 Jan 2015 23:57:14 GMT
Author: sershe
Date: Thu Jan 15 23:57:14 2015
New Revision: 1652314

URL: http://svn.apache.org/r1652314
Log:
HIVE-9331 : get rid of pre-optimized-hashtable memory optimizations (Sergey Shelukhin, reviewed
by Ashutosh Chauhan)

Added:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java
Removed:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/LazyFlatRowContainer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyBytes.java
Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
    hive/trunk/ql/src/test/queries/clientpositive/mapjoin_decimal.q
    hive/trunk/ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Jan 15 23:57:14
2015
@@ -699,13 +699,6 @@ public class HiveConf extends Configurat
     HIVEMAPJOINUSEOPTIMIZEDTABLE("hive.mapjoin.optimized.hashtable", true,
         "Whether Hive should use memory-optimized hash table for MapJoin. Only works on Tez,\n"
+
         "because memory-optimized hashtable cannot be serialized."),
-    HIVEMAPJOINUSEOPTIMIZEDKEYS("hive.mapjoin.optimized.keys", true,
-        "Whether MapJoin hashtable should use optimized (size-wise), keys, allowing the table
to take less\n" +
-        "memory. Depending on key, the memory savings for entire table can be 5-15% or so."),
-    HIVEMAPJOINLAZYHASHTABLE("hive.mapjoin.lazy.hashtable", true,
-        "Whether MapJoin hashtable should deserialize values on demand. Depending on how
many values in\n" +
-        "the table the join will actually touch, it can save a lot of memory by not creating
objects for\n" +
-        "rows that are not needed. If all rows are needed obviously there's no gain."),
     HIVEHASHTABLEWBSIZE("hive.mapjoin.optimized.hashtable.wbsize", 10 * 1024 * 1024,
         "Optimized hashtable (see hive.mapjoin.optimized.hashtable) uses a chain of buffers
to\n" +
         "store data. This is one buffer size. HT may be slightly faster if this is larger,
but for small\n" +

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java?rev=1652314&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java
(added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/FlatRowContainer.java
Thu Jan 15 23:57:14 2015
@@ -0,0 +1,401 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.persistence;
+
+import java.io.ObjectOutputStream;
+import java.util.AbstractCollection;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Writable;
+
+@SuppressWarnings("deprecation")
+public class FlatRowContainer extends AbstractCollection<Object>
+    implements MapJoinRowContainer, AbstractRowContainer.RowIterator<List<Object>>,
List<Object> {
+  private static final Object[] EMPTY_OBJECT_ARRAY = new Object[0];
+  private static final int UNKNOWN = Integer.MAX_VALUE;
+
+  private static Log LOG = LogFactory.getLog(FlatRowContainer.class);
+
+  /**
+   * In lazy mode, 0s element contains context for deserialization and all the other
+   * elements contains byte arrays to be deserialized. After deserialization, the array
+   * contains row count * row size elements - a matrix of rows stored.
+   */
+  private Object[] array;
+  /**
+   * This is kind of tricky. UNKNOWN number means unknown. Other positive numbers represent
+   * row length (see array javadoc). Non-positive numbers mean row length is zero (thus,
+   * array is empty); they represent (negated) number of rows (for joins w/o projections).
+   */
+  private int rowLength = UNKNOWN;
+  private byte aliasFilter = (byte) 0xff;
+  private boolean isAliasFilterSet = true; // by default assume no filter tag so we are good
+
+  public FlatRowContainer() {
+    this.array = EMPTY_OBJECT_ARRAY;
+  }
+
+  /** Called when loading the hashtable. */
+  public void add(MapJoinObjectSerDeContext context,
+      BytesWritable value) throws HiveException {
+    SerDe serde = context.getSerDe();
+    isAliasFilterSet = !context.hasFilterTag(); // has tag => need to set later
+    if (rowLength == UNKNOWN) {
+      try {
+        rowLength = ObjectInspectorUtils.getStructSize(serde.getObjectInspector());
+      } catch (SerDeException ex) {
+        throw new HiveException("Get structure size error", ex);
+      }
+      if (rowLength == 0) {
+        array = EMPTY_OBJECT_ARRAY;
+      }
+    }
+    if (rowLength > 0) {
+      int rowCount = (array.length / rowLength);
+      listRealloc(array.length + rowLength);
+      read(serde, value, rowCount);
+    } else {
+      --rowLength; // see rowLength javadoc
+    }
+  }
+
+  // Implementation of AbstractRowContainer and assorted methods
+
+  @Override
+  public void addRow(List<Object> t) throws HiveException {
+    LOG.debug("Add is called with " + t.size() + " objects");
+    // This is not called when building HashTable; we don't expect it to be called ever.
+    int offset = prepareForAdd(t.size());
+    if (offset < 0) return;
+    for (int i = 0; i < t.size(); ++i) {
+      this.array[offset + i] = t.get(i);
+    }
+  }
+
+  @Override
+  public void addRow(Object[] value) throws HiveException {
+    LOG.debug("Add is called with " + value.length + " objects");
+    // This is not called when building HashTable; we don't expect it to be called ever.
+    int offset = prepareForAdd(value.length);
+    if (offset < 0) return;
+    System.arraycopy(value, 0, this.array, offset, value.length);
+  }
+
+  private int prepareForAdd(int len) throws HiveException {
+    if (rowLength < 0) {
+      if (len != 0) {
+        throw new HiveException("Different size rows: 0 and " + len);
+      }
+      --rowLength; // see rowLength javadoc
+      return -1;
+    }
+    if (rowLength != len) {
+      throw new HiveException("Different size rows: " + rowLength + " and " + len);
+    }
+    int oldLen = this.array.length;
+    listRealloc(oldLen + len);
+    return oldLen;
+  }
+
+  @Override
+  public void write(MapJoinObjectSerDeContext valueContext, ObjectOutputStream out) {
+    throw new UnsupportedOperationException(this.getClass().getName() + " cannot be serialized");
+  }
+
+  @Override
+  public AbstractRowContainer.RowIterator<List<Object>> rowIter() throws HiveException
{
+    if (array.length == rowLength) {
+      // optimize for common case - just one row for a key, container acts as iterator
+      return this;
+    }
+    return rowLength > 0 ? new RowIterator() : new EmptyRowIterator(-rowLength);
+  }
+
+  @Override
+  public List<Object> first() throws HiveException {
+    if (array.length != rowLength) {
+      throw new AssertionError("Incorrect iterator usage, not single-row");
+    }
+    return this; // optimize for common case - just one row for a key, container acts as
row
+  }
+
+  @Override
+  public List<Object> next() {
+    return null; // single-row case, there's no next
+  }
+
+  /** Iterator for row length 0. */
+  private static class EmptyRowIterator implements AbstractRowContainer.RowIterator<List<Object>>
{
+    private static final List<Object> EMPTY_ROW = new ArrayList<Object>();
+    private int rowCount;
+    public EmptyRowIterator(int rowCount) {
+      this.rowCount = rowCount;
+    }
+
+    @Override
+    public List<Object> first() throws HiveException {
+      return next();
+    }
+
+    @Override
+    public List<Object> next() throws HiveException {
+      return (--rowCount < 0) ? null : EMPTY_ROW;
+    }
+  }
+
+  /** Row iterator for non-zero-length rows. */
+  private class RowIterator implements AbstractRowContainer.RowIterator<List<Object>>
{
+    private int index = 0;
+
+    @Override
+    public List<Object> first() throws HiveException {
+      index = 0;
+      if (array.length > 0) {
+        return new ReadOnlySubList(0, rowLength);
+      }
+      return null;
+    }
+
+    @Override
+    public List<Object> next() {
+      index += rowLength;
+      if (index < array.length) {
+        return new ReadOnlySubList(index, rowLength);
+      }
+      return null;
+    }
+  }
+
+  private void read(SerDe serde, Writable writable, int rowOffset) throws HiveException {
+    try {
+      ObjectInspectorUtils.copyStructToArray(
+          serde.deserialize(writable), serde.getObjectInspector(),
+          ObjectInspectorCopyOption.WRITABLE, this.array, rowOffset * rowLength);
+    } catch (SerDeException ex) {
+      throw new HiveException("Lazy deserialize error", ex);
+    }
+  }
+
+  @Override
+  public int rowCount() throws HiveException {
+    return rowLength > 0 ? (array.length / rowLength) : -rowLength; // see rowLength javadoc
+  }
+
+  @Override
+  public void clearRows() {
+    array = EMPTY_OBJECT_ARRAY;
+    rowLength = 0;
+  }
+
+  @Override
+  public byte getAliasFilter() throws HiveException {
+    ensureAliasFilter();
+    return this.aliasFilter;
+  }
+
+  private void ensureAliasFilter() throws HiveException {
+    if (!isAliasFilterSet && rowLength > 0) {
+      for (int offset = rowLength - 1; offset < array.length; offset += rowLength) {
+        aliasFilter &= ((ShortWritable)array[offset]).get();
+      }
+    }
+    isAliasFilterSet = true;
+  }
+
+  @Override
+  public MapJoinRowContainer copy() throws HiveException {
+    FlatRowContainer result = new FlatRowContainer();
+    result.array = new Object[this.array.length];
+    System.arraycopy(this.array, 0, result.array, 0, this.array.length);
+    result.rowLength = rowLength;
+    result.aliasFilter = aliasFilter;
+    return result;
+  }
+
+  // Implementation of List<Object> and assorted methods
+
+  private void listRealloc(int length) {
+    Object[] array = new Object[length];
+    if (this.array.length > 0) {
+      System.arraycopy(this.array, 0, array, 0, this.array.length);
+    }
+    this.array = array;
+  }
+
+  @Override
+  public int size() {
+    checkSingleRow();
+    return array.length;
+  }
+
+  @Override
+  public Object get(int index) {
+    return array[index];
+  }
+
+  private class ReadOnlySubList extends AbstractList<Object> {
+    private int offset;
+    private int size;
+
+    ReadOnlySubList(int from, int size) {
+      this.offset = from;
+      this.size = size;
+    }
+
+    public Object get(int index) {
+      return array[index + offset];
+    }
+
+    public int size() {
+      return size;
+    }
+
+    public Iterator<Object> iterator() {
+      return listIterator();
+    }
+
+    public ListIterator<Object> listIterator(int index) {
+      return listIteratorInternal(offset + index, offset, offset + size);
+    }
+
+    public List<Object> subList(int fromIndex, int toIndex) {
+      return new ReadOnlySubList(offset + fromIndex, toIndex - fromIndex);
+    }
+
+    public Object[] toArray() {
+      Object[] result = new Object[size];
+      System.arraycopy(array, offset, result, 0, size);
+      return result;
+    }
+  } // end ReadOnlySubList
+
+  @Override
+  public Object[] toArray() {
+    checkSingleRow();
+    return array;
+  }
+
+  @Override
+  public Iterator<Object> iterator() {
+    return listIterator();
+  }
+
+  @Override
+  public ListIterator<Object> listIterator() {
+    return listIterator(0);
+  }
+
+  @Override
+  public ListIterator<Object> listIterator(final int index) {
+    checkSingleRow();
+    return listIteratorInternal(index, 0, array.length);
+  }
+
+  private ListIterator<Object> listIteratorInternal(
+      final int index, final int iterMinPos, final int iterMaxPos) {
+    return new ListIterator<Object>() {
+      private int pos = index - 1;
+      public int nextIndex() {
+        return pos + 1;
+      }
+      public int previousIndex() {
+        return pos - 1;
+      }
+      public boolean hasNext() {
+        return nextIndex() < iterMaxPos;
+      }
+      public boolean hasPrevious() {
+        return previousIndex() >= iterMinPos;
+      }
+      public Object next() {
+        if (!hasNext()) throw new NoSuchElementException();
+        return get(++pos);
+      }
+      public Object previous() {
+        if (!hasPrevious()) throw new NoSuchElementException();
+        return get(--pos);
+      }
+
+      public void remove() { throw new UnsupportedOperationException(); }
+      public void set(Object e) { throw new UnsupportedOperationException(); }
+      public void add(Object e) { throw new UnsupportedOperationException(); }
+    }; // end ListIterator
+  }
+
+  @Override
+  public int indexOf(Object o) {
+    checkSingleRow();
+    for (int i = 0; i < array.length; ++i) {
+      if (o == null) {
+        if (array[i] == null) return i;
+      } else {
+        if (o.equals(array[i])) return i;
+      }
+    }
+    return -1;
+  }
+
+  private void checkSingleRow() throws AssertionError {
+    if (array.length != rowLength) {
+      throw new AssertionError("Incorrect list usage, not single-row");
+    }
+  }
+
+  @Override
+  public int lastIndexOf(Object o) {
+    checkSingleRow();
+    for (int i = array.length - 1; i >= 0; --i) {
+      if (o == null) {
+        if (array[i] == null) return i;
+      } else {
+        if (o.equals(array[i])) return i;
+      }
+    }
+    return -1;
+  }
+
+  @Override
+  public List<Object> subList(int fromIndex, int toIndex) {
+    checkSingleRow();
+    return new ReadOnlySubList(fromIndex, toIndex - fromIndex);
+  }
+
+  public boolean addAll(int index, Collection<? extends Object> c) {
+    throw new UnsupportedOperationException();
+  }
+  public Object set(int index, Object element) { throw new UnsupportedOperationException();
}
+  public void add(int index, Object element) { throw new UnsupportedOperationException();
}
+  public Object remove(int index) { throw new UnsupportedOperationException(); }
+}
+

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
Thu Jan 15 23:57:14 2015
@@ -58,8 +58,6 @@ public class HashMapWrapper extends Abst
   private static final float LOADFACTOR = 0.75f;
   private final HashMap<MapJoinKey, MapJoinRowContainer> mHash; // main memory HashMap
   private MapJoinKey lastKey = null;
-  private final boolean useLazyRows;
-  private final boolean useOptimizedKeys;
   private Output output = new Output(0); // Reusable output for serialization
 
   public HashMapWrapper(Map<String, String> metaData) {
@@ -67,30 +65,24 @@ public class HashMapWrapper extends Abst
     int threshold = Integer.parseInt(metaData.get(THESHOLD_NAME));
     float loadFactor = Float.parseFloat(metaData.get(LOAD_NAME));
     mHash = new HashMap<MapJoinKey, MapJoinRowContainer>(threshold, loadFactor);
-    useLazyRows = useOptimizedKeys = false;
   }
 
   public HashMapWrapper() {
     this(HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT.defaultFloatVal,
         HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD.defaultIntVal,
-        HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR.defaultFloatVal, false, false, -1);
+        HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR.defaultFloatVal, -1);
   }
 
   public HashMapWrapper(Configuration hconf, long keyCount) {
     this(HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLEKEYCOUNTADJUSTMENT),
         HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEHASHTABLETHRESHOLD),
-        HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR),
-        HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINLAZYHASHTABLE),
-        HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDKEYS), keyCount);
+        HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEHASHTABLELOADFACTOR), keyCount);
   }
 
-  private HashMapWrapper(float keyCountAdj, int threshold, float loadFactor,
-      boolean useLazyRows, boolean useOptimizedKeys, long keyCount) {
+  private HashMapWrapper(float keyCountAdj, int threshold, float loadFactor, long keyCount)
{
     super(createConstructorMetaData(threshold, loadFactor));
     threshold = calculateTableSize(keyCountAdj, threshold, loadFactor, keyCount);
     mHash = new HashMap<MapJoinKey, MapJoinRowContainer>(threshold, loadFactor);
-    this.useLazyRows = useLazyRows;
-    this.useOptimizedKeys = useOptimizedKeys;
   }
 
   public static int calculateTableSize(
@@ -131,21 +123,14 @@ public class HashMapWrapper extends Abst
   public MapJoinKey putRow(MapJoinObjectSerDeContext keyContext, Writable currentKey,
       MapJoinObjectSerDeContext valueContext, Writable currentValue)
           throws SerDeException, HiveException {
-    // We pass key in as reference, to find out quickly if optimized keys can be used.
-    // However, we do not reuse the object since we are putting them into the hashmap.
-    // Later, we don't create optimized keys in MapJoin if hash map doesn't have optimized
keys.
-    if (lastKey == null && !useOptimizedKeys) {
-      lastKey = new MapJoinKeyObject();
-    }
-
-    lastKey = MapJoinKey.read(output, lastKey, keyContext, currentKey, false);
-    LazyFlatRowContainer values = (LazyFlatRowContainer)get(lastKey);
+    MapJoinKey key = MapJoinKey.read(output, keyContext, currentKey);
+    FlatRowContainer values = (FlatRowContainer)get(key);
     if (values == null) {
-      values = new LazyFlatRowContainer();
-      put(lastKey, values);
+      values = new FlatRowContainer();
+      put(key, values);
     }
-    values.add(valueContext, (BytesWritable)currentValue, useLazyRows);
-    return lastKey;
+    values.add(valueContext, (BytesWritable)currentValue);
+    return key;
   }
 
   @Override

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java Thu
Jan 15 23:57:14 2015
@@ -57,22 +57,11 @@ public abstract class MapJoinKey {
   public abstract boolean hasAnyNulls(int fieldCount, boolean[] nullsafes);
 
   @SuppressWarnings("deprecation")
-  public static MapJoinKey read(Output output, MapJoinKey key,
-      MapJoinObjectSerDeContext context, Writable writable, boolean mayReuseKey)
-      throws SerDeException, HiveException {
+  public static MapJoinKey read(Output output, MapJoinObjectSerDeContext context,
+      Writable writable) throws SerDeException, HiveException {
     SerDe serde = context.getSerDe();
     Object obj = serde.deserialize(writable);
-    boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
-    if (useOptimized || key == null) {
-      byte[] structBytes = serialize(output, obj, serde.getObjectInspector(), !useOptimized);
-      if (structBytes != null) {
-        return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes);
-      } else if (useOptimized) {
-        throw new SerDeException(
-            "Failed to serialize " + obj + " even though optimized keys are used");
-      }
-    }
-    MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject();
+    MapJoinKeyObject result = new MapJoinKeyObject();
     result.read(serde.getObjectInspector(), obj);
     return result;
   }
@@ -98,35 +87,6 @@ public abstract class MapJoinKey {
     SUPPORTED_PRIMITIVES.add(PrimitiveCategory.CHAR);
   }
 
-  private static byte[] serialize(Output byteStream,
-      Object obj, ObjectInspector oi, boolean checkTypes) throws HiveException {
-    if (null == obj || !(oi instanceof StructObjectInspector)) {
-      return null; // not supported
-    }
-    StructObjectInspector soi = (StructObjectInspector)oi;
-    List<? extends StructField> fields = soi.getAllStructFieldRefs();
-    int size = fields.size();
-    if (size > 8) {
-      return null; // not supported
-    } else if (size == 0) {
-      return EMPTY_BYTE_ARRAY; // shortcut for null keys
-    }
-    Object[] fieldData = new Object[size];
-    List<ObjectInspector> fieldOis = new ArrayList<ObjectInspector>(size);
-    for (int i = 0; i < size; ++i) {
-      StructField field = fields.get(i);
-      ObjectInspector foi = field.getFieldObjectInspector();
-      if (checkTypes && !isSupportedField(foi)) {
-        return null;
-      }
-      fieldData[i] = soi.getStructFieldData(obj, field);
-      fieldOis.add(foi);
-    }
-
-    byteStream = serializeRow(byteStream, fieldData, fieldOis, null);
-    return Arrays.copyOf(byteStream.getData(), byteStream.getLength());
-  }
-
   public static boolean isSupportedField(ObjectInspector foi) {
     if (foi.getCategory() != Category.PRIMITIVE) return false; // not supported
     PrimitiveCategory pc = ((PrimitiveObjectInspector)foi).getPrimitiveCategory();
@@ -136,19 +96,6 @@ public abstract class MapJoinKey {
 
   public static MapJoinKey readFromVector(Output output, MapJoinKey key, Object[] keyObject,
       List<ObjectInspector> keyOIs, boolean mayReuseKey) throws HiveException {
-    boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
-    if (useOptimized || key == null) {
-      if (keyObject.length <= 8) {
-        output = serializeRow(output, keyObject, keyOIs, null);
-        return MapJoinKeyBytes.fromBytes(key, mayReuseKey,
-            Arrays.copyOf(output.getData(), output.getLength()));
-      }
-      if (useOptimized) {
-        throw new HiveException(
-            "Failed to serialize " + Arrays.toString(keyObject) +
-                " even though optimized keys are used");
-      }
-    }
     MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject();
     result.setKeyObjects(keyObject);
     return result;
@@ -178,32 +125,11 @@ public abstract class MapJoinKey {
 
   public static MapJoinKey readFromRow(Output output, MapJoinKey key, Object[] keyObject,
       List<ObjectInspector> keyFieldsOI, boolean mayReuseKey) throws HiveException
{
-    boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
-    if (useOptimized || key == null) {
-      if (keyObject.length <= 8) {
-        byte[] structBytes;
-        if (keyObject.length == 0) {
-          structBytes = EMPTY_BYTE_ARRAY; // shortcut for null keys
-        } else {
-          output = serializeRow(output, keyObject, keyFieldsOI, null);
-          structBytes = Arrays.copyOf(output.getData(), output.getLength());
-        }
-        return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes);
-      }
-      if (useOptimized) {
-        throw new HiveException(
-            "Failed to serialize " + Arrays.toString(keyObject) +
-                " even though optimized keys are used");
-      }
-    }
     MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject();
     result.readFromRow(keyObject, keyFieldsOI);
     return result;
   }
 
-  private static final Log LOG = LogFactory.getLog(MapJoinKey.class);
-
-
   /**
    * Serializes row to output.
    * @param byteStream Output to reuse. Can be null, in that case a new one would be created.
@@ -228,8 +154,4 @@ public abstract class MapJoinKey {
     }
     return byteStream;
   }
-
-  private static boolean useOptimizedKeyBasedOnPrev(MapJoinKey key) {
-    return (key != null) && (key instanceof MapJoinKeyBytes);
-  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java Thu Jan
15 23:57:14 2015
@@ -57,8 +57,6 @@ public class HashTableLoader implements
   private ExecMapperContext context;
   private Configuration hconf;
   private MapJoinDesc desc;
-  private MapJoinKey lastKey = null;
-  private int rowCount = 0;
 
   @Override
   public void init(ExecMapperContext context, Configuration hconf, MapJoinOperator joinOp)
{
@@ -111,8 +109,7 @@ public class HashTableLoader implements
             : new HashMapWrapper(hconf, keyCount);
 
         while (kvReader.next()) {
-          rowCount++;
-          lastKey = tableContainer.putRow(keyCtx, (Writable)kvReader.getCurrentKey(),
+          tableContainer.putRow(keyCtx, (Writable)kvReader.getCurrentKey(),
               valCtx, (Writable)kvReader.getCurrentValue());
         }
 

Modified: hive/trunk/ql/src/test/queries/clientpositive/mapjoin_decimal.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/mapjoin_decimal.q?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/mapjoin_decimal.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/mapjoin_decimal.q Thu Jan 15 23:57:14 2015
@@ -29,11 +29,6 @@ explain
 select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec);
 
 set hive.mapjoin.optimized.hashtable=false;
-set hive.mapjoin.optimized.keys=false;
-
-select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec);
-
-set hive.mapjoin.optimized.keys=true;
 
 select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec);
 

Modified: hive/trunk/ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/spark/mapjoin_decimal.q.out Thu Jan 15 23:57:14
2015
@@ -381,119 +381,3 @@ POSTHOOK: Input: default@t2
 9	9
 9	9
 9	9
-PREHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t1
-PREHOOK: Input: default@t2
-#### A masked pattern was here ####
-POSTHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t1
-POSTHOOK: Input: default@t2
-#### A masked pattern was here ####
-14	14
-14	14
-14	14
-14	14
-14	14
-14	14
-14	14
-14	14
-14	14
-17	17
-17	17
-17	17
-17	17
-17	17
-17	17
-17	17
-17	17
-17	17
-17	17
-45	45
-45	45
-45	45
-45	45
-45	45
-6	6
-6	6
-6	6
-6	6
-6	6
-6	6
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-70	70
-70	70
-70	70
-70	70
-70	70
-70	70
-70	70
-79	79
-79	79
-79	79
-79	79
-79	79
-79	79
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9

Modified: hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out?rev=1652314&r1=1652313&r2=1652314&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out Thu Jan 15 23:57:14
2015
@@ -374,119 +374,3 @@ POSTHOOK: Input: default@t2
 9	9
 9	9
 9	9
-PREHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec)
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t1
-PREHOOK: Input: default@t2
-#### A masked pattern was here ####
-POSTHOOK: query: select t1.dec, t2.dec from t1 join t2 on (t1.dec=t2.dec)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t1
-POSTHOOK: Input: default@t2
-#### A masked pattern was here ####
-14	14
-14	14
-14	14
-14	14
-14	14
-14	14
-14	14
-14	14
-14	14
-17	17
-17	17
-17	17
-17	17
-17	17
-17	17
-17	17
-17	17
-17	17
-17	17
-45	45
-45	45
-45	45
-45	45
-45	45
-6	6
-6	6
-6	6
-6	6
-6	6
-6	6
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-62	62
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-64	64
-70	70
-70	70
-70	70
-70	70
-70	70
-70	70
-70	70
-79	79
-79	79
-79	79
-79	79
-79	79
-79	79
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-89	89
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9
-9	9



Mime
View raw message