hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r1033824 - in /hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/exec/ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/
Date Thu, 11 Nov 2010 06:46:23 GMT
Author: namit
Date: Thu Nov 11 06:46:22 2010
New Revision: 1033824

URL: http://svn.apache.org/viewvc?rev=1033824&view=rev
Log:
HIVE-1758 Reduce memory for groupby map-side hashmap
(Siying Dong via namit)


Added:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapper.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java
Modified:
    hive/trunk/CHANGES.txt
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectsEqualComparer.java

Modified: hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hive/trunk/CHANGES.txt?rev=1033824&r1=1033823&r2=1033824&view=diff
==============================================================================
--- hive/trunk/CHANGES.txt (original)
+++ hive/trunk/CHANGES.txt Thu Nov 11 06:46:22 2010
@@ -235,6 +235,9 @@ Trunk -  Unreleased
     HIVE-1743 Compare from the end for GroupBy
     (Siying Dong via namit)
 
+    HIVE-1758 Reduce memory for groupby map-side hashmap
+    (Siying Dong via namit)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=1033824&r1=1033823&r2=1033824&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java Thu Nov 11
06:46:22 2010
@@ -44,7 +44,6 @@ import org.apache.hadoop.hive.ql.udf.gen
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer;
 import org.apache.hadoop.hive.serde2.lazy.LazyPrimitive;
 import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ListObjectsEqualComparer;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
@@ -73,7 +72,6 @@ public class GroupByOperator extends Ope
 
   protected transient ExprNodeEvaluator[] keyFields;
   protected transient ObjectInspector[] keyObjectInspectors;
-  protected transient Object[] keyObjects;
 
   protected transient ExprNodeEvaluator[][] aggregationParameterFields;
   protected transient ObjectInspector[][] aggregationParameterObjectInspectors;
@@ -99,10 +97,11 @@ public class GroupByOperator extends Ope
   protected transient ArrayList<ObjectInspector> objectInspectors;
   transient ArrayList<String> fieldNames;
 
+  transient KeyWrapperFactory keyWrapperFactory;
   // Used by sort-based GroupBy: Mode = COMPLETE, PARTIAL1, PARTIAL2,
   // MERGEPARTIAL
-  protected transient ArrayList<Object> currentKeys;
-  protected transient ArrayList<Object> newKeys;
+  protected transient KeyWrapper currentKeys;
+  protected transient KeyWrapper newKeys;
   protected transient AggregationBuffer[] aggregations;
   protected transient Object[][] aggregationsParametersLastInvoke;
 
@@ -110,7 +109,7 @@ public class GroupByOperator extends Ope
   protected transient HashMap<KeyWrapper, AggregationBuffer[]> hashAggregations;
 
   // Used by hash distinct aggregations when hashGrpKeyNotRedKey is true
-  protected transient HashSet<ArrayList<Object>> keysCurrentGroup;
+  protected transient HashSet<KeyWrapper> keysCurrentGroup;
 
   transient boolean bucketGroup;
 
@@ -134,8 +133,6 @@ public class GroupByOperator extends Ope
   // new Key ObjectInspectors are objectInspectors from the parent
   transient StructObjectInspector newKeyObjectInspector;
   transient StructObjectInspector currentKeyObjectInspector;
-  transient ListObjectsEqualComparer currentStructEqualComparer;
-  transient ListObjectsEqualComparer newKeyStructEqualComparer;
 
   /**
    * This is used to store the position and field names for variable length
@@ -192,16 +189,13 @@ public class GroupByOperator extends Ope
     keyFields = new ExprNodeEvaluator[conf.getKeys().size()];
     keyObjectInspectors = new ObjectInspector[conf.getKeys().size()];
     currentKeyObjectInspectors = new ObjectInspector[conf.getKeys().size()];
-    keyObjects = new Object[conf.getKeys().size()];
     for (int i = 0; i < keyFields.length; i++) {
       keyFields[i] = ExprNodeEvaluatorFactory.get(conf.getKeys().get(i));
       keyObjectInspectors[i] = keyFields[i].initialize(rowInspector);
       currentKeyObjectInspectors[i] = ObjectInspectorUtils
           .getStandardObjectInspector(keyObjectInspectors[i],
           ObjectInspectorCopyOption.WRITABLE);
-      keyObjects[i] = null;
     }
-    newKeys = new ArrayList<Object>(keyFields.length);
 
     // initialize unionExpr for reduce-side
     // reduce KEY has union field as the last field if there are distinct
@@ -344,7 +338,7 @@ public class GroupByOperator extends Ope
           HiveConf.ConfVars.HIVEMAPAGGRHASHMINREDUCTION);
       groupKeyIsNotReduceKey = conf.getGroupKeyNotReductionKey();
       if (groupKeyIsNotReduceKey) {
-        keysCurrentGroup = new HashSet<ArrayList<Object>>();
+        keysCurrentGroup = new HashSet<KeyWrapper>();
       }
     }
 
@@ -365,12 +359,14 @@ public class GroupByOperator extends Ope
     currentKeyObjectInspector = ObjectInspectorFactory
         .getStandardStructObjectInspector(keyNames, Arrays
         .asList(currentKeyObjectInspectors));
-    currentStructEqualComparer = new ListObjectsEqualComparer(currentKeyObjectInspectors,
currentKeyObjectInspectors);
-    newKeyStructEqualComparer = new ListObjectsEqualComparer(currentKeyObjectInspectors,
keyObjectInspectors);
 
     outputObjInspector = ObjectInspectorFactory
         .getStandardStructObjectInspector(fieldNames, objectInspectors);
 
+    keyWrapperFactory = new KeyWrapperFactory(keyFields, keyObjectInspectors, currentKeyObjectInspectors);
+
+    newKeys = keyWrapperFactory.getKeyWrapper();
+
     firstRow = true;
     // estimate the number of hash table entries based on the size of each
     // entry. Since the size of a entry
@@ -713,17 +709,9 @@ public class GroupByOperator extends Ope
     try {
       countAfterReport++;
 
-      // Compute the keys
-      newKeys.clear();
-      for (int i = 0; i < keyFields.length; i++) {
-        if (keyObjectInspectors[i] == null) {
-          keyObjectInspectors[i] = keyFields[i].initialize(rowInspector);
-        }
-        keyObjects[i] = keyFields[i].evaluate(row);
-        newKeys.add(keyObjects[i]);
-      }
-
+      newKeys.getNewKey(row, rowInspector);
       if (hashAggr) {
+        newKeys.setHashKey();
         processHashAggr(row, rowInspector, newKeys);
       } else {
         processAggr(row, rowInspector, newKeys);
@@ -743,84 +731,16 @@ public class GroupByOperator extends Ope
     }
   }
 
-  private static ArrayList<Object> deepCopyElements(Object[] keys,
-      ObjectInspector[] keyObjectInspectors,
-      ObjectInspectorCopyOption copyOption) {
-    ArrayList<Object> result = new ArrayList<Object>(keys.length);
-    deepCopyElements(keys, keyObjectInspectors, result, copyOption);
-    return result;
-  }
-
-  private static void deepCopyElements(Object[] keys,
-      ObjectInspector[] keyObjectInspectors, ArrayList<Object> result,
-      ObjectInspectorCopyOption copyOption) {
-    result.clear();
-    for (int i = 0; i < keys.length; i++) {
-      result.add(ObjectInspectorUtils.copyToStandardObject(keys[i],
-          keyObjectInspectors[i], copyOption));
-    }
-  }
-
-  class KeyWrapper {
-    int hashcode;
-    ArrayList<Object> keys;
-    // decide whether this is already in hashmap (keys in hashmap are deepcopied
-    // version, and we need to use 'currentKeyObjectInspector').
-    boolean copy = false;
-
-    KeyWrapper() {
-    }
-
-    public KeyWrapper(int hashcode, ArrayList<Object> copiedKeys) {
-      this(hashcode, copiedKeys, false);
-    }
-
-    public KeyWrapper(int hashcode, ArrayList<Object> copiedKeys,
-        boolean inHashMap) {
-      super();
-      this.hashcode = hashcode;
-      keys = copiedKeys;
-      copy = inHashMap;
-    }
-
-    @Override
-    public int hashCode() {
-      return hashcode;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-      ArrayList<Object> copied_in_hashmap = ((KeyWrapper) obj).keys;
-      if (!copy) {
-        return newKeyStructEqualComparer.areEqual(copied_in_hashmap, keys);
-      } else {
-        return currentStructEqualComparer.areEqual(copied_in_hashmap, keys);
-      }
-    }
-  }
-
-
-
-  KeyWrapper keyProber = new KeyWrapper();
-
   private void processHashAggr(Object row, ObjectInspector rowInspector,
-      ArrayList<Object> newKeys) throws HiveException {
+      KeyWrapper newKeys) throws HiveException {
     // Prepare aggs for updating
     AggregationBuffer[] aggs = null;
     boolean newEntryForHashAggr = false;
 
-    keyProber.hashcode = newKeys.hashCode();
-    // use this to probe the hashmap
-    keyProber.keys = newKeys;
-
     // hash-based aggregations
-    aggs = hashAggregations.get(keyProber);
-    ArrayList<Object> newDefaultKeys = null;
+    aggs = hashAggregations.get(newKeys);
     if (aggs == null) {
-      newDefaultKeys = deepCopyElements(keyObjects, keyObjectInspectors,
-          ObjectInspectorCopyOption.WRITABLE);
-      KeyWrapper newKeyProber = new KeyWrapper(keyProber.hashcode,
-          newDefaultKeys, true);
+      KeyWrapper newKeyProber = newKeys.copyKey();
       aggs = newAggregations();
       hashAggregations.put(newKeyProber, aggs);
       newEntryForHashAggr = true;
@@ -833,11 +753,7 @@ public class GroupByOperator extends Ope
     // Peek into the set to find out if a new grouping key is seen for the given
     // reduction key
     if (groupKeyIsNotReduceKey) {
-      if (newDefaultKeys == null) {
-        newDefaultKeys = deepCopyElements(keyObjects, keyObjectInspectors,
-            ObjectInspectorCopyOption.WRITABLE);
-      }
-      newEntryForHashAggr = keysCurrentGroup.add(newDefaultKeys);
+      newEntryForHashAggr = keysCurrentGroup.add(newKeys.copyKey());
     }
 
     // Update the aggs
@@ -859,27 +775,30 @@ public class GroupByOperator extends Ope
 
   // Non-hash aggregation
   private void processAggr(Object row, ObjectInspector rowInspector,
-      ArrayList<Object> newKeys) throws HiveException {
+      KeyWrapper newKeys) throws HiveException {
     // Prepare aggs for updating
     AggregationBuffer[] aggs = null;
     Object[][] lastInvoke = null;
+    //boolean keysAreEqual = (currentKeys != null && newKeys != null)?
+    //  newKeyStructEqualComparer.areEqual(currentKeys, newKeys) : false;
+
     boolean keysAreEqual = (currentKeys != null && newKeys != null)?
-      newKeyStructEqualComparer.areEqual(currentKeys, newKeys) : false;
+        newKeys.equals(currentKeys) : false;
 
 
     // Forward the current keys if needed for sort-based aggregation
     if (currentKeys != null && !keysAreEqual) {
-      forward(currentKeys, aggregations);
+      forward(currentKeys.getKeyArray(), aggregations);
       countAfterReport = 0;
     }
 
     // Need to update the keys?
     if (currentKeys == null || !keysAreEqual) {
       if (currentKeys == null) {
-        currentKeys = new ArrayList<Object>(keyFields.length);
+        currentKeys = newKeys.copyKey();
+      } else {
+        currentKeys.copyKey(newKeys);
       }
-      deepCopyElements(keyObjects, keyObjectInspectors, currentKeys,
-          ObjectInspectorCopyOption.WRITABLE);
 
       // Reset the aggregations
       resetAggregations(aggregations);
@@ -904,14 +823,14 @@ public class GroupByOperator extends Ope
    * @param newKeys
    *          keys for the row under consideration
    **/
-  private boolean shouldBeFlushed(ArrayList<Object> newKeys) {
+  private boolean shouldBeFlushed(KeyWrapper newKeys) {
     int numEntries = hashAggregations.size();
 
     // The fixed size for the aggregation class is already known. Get the
     // variable portion of the size every NUMROWSESTIMATESIZE rows.
     if ((numEntriesHashTable == 0) || ((numEntries % NUMROWSESTIMATESIZE) == 0)) {
       for (Integer pos : keyPositionsSize) {
-        Object key = newKeys.get(pos.intValue());
+        Object key = newKeys.getKeyArray()[pos.intValue()];
         // Ignore nulls
         if (key != null) {
           if (key instanceof LazyPrimitive) {
@@ -928,8 +847,7 @@ public class GroupByOperator extends Ope
 
       AggregationBuffer[] aggs = null;
       if (aggrPositions.size() > 0) {
-	KeyWrapper newKeyProber = new KeyWrapper(
-	    newKeys.hashCode(), newKeys);
+        KeyWrapper newKeyProber = newKeys.copyKey();
         aggs = hashAggregations.get(newKeyProber);
       }
 
@@ -975,7 +893,7 @@ public class GroupByOperator extends Ope
           .entrySet().iterator();
       while (iter.hasNext()) {
         Map.Entry<KeyWrapper, AggregationBuffer[]> m = iter.next();
-        forward(m.getKey().keys, m.getValue());
+        forward(m.getKey().getKeyArray(), m.getValue());
       }
       hashAggregations.clear();
       hashAggregations = null;
@@ -990,7 +908,7 @@ public class GroupByOperator extends Ope
     int numDel = 0;
     while (iter.hasNext()) {
       Map.Entry<KeyWrapper, AggregationBuffer[]> m = iter.next();
-      forward(m.getKey().keys, m.getValue());
+      forward(m.getKey().getKeyArray(), m.getValue());
       iter.remove();
       numDel++;
       if (numDel * 10 >= oldSize) {
@@ -1009,19 +927,20 @@ public class GroupByOperator extends Ope
    *          The keys in the record
    * @throws HiveException
    */
-  protected void forward(ArrayList<Object> keys, AggregationBuffer[] aggs)
+  protected void forward(Object[] keys, AggregationBuffer[] aggs)
       throws HiveException {
-    int totalFields = keys.size() + aggs.length;
+    int totalFields = keys.length+ aggs.length;
     if (forwardCache == null) {
       forwardCache = new Object[totalFields];
     }
-    for (int i = 0; i < keys.size(); i++) {
-      forwardCache[i] = keys.get(i);
+    for (int i = 0; i < keys.length; i++) {
+      forwardCache[i] = keys[i];
     }
     for (int i = 0; i < aggs.length; i++) {
-      forwardCache[keys.size() + i] = aggregationEvaluators[i]
+      forwardCache[keys.length + i] = aggregationEvaluators[i]
           .evaluate(aggs[i]);
     }
+
     forward(forwardCache, outputObjInspector);
   }
 
@@ -1058,7 +977,7 @@ public class GroupByOperator extends Ope
           }
 
           // create dummy keys - size 0
-          forward(new ArrayList<Object>(0), aggregations);
+          forward(new Object[0], aggregations);
         } else {
           if (hashAggregations != null) {
             LOG.warn("Begin Hash Table flush at close: size = "
@@ -1067,14 +986,15 @@ public class GroupByOperator extends Ope
             while (iter.hasNext()) {
               Map.Entry<KeyWrapper, AggregationBuffer[]> m = (Map.Entry) iter
                   .next();
-              forward(m.getKey().keys, m.getValue());
+
+              forward(m.getKey().getKeyArray(), m.getValue());
               iter.remove();
             }
             hashAggregations.clear();
           } else if (aggregations != null) {
             // sort-based aggregations
             if (currentKeys != null) {
-              forward(currentKeys, aggregations);
+              forward(currentKeys.getKeyArray(), aggregations);
             }
             currentKeys = null;
           } else {

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapper.java?rev=1033824&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapper.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapper.java Thu Nov 11 06:46:22
2010
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+public abstract class KeyWrapper {
+  abstract void getNewKey(Object row, ObjectInspector rowInspector) throws HiveException;
+  abstract void setHashKey();
+  abstract KeyWrapper copyKey();
+  abstract void copyKey(KeyWrapper oldWrapper);
+  abstract Object[] getKeyArray();
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java?rev=1033824&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java Thu Nov 11
06:46:22 2010
@@ -0,0 +1,238 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import java.util.Arrays;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectsEqualComparer;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.Text;
+
+public class KeyWrapperFactory {
+  public KeyWrapperFactory(ExprNodeEvaluator[] keyFields, ObjectInspector[] keyObjectInspectors,
+      ObjectInspector[] currentKeyObjectInspectors) {
+    this.keyFields = keyFields;
+    this.keyObjectInspectors = keyObjectInspectors;
+    this.currentKeyObjectInspectors = currentKeyObjectInspectors;
+
+  }
+
+  public KeyWrapper getKeyWrapper() {
+    if (keyFields.length == 1
+        && TypeInfoUtils.getTypeInfoFromObjectInspector(keyObjectInspectors[0]).equals(
+            TypeInfoFactory.stringTypeInfo)) {
+      assert(TypeInfoUtils.getTypeInfoFromObjectInspector(currentKeyObjectInspectors[0]).equals(
+            TypeInfoFactory.stringTypeInfo));
+      soi_new = (StringObjectInspector) keyObjectInspectors[0];
+      soi_copy = (StringObjectInspector) currentKeyObjectInspectors[0];
+      return new TextKeyWrapper(false);
+    } else {
+      currentStructEqualComparer = new ListObjectsEqualComparer(currentKeyObjectInspectors,
currentKeyObjectInspectors);
+      newKeyStructEqualComparer = new ListObjectsEqualComparer(currentKeyObjectInspectors,
keyObjectInspectors);
+      return new ListKeyWrapper(false);
+    }
+  }
+
+  transient ExprNodeEvaluator[] keyFields;
+  transient ObjectInspector[] keyObjectInspectors;
+  transient ObjectInspector[] currentKeyObjectInspectors;
+
+
+  transient ListObjectsEqualComparer currentStructEqualComparer;
+  transient ListObjectsEqualComparer newKeyStructEqualComparer;
+
+  class ListKeyWrapper extends KeyWrapper {
+    int hashcode;
+    Object[] keys;
+    // decide whether this is already in hashmap (keys in hashmap are deepcopied
+    // version, and we need to use 'currentKeyObjectInspector').
+    ListObjectsEqualComparer equalComparer;
+
+    public ListKeyWrapper(boolean isCopy) {
+      this(-1, new Object[keyFields.length], isCopy);
+    }
+
+    private ListKeyWrapper(int hashcode, Object[] copiedKeys,
+        boolean isCopy) {
+      super();
+      this.hashcode = hashcode;
+      keys = copiedKeys;
+      setEqualComparer(isCopy);
+    }
+
+    private void setEqualComparer(boolean copy) {
+      if (!copy) {
+        equalComparer = newKeyStructEqualComparer;
+      } else {
+        equalComparer = currentStructEqualComparer;
+      }
+    }
+
+    @Override
+    public int hashCode() {
+      return hashcode;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      Object[] copied_in_hashmap = ((ListKeyWrapper) obj).keys;
+      return equalComparer.areEqual(copied_in_hashmap, keys);
+    }
+
+    @Override
+    public void setHashKey() {
+      hashcode = Arrays.hashCode(keys);
+    }
+
+    @Override
+    public void getNewKey(Object row, ObjectInspector rowInspector) throws HiveException
{
+      // Compute the keys
+      for (int i = 0; i < keyFields.length; i++) {
+        keys[i]  = keyFields[i].evaluate(row);
+      }
+    }
+
+    @Override
+    public KeyWrapper copyKey() {
+      Object[] newDefaultKeys = deepCopyElements(keys, keyObjectInspectors,
+          ObjectInspectorCopyOption.WRITABLE);
+      return new ListKeyWrapper(hashcode, newDefaultKeys, true);
+    }
+
+    @Override
+    public void copyKey(KeyWrapper oldWrapper) {
+      ListKeyWrapper listWrapper = (ListKeyWrapper) oldWrapper;
+      hashcode = listWrapper.hashcode;
+      equalComparer = currentStructEqualComparer;
+      deepCopyElements(listWrapper.keys, keyObjectInspectors, keys,
+          ObjectInspectorCopyOption.WRITABLE);
+    }
+
+    @Override
+    public Object[] getKeyArray() {
+      return keys;
+    }
+
+    private Object[] deepCopyElements(Object[] keys,
+        ObjectInspector[] keyObjectInspectors,
+        ObjectInspectorCopyOption copyOption) {
+      Object[] result = new Object[keys.length];
+      deepCopyElements(keys, keyObjectInspectors, result, copyOption);
+      return result;
+    }
+
+    private void deepCopyElements(Object[] keys,
+        ObjectInspector[] keyObjectInspectors, Object[] result,
+        ObjectInspectorCopyOption copyOption) {
+      for (int i = 0; i < keys.length; i++) {
+        result[i] = ObjectInspectorUtils.copyToStandardObject(keys[i],
+            keyObjectInspectors[i], copyOption);
+      }
+    }
+  }
+
+  transient Object[] singleEleArray = new Object[1];
+  transient StringObjectInspector soi_new, soi_copy;
+
+  class TextKeyWrapper extends KeyWrapper {
+    int hashcode;
+    Object key;
+    boolean isCopy;
+
+    public TextKeyWrapper(boolean isCopy) {
+      this(-1, null, isCopy);
+    }
+
+    private TextKeyWrapper(int hashcode, Object key,
+        boolean isCopy) {
+      super();
+      this.hashcode = hashcode;
+      this.key = key;
+      this.isCopy = isCopy;
+    }
+
+    @Override
+    public int hashCode() {
+      return hashcode;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      Object obj = ((TextKeyWrapper) other).key;
+      Text t1;
+      Text t2;
+      if (isCopy) {
+        t1 =  soi_copy.getPrimitiveWritableObject(key);
+        t2 =  soi_copy.getPrimitiveWritableObject(obj);
+      } else {
+        t1 = soi_new.getPrimitiveWritableObject(key);
+        t2 = soi_copy.getPrimitiveWritableObject(obj);
+      }
+      if (t1 == null && t2 == null) {
+        return true;
+      } else if (t1 == null || t2 == null) {
+        return false;
+      } else {
+        return t1.equals(t2);
+      }
+    }
+
+    @Override
+    public void setHashKey() {
+      if (key == null) {
+        hashcode = 0;
+      } else{
+        hashcode = key.hashCode();
+      }
+    }
+
+    @Override
+    public void getNewKey(Object row, ObjectInspector rowInspector) throws HiveException
{
+      // Compute the keys
+      key = keyFields[0].evaluate(row);
+    }
+
+    @Override
+    public KeyWrapper copyKey() {
+      return new TextKeyWrapper(hashcode, ObjectInspectorUtils.copyToStandardObject(key,
+          soi_new, ObjectInspectorCopyOption.WRITABLE), true);
+    }
+
+    @Override
+    public void copyKey(KeyWrapper oldWrapper) {
+      TextKeyWrapper textWrapper = (TextKeyWrapper) oldWrapper;
+      hashcode = textWrapper.hashcode;
+      isCopy = true;
+      key = ObjectInspectorUtils.copyToStandardObject(textWrapper.key,
+          soi_new, ObjectInspectorCopyOption.WRITABLE);
+    }
+
+    @Override
+    public Object[] getKeyArray() {
+      singleEleArray[0] = key;
+      return singleEleArray;
+    }
+  }
+}

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectsEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectsEqualComparer.java?rev=1033824&r1=1033823&r2=1033824&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectsEqualComparer.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectsEqualComparer.java
Thu Nov 11 06:46:22 2010
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.hive.serde2.objectinspector;
 
-import java.util.ArrayList;
-
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
@@ -155,15 +153,15 @@ public class ListObjectsEqualComparer {
    * @param ol1
    * @return True if object in ol0 and ol1 are all identical
    */
-  public boolean areEqual(ArrayList<Object> ol0, ArrayList<Object> ol1) {
-    if (ol0.size() != numFields || ol1.size() != numFields) {
-      if (ol0.size() != ol1.size()) {
+  public boolean areEqual(Object[] ol0, Object[] ol1) {
+    if (ol0.length != numFields || ol1.length != numFields) {
+      if (ol0.length != ol1.length) {
         return false;
       }
-      assert (ol0.size() <= numFields);
-      assert (ol1.size() <= numFields);
-      for (int i = 0; i < Math.min(ol0.size(), ol1.size()); i++) {
-        if (!fieldComparers[i].areEqual(ol0.get(i), ol1.get(i))) {
+      assert (ol0.length <= numFields);
+      assert (ol1.length <= numFields);
+      for (int i = 0; i < Math.min(ol0.length, ol1.length); i++) {
+        if (!fieldComparers[i].areEqual(ol0[i], ol1[i])) {
           return false;
         }
       }
@@ -171,7 +169,7 @@ public class ListObjectsEqualComparer {
     }
 
     for (int i = numFields - 1; i >= 0; i--) {
-      if (!fieldComparers[i].areEqual(ol0.get(i), ol1.get(i))) {
+      if (!fieldComparers[i].areEqual(ol0[i], ol1[i])) {
         return false;
       }
     }



Mime
View raw message