hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1489094 - in /hive/branches/vectorization/ql/src: java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/exec/vector/ java/org/apache/hadoop/hive/ql/exec/vector/expressions/ java/org/apache/hadoop/hive/ql/exec/vector/expressio...
Date Mon, 03 Jun 2013 18:29:44 GMT
Author: hashutosh
Date: Mon Jun  3 18:29:44 2013
New Revision: 1489094

URL: http://svn.apache.org/r1489094
Log:
HIVE-4612 : Vectorized aggregates do not emit proper rows in presence of GROUP BY (Remus Rusanu via Ashutosh Chauhan)

Added:
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
Modified:
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/VectorHashKeyWrapperBatch.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgDouble.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgLong.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopDouble.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopLong.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampDouble.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampLong.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumDouble.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumLong.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopDouble.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopLong.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt
    hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt
    hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/FakeVectorRowBatchFromObjectIterables.java
    hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/VectorHashKeyWrapperBatch.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/VectorHashKeyWrapperBatch.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/VectorHashKeyWrapperBatch.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/VectorHashKeyWrapperBatch.java Mon Jun  3 18:29:44 2013
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
 import org.apache.hadoop.io.BytesWritable;
@@ -82,18 +83,6 @@ public class VectorHashKeyWrapperBatch {
   private KeyLookupHelper[] indexLookup;
 
   /**
-   * preallocated and reused LongWritable objects for emiting row mode key values
-   */
-  private LongWritable[] longKeyValueOutput;
-
-  /**
-   * preallocated and reused DoubleWritable objects for emiting row mode key values
-   */
-  private DoubleWritable[] doubleKeyValueOutput;
-
-  private BytesWritable[] stringKeyValueOutput;
-
-  /**
    * Accessor for the batch-sized array of key wrappers
    */
   public VectorHashKeyWrapper[] getVectorHashKeyWrappers() {
@@ -452,15 +441,19 @@ public class VectorHashKeyWrapperBatch {
     for(int i=0; i < keyExpressions.length; ++i) {
       indexLookup[i] = new KeyLookupHelper();
       String outputType = keyExpressions[i].getOutputType();
-      if (outputType.equalsIgnoreCase("long") ||
-          outputType.equalsIgnoreCase("bigint") ||
-          outputType.equalsIgnoreCase("int")) {
+      if (outputType.equalsIgnoreCase("tinyint") ||
+          outputType.equalsIgnoreCase("smallint") ||
+          outputType.equalsIgnoreCase("int")  ||
+          outputType.equalsIgnoreCase("bigint")  ||
+          outputType.equalsIgnoreCase("timestamp") ||
+          outputType.equalsIgnoreCase("boolean")) {
         longIndices[longIndicesIndex] = i;
         indexLookup[i].longIndex = longIndicesIndex;
         indexLookup[i].doubleIndex = -1;
         indexLookup[i].stringIndex = -1;
         ++longIndicesIndex;
-      } else if (outputType.equalsIgnoreCase("double")) {
+      } else if (outputType.equalsIgnoreCase("double") ||
+          outputType.equalsIgnoreCase("float")) {
         doubleIndices[doubleIndicesIndex] = i;
         indexLookup[i].longIndex = -1;
         indexLookup[i].doubleIndex = doubleIndicesIndex;
@@ -477,18 +470,6 @@ public class VectorHashKeyWrapperBatch {
       }
     }
     compiledKeyWrapperBatch.indexLookup = indexLookup;
-    compiledKeyWrapperBatch.longKeyValueOutput = new LongWritable[longIndicesIndex];
-    for (int i=0; i < longIndicesIndex; ++i) {
-      compiledKeyWrapperBatch.longKeyValueOutput[i] = new LongWritable();
-    }
-    compiledKeyWrapperBatch.doubleKeyValueOutput = new DoubleWritable[doubleIndicesIndex];
-    for (int i=0; i < doubleIndicesIndex; ++i) {
-      compiledKeyWrapperBatch.doubleKeyValueOutput[i] = new DoubleWritable();
-    }
-    compiledKeyWrapperBatch.stringKeyValueOutput = new BytesWritable[stringIndicesIndex];
-    for (int i = 0; i < stringIndicesIndex; ++i) {
-      compiledKeyWrapperBatch.stringKeyValueOutput[i] = new BytesWritable();
-    }
     compiledKeyWrapperBatch.longIndices = Arrays.copyOf(longIndices, longIndicesIndex);
     compiledKeyWrapperBatch.doubleIndices = Arrays.copyOf(doubleIndices, doubleIndicesIndex);
     compiledKeyWrapperBatch.stringIndices = Arrays.copyOf(stringIndices, stringIndicesIndex);
@@ -503,23 +484,22 @@ public class VectorHashKeyWrapperBatch {
 
   /**
    * Get the row-mode writable object value of a key from a key wrapper
+   * @param keyOutputWriter 
    */
-  public Object getWritableKeyValue(VectorHashKeyWrapper kw, int i)
+  public Object getWritableKeyValue(VectorHashKeyWrapper kw, int i, 
+      VectorExpressionWriter keyOutputWriter)
     throws HiveException {
     if (kw.getIsNull(i)) {
       return null;
     }
     KeyLookupHelper klh = indexLookup[i];
     if (klh.longIndex >= 0) {
-      longKeyValueOutput[klh.longIndex].set(kw.getLongValue(i));
-      return longKeyValueOutput[klh.longIndex];
+      return keyOutputWriter.writeValue(kw.getLongValue(i));
     } else if (klh.doubleIndex >= 0) {
-      doubleKeyValueOutput[klh.doubleIndex].set(kw.getDoubleValue(i));
-      return doubleKeyValueOutput[klh.doubleIndex];
+      return keyOutputWriter.writeValue(kw.getDoubleValue(i));
     } else if (klh.stringIndex >= 0) {
-      stringKeyValueOutput[klh.stringIndex].set(
+      return keyOutputWriter.writeValue(
           kw.getBytes(i), kw.getByteStart(i), kw.getByteLength(i));
-      return stringKeyValueOutput[klh.stringIndex];
     } else {
       throw new HiveException(String.format(
           "Internal inconsistent KeyLookupHelper at index [%d]:%d %d %d",

Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java?rev=1489094&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampUtils.java Mon Jun  3 18:29:44 2013
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.sql.Timestamp;
+
+public final class TimestampUtils {
+
+  public static void assignTimeInNanoSec(long timeInNanoSec, Timestamp t) {
+    t.setTime((timeInNanoSec)/1000000);
+    t.setNanos((int)((t.getNanos()) + (timeInNanoSec % 1000000)));
+  }
+
+  public static long getTimeNanoSec(Timestamp t) {
+    long time = t.getTime();
+    int nanos = t.getNanos();
+    return (time * 1000000) + (nanos % 1000000);
+  }
+}

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java Mon Jun  3 18:29:44 2013
@@ -33,9 +33,12 @@ import org.apache.hadoop.hive.ql.exec.Op
 import org.apache.hadoop.hive.ql.exec.VectorHashKeyWrapper;
 import org.apache.hadoop.hive.ql.exec.VectorHashKeyWrapperBatch;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.ql.plan.GroupByDesc;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
@@ -66,6 +69,8 @@ public class VectorGroupByOperator exten
    */
   private transient VectorExpression[] keyExpressions;
   
+  private VectorExpressionWriter[] keyOutputWriters;
+  
   /**
    * The aggregation buffers to use for the current batch.
    */
@@ -98,13 +103,18 @@ public class VectorGroupByOperator exten
     try {
       vContext.setOperatorType(OperatorType.GROUPBY);
 
-      ArrayList<AggregationDesc> aggrDesc = conf.getAggregators();
-      keyExpressions = vContext.getVectorExpressions(conf.getKeys());
+      List<ExprNodeDesc> keysDesc = conf.getKeys();
+      keyExpressions = vContext.getVectorExpressions(keysDesc);
+      
+      keyOutputWriters = new VectorExpressionWriter[keyExpressions.length];
       
       for(int i = 0; i < keyExpressions.length; ++i) {
-        objectInspectors.add(vContext.createObjectInspector(keyExpressions[i]));
+        keyOutputWriters[i] = VectorExpressionWriterFactory.
+            genVectorExpressionWritable(keysDesc.get(i));
+        objectInspectors.add(keyOutputWriters[i].getObjectInspector());
       }
-
+      
+      ArrayList<AggregationDesc> aggrDesc = conf.getAggregators();
       aggregators = new VectorAggregateExpression[aggrDesc.size()];
       for (int i = 0; i < aggrDesc.size(); ++i) {
         AggregationDesc desc = aggrDesc.get(i);
@@ -233,7 +243,8 @@ public class VectorGroupByOperator exten
           int fi = 0;
           for (int i = 0; i < keyExpressions.length; ++i) {
             VectorHashKeyWrapper kw = (VectorHashKeyWrapper)pair.getKey();
-            forwardCache[fi++] = keyWrappersBatch.getWritableKeyValue (kw, i);
+            forwardCache[fi++] = keyWrappersBatch.getWritableKeyValue (
+                kw, i, keyOutputWriters[i]);
           }
           for (int i = 0; i < aggregators.length; ++i) {
             forwardCache[fi++] = aggregators[i].evaluateOutput(pair.getValue()

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorReduceSinkOperator.java Mon Jun  3 18:29:44 2013
@@ -28,6 +28,8 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.ql.exec.TerminalOperator;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory;
 import org.apache.hadoop.hive.ql.io.HiveKey;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
@@ -57,11 +59,24 @@ public class VectorReduceSinkOperator ex
    * the reducer side. Key columns are passed to the reducer in the "key".
    */
   protected transient VectorExpression[] keyEval;
+  
+  /**
+   * The key value writers. These know how to write the necessary writable type
+   * based on key column metadata, from the primitive vector type.
+   */
+  protected transient VectorExpressionWriter[] keyWriters;
+  
   /**
    * The evaluators for the value columns. Value columns are passed to reducer
    * in the "value".
    */
   protected transient VectorExpression[] valueEval;
+  
+  /**
+   * The output value writers. These know how to write the necessary writable type
+   * based on value column metadata, from the primitive vector type.
+   */
+  protected transient VectorExpressionWriter[] valueWriters;
 
   /**
    * The evaluators for the partition columns (CLUSTER BY or DISTRIBUTE BY in
@@ -69,6 +84,12 @@ public class VectorReduceSinkOperator ex
    * goes to. Partition columns are not passed to reducer.
    */
   protected transient VectorExpression[] partitionEval;
+  
+  /**
+   * The partition value writers. These know how to write the necessary writable type
+   * based on partition column metadata, from the primitive vector type.
+   */  
+  protected transient VectorExpressionWriter[] partitionWriters;
 
   private int numDistributionKeys;
 
@@ -112,15 +133,22 @@ public class VectorReduceSinkOperator ex
           .newInstance();
       keySerializer.initialize(null, keyTableDesc.getProperties());
       keyIsText = keySerializer.getSerializedClass().equals(Text.class);
-
-      keyObjectInspector = vContext.createObjectInspector(keyEval,
-          conf.getOutputKeyColumnNames());
-
-      partitionObjectInspectors = new ObjectInspector[partitionEval.length];
-      for (int i = 0; i < partitionEval.length; i++) {
-        partitionObjectInspectors[i] = vContext.createObjectInspector(partitionEval[i]);
-      }
-
+      
+      /*
+       * Compute and assign the key writers and the key object inspector 
+       */
+      VectorExpressionWriterFactory.processVectorExpressions(
+          conf.getKeyCols(), 
+          conf.getOutputKeyColumnNames(),
+          new VectorExpressionWriterFactory.Closure() {
+            @Override
+            public void assign(VectorExpressionWriter[] writers,
+              ObjectInspector objectInspector) {
+              keyWriters = writers;
+              keyObjectInspector = objectInspector;
+            }
+          });
+      
       String colNames = "";
       for(String colName : conf.getOutputKeyColumnNames()) {
         colNames = String.format("%s %s", colNames, colName);
@@ -131,18 +159,27 @@ public class VectorReduceSinkOperator ex
           keyObjectInspector,
           colNames));
 
-      conf.getOutputKeyColumnNames();
-      conf.getOutputValueColumnNames();
-
-      //keyObjectInspector = ObjectInspectorFactory.
-
+      partitionWriters = VectorExpressionWriterFactory.getExpressionWriters(conf.getPartitionCols());
+      
       TableDesc valueTableDesc = conf.getValueSerializeInfo();
       valueSerializer = (Serializer) valueTableDesc.getDeserializerClass()
           .newInstance();
       valueSerializer.initialize(null, valueTableDesc.getProperties());
-
-      valueObjectInspector = vContext.createObjectInspector (valueEval,
-          conf.getOutputValueColumnNames());
+      
+      /*
+       * Compute and assign the value writers and the value object inspector
+       */
+      VectorExpressionWriterFactory.processVectorExpressions(
+          conf.getValueCols(),
+          conf.getOutputValueColumnNames(),
+          new VectorExpressionWriterFactory.Closure() {
+            @Override
+            public void assign(VectorExpressionWriter[] writers,
+                ObjectInspector objectInspector) {
+                valueWriters = writers;
+                valueObjectInspector = objectInspector;
+              }
+          });
 
       colNames = "";
       for(String colName : conf.getOutputValueColumnNames()) {
@@ -202,7 +239,7 @@ public class VectorReduceSinkOperator ex
         for (int i = 0; i < valueEval.length; i++) {
           int batchColumn = valueEval[i].getOutputColumn();
           ColumnVector vectorColumn = vrg.cols[batchColumn];
-          cachedValues[i] = vectorColumn.getWritableObject(rowIndex);
+          cachedValues[i] = valueWriters[i].writeValue(vectorColumn, rowIndex);
         }
         // Serialize the value
         value = valueSerializer.serialize(cachedValues, valueObjectInspector);
@@ -210,7 +247,7 @@ public class VectorReduceSinkOperator ex
         for (int i = 0; i < keyEval.length; i++) {
           int batchColumn = keyEval[i].getOutputColumn();
           ColumnVector vectorColumn = vrg.cols[batchColumn];
-          distributionKeys[i] = vectorColumn.getWritableObject(rowIndex);
+          distributionKeys[i] = keyWriters[i].writeValue(vectorColumn, rowIndex);
         }
         // no distinct key
         System.arraycopy(distributionKeys, 0, cachedKeys[0], 0, numDistributionKeys);
@@ -255,11 +292,13 @@ public class VectorReduceSinkOperator ex
             keyHashCode = random.nextInt();
           } else {
             for (int p = 0; p < partitionEval.length; p++) {
+              ColumnVector columnVector = vrg.cols[partitionEval[p].getOutputColumn()];
+              Object partitionValue = partitionWriters[p].writeValue(columnVector, rowIndex);
               keyHashCode = keyHashCode
                   * 31
                   + ObjectInspectorUtils.hashCode(
-                      vrg.cols[partitionEval[p].getOutputColumn()].getWritableObject(rowIndex),
-                      partitionObjectInspectors[i]);
+                      partitionValue,
+                      partitionWriters[p].getObjectInspector());
             }
           }
           keyWritable.setHashCode(keyHashCode);

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Mon Jun  3 18:29:44 2013
@@ -1049,11 +1049,6 @@ public class VectorizationContext {
     return ret;
   }
 
-  Object[][] mapObjectInspectors = {
-      {"double", PrimitiveObjectInspectorFactory.writableDoubleObjectInspector},
-      {"long", PrimitiveObjectInspectorFactory.writableLongObjectInspector},
-  };
-
   public Map<Integer, String> getOutputColumnTypeMap() {
     Map<Integer, String> map = new HashMap<Integer, String>();
     for (int i = 0; i < ocm.outputColCount; i++) {
@@ -1073,34 +1068,6 @@ public class VectorizationContext {
     }
   }
 
-  public ObjectInspector createObjectInspector(VectorExpression vectorExpression)
-      throws HiveException {
-    String columnType = vectorExpression.getOutputType();
-    if (columnType.equalsIgnoreCase("long") ||
-        columnType.equalsIgnoreCase("bigint") ||
-        columnType.equalsIgnoreCase("int") || 
-        columnType.equalsIgnoreCase("smallint")) {
-      return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
-    } else if (columnType.equalsIgnoreCase("double")) {
-      return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector;
-    } else if (columnType.equalsIgnoreCase("string")) {
-      return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
-    } else {
-      throw new HiveException(String.format("Must implement type %s", columnType));
-    }
-  }
-
-  public ObjectInspector createObjectInspector(
-      VectorExpression[] vectorExpressions, List<String> columnNames)
-      throws HiveException {
-    List<ObjectInspector> oids = new ArrayList<ObjectInspector>();
-    for (VectorExpression vexpr : vectorExpressions) {
-      ObjectInspector oi = createObjectInspector(vexpr);
-      oids.add(oi);
-    }
-    return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames,
-        oids);
-  }
 
   public void addToColumnMap(String columnName, int outputColumn) {
     if (columnMap != null) {

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java Mon Jun  3 18:29:44 2013
@@ -173,7 +173,7 @@ public class VectorizedBatchUtil {
         LongColumnVector lcv = (LongColumnVector) batch.cols[i];
         if (writableCol != null) {
           Timestamp t = ((TimestampWritable) writableCol).getTimestamp();
-          lcv.vector[rowIndex] = (t.getTime() * 1000000) + (t.getNanos() % 1000000);
+          lcv.vector[rowIndex] = TimestampUtils.getTimeNanoSec(t);
           lcv.isNull[rowIndex] = false;
         } else {
           lcv.vector[rowIndex] = 1;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java Mon Jun  3 18:29:44 2013
@@ -161,8 +161,7 @@ public class VectorizedColumnarSerDe ext
                 LongColumnVector tcv = (LongColumnVector) batch.cols[k];
                 long timeInNanoSec = tcv.vector[rowIndex];
                 Timestamp t = new Timestamp(0);
-                t.setTime((timeInNanoSec)/1000000);
-                t.setNanos((int)((t.getNanos()) + (timeInNanoSec % 1000000)));
+                TimestampUtils.assignTimeInNanoSec(timeInNanoSec, t);
                 TimestampWritable tw = new TimestampWritable();
                 tw.set(t);
                 LazyTimestamp.writeUTF8(serializeVectorStream, tw);

Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java?rev=1489094&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java Mon Jun  3 18:29:44 2013
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Interface used to create Writable objects from vector expression primitives.
+ *
+ */
+public interface VectorExpressionWriter {
+  ObjectInspector getObjectInspector();
+  Object writeValue(ColumnVector column, int row) throws HiveException;
+  Object writeValue(long value) throws HiveException;
+  Object writeValue(double value) throws HiveException;
+  Object writeValue(byte[] value, int start, int length) throws HiveException;
+}

Added: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java?rev=1489094&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java (added)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java Mon Jun  3 18:29:44 2013
@@ -0,0 +1,390 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions;
+
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory.Closure;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+
+/**
+ * VectorExpressionWritableFactory helper class for generating VectorExpressionWritable objects.
+ */
+public final class VectorExpressionWriterFactory {
+  
+  /**
+   * VectorExpressionWriter base implementation, to be specialized for Long/Double/Bytes columns
+   */
+  private static abstract class VectorExpressionWriterBase implements VectorExpressionWriter {
+    
+    protected ObjectInspector objectInspector;
+    
+    /**
+     * The object inspector associated with this expression. This is created from the expression
+     * NodeDesc (compile metadata) not from the VectorColumn info and thus preserves the type info
+     * lost by the vectorization process.
+     */
+    public ObjectInspector getObjectInspector() {
+      return objectInspector;
+    }
+    
+    public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
+      this.objectInspector = nodeDesc.getWritableObjectInspector();
+      return this;
+    }
+    
+    /**
+     * The base implementation must be overridden by the Long specialization
+     */
+    @Override
+    public Object writeValue(long value) throws HiveException {
+      throw new HiveException("Internal error: should not reach here");
+    }
+
+    /**
+     * The base implementation must be overridden by the Double specialization
+     */
+    @Override
+    public Object writeValue(double value) throws HiveException {
+      throw new HiveException("Internal error: should not reach here");
+    }
+
+    /**
+     * The base implementation must be overridden by the Bytes specialization
+     */
+    @Override
+    public Object writeValue(byte[] value, int start, int length) throws HiveException {
+      throw new HiveException("Internal error: should not reach here");
+    }
+  }
+  
+  /**
+   * Specialized writer for LongVectorColumn expressions. Will throw cast exception
+   * if the wrong vector column is used. 
+   */
+  private static abstract class VectorExpressionWriterLong
+    extends VectorExpressionWriterBase {
+    @Override
+    public Object writeValue(ColumnVector column, int row) throws HiveException {
+      LongColumnVector lcv = (LongColumnVector) column;
+      if (lcv.noNulls && !lcv.isRepeating) {
+        return writeValue(lcv.vector[row]);
+      } else if (lcv.noNulls && lcv.isRepeating) {
+        return writeValue(lcv.vector[0]);
+      } else if (!lcv.noNulls && !lcv.isRepeating && !lcv.isNull[row]) {
+        return writeValue(lcv.vector[row]);
+      } else if (!lcv.noNulls && !lcv.isRepeating && lcv.isNull[row]) {
+        return null;
+      } else if (!lcv.noNulls && lcv.isRepeating && !lcv.isNull[0]) {
+        return writeValue(lcv.vector[0]);
+      } else if (!lcv.noNulls && lcv.isRepeating && lcv.isNull[0]) {
+        return null;
+      }
+      throw new HiveException(
+        String.format(
+          "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b",
+          row, lcv.noNulls, lcv.isRepeating, lcv.isNull[row], lcv.isNull[0]));
+    }
+   }
+  
+  /**
+   * Specialized writer for DoubleColumnVector. Will throw cast exception
+   * if the wrong vector column is used. 
+   */
+  private static abstract class VectorExpressionWriterDouble extends VectorExpressionWriterBase {
+    @Override
+    public Object writeValue(ColumnVector column, int row) throws HiveException {
+      DoubleColumnVector dcv = (DoubleColumnVector) column;
+      if (dcv.noNulls && !dcv.isRepeating) {
+        return writeValue(dcv.vector[row]);
+      } else if (dcv.noNulls && dcv.isRepeating) {
+        return writeValue(dcv.vector[0]);
+      } else if (!dcv.noNulls && !dcv.isRepeating && !dcv.isNull[row]) {
+        return writeValue(dcv.vector[row]);
+      } else if (!dcv.noNulls && !dcv.isRepeating && dcv.isNull[row]) {
+        return null;
+      } else if (!dcv.noNulls && dcv.isRepeating && !dcv.isNull[0]) {
+        return writeValue(dcv.vector[0]);
+      } else if (!dcv.noNulls && dcv.isRepeating && dcv.isNull[0]) {
+        return null;
+      }
+      throw new HiveException(
+        String.format(
+          "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b",
+          row, dcv.noNulls, dcv.isRepeating, dcv.isNull[row], dcv.isNull[0]));
+    }
+   }
+  
+  /**
+   * Specialized writer for BytesColumnVector. Will throw cast exception
+   * if the wrong vector column is used. 
+   */
+  private static abstract class VectorExpressionWriterBytes extends VectorExpressionWriterBase {
+    @Override
+    public Object writeValue(ColumnVector column, int row) throws HiveException {
+      BytesColumnVector bcv = (BytesColumnVector) column;
+      if (bcv.noNulls && !bcv.isRepeating) {
+        return writeValue(bcv.vector[row], bcv.start[row], bcv.length[row]);
+      } else if (bcv.noNulls && bcv.isRepeating) {
+        return writeValue(bcv.vector[0], bcv.start[0], bcv.length[0]);
+      } else if (!bcv.noNulls && !bcv.isRepeating && !bcv.isNull[row]) {
+        return writeValue(bcv.vector[row], bcv.start[row], bcv.length[row]);
+      } else if (!bcv.noNulls && !bcv.isRepeating && bcv.isNull[row]) {
+        return null;
+      } else if (!bcv.noNulls && bcv.isRepeating && !bcv.isNull[0]) {
+        return writeValue(bcv.vector[0], bcv.start[0], bcv.length[0]);
+      } else if (!bcv.noNulls && bcv.isRepeating && bcv.isNull[0]) {
+        return null;
+      }
+      throw new HiveException(
+        String.format(
+          "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b",
+          row, bcv.noNulls, bcv.isRepeating, bcv.isNull[row], bcv.isNull[0]));
+    }
+   }
+
+  /**
+   * Compiles the appropriate vector expression writer based on an expression info (ExprNodeDesc)
+   */
+  public static VectorExpressionWriter genVectorExpressionWritable(ExprNodeDesc nodeDesc) 
+    throws HiveException {
+    String nodeType = nodeDesc.getTypeString();
+    if (nodeType.equalsIgnoreCase("tinyint")) {
+      return new VectorExpressionWriterLong()
+      {
+        private ByteWritable writable;
+        
+        @Override 
+        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
+          super.init(nodeDesc);
+          writable = new ByteWritable();
+          return this;
+        }
+
+        @Override
+        public Object writeValue(long value) {
+          writable.set((byte) value);
+          return writable;
+        }
+      }.init(nodeDesc);
+    } else if (nodeType.equalsIgnoreCase("smallint")) {
+      return new VectorExpressionWriterLong()
+      {
+        private ShortWritable writable;
+        @Override 
+        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
+          super.init(nodeDesc);
+          writable = new ShortWritable();
+          return this;
+        }
+
+        @Override
+        public Object writeValue(long value) {
+          writable.set((short) value);
+          return writable;
+        }      
+      }.init(nodeDesc);
+    } else if (nodeType.equalsIgnoreCase("int")) {
+      return new VectorExpressionWriterLong()
+      {
+        private IntWritable writable;
+        @Override 
+        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
+          super.init(nodeDesc);
+          writable = new IntWritable();
+          return this;
+        }
+
+        @Override
+        public Object writeValue(long value) {
+          writable.set((int) value);
+          return writable;
+        }      
+      }.init(nodeDesc);      
+    } else if (nodeType.equalsIgnoreCase("bigint")) {
+      return new VectorExpressionWriterLong()
+      {
+        private LongWritable writable;
+        @Override 
+        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
+          super.init(nodeDesc);
+          writable = new LongWritable();
+          return this;
+        }
+
+        @Override
+        public Object writeValue(long value) {
+          writable.set(value);
+          return writable;
+        }
+      }.init(nodeDesc);
+    } else if (nodeType.equalsIgnoreCase("boolean")) {
+      return new VectorExpressionWriterLong()
+      {
+        private BooleanWritable writable;
+        @Override 
+        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
+          super.init(nodeDesc);
+          writable = new BooleanWritable();
+          return this;
+        }
+
+        @Override
+        public Object writeValue(long value) {
+          writable.set(value != 0 ? true : false);
+          return writable;
+        }
+      }.init(nodeDesc);
+    } else if (nodeType.equalsIgnoreCase("timestamp")) {
+      return new VectorExpressionWriterLong()
+      {
+        private TimestampWritable writable;
+        private Timestamp timestamp;
+        @Override 
+        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
+          super.init(nodeDesc);
+          writable = new TimestampWritable();
+          timestamp = new Timestamp(0);
+          return this;
+        }
+
+        @Override
+        public Object writeValue(long value) {
+          TimestampUtils.assignTimeInNanoSec(value, timestamp);
+          writable.set(timestamp);
+          return writable;
+        }
+      }.init(nodeDesc);
+    } else if (nodeType.equalsIgnoreCase("string")) {
+      return new VectorExpressionWriterBytes()
+      {
+        private BytesWritable writable;
+        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
+          super.init(nodeDesc);
+          writable = new BytesWritable();
+          return this;
+        }
+        
+        @Override 
+        public Object writeValue(byte[] value, int start, int length) throws HiveException {
+          writable.set(value, start, length);
+          return writable;
+        }
+      }.init(nodeDesc);
+    } else if (nodeType.equalsIgnoreCase("float")) {
+      return new VectorExpressionWriterDouble()
+      {
+        private FloatWritable writable;
+        @Override 
+        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
+          super.init(nodeDesc);
+          writable = new FloatWritable();
+          return this;
+        }
+
+        @Override
+        public Object writeValue(double value) {
+          writable.set((float)value);
+          return writable;
+        }
+      }.init(nodeDesc);
+    } else if (nodeType.equalsIgnoreCase("double")) {
+      return new VectorExpressionWriterDouble()
+      {
+        private DoubleWritable writable;
+        @Override 
+        public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException {
+          super.init(nodeDesc);
+          writable = new DoubleWritable();
+          return this;
+        }
+
+        @Override
+        public Object writeValue(double value) {
+          writable.set(value);
+          return writable;
+        }
+      }.init(nodeDesc);
+    }
+    
+    throw new HiveException(String.format(
+        "Unimplemented genVectorExpressionWritable type: %s for expression: %s", 
+        nodeType, nodeDesc));
+  }
+  
+  /**
+   * Helper function to create an array of writers from a list of expression descriptors.
+   */
+  public static VectorExpressionWriter[] getExpressionWriters(List<ExprNodeDesc> nodesDesc) 
+      throws HiveException {
+    VectorExpressionWriter[] writers = new VectorExpressionWriter[nodesDesc.size()];
+    for(int i=0; i<writers.length; ++i) {
+      ExprNodeDesc nodeDesc = nodesDesc.get(i);
+      writers[i] = genVectorExpressionWritable(nodeDesc);
+    }
+    return writers;
+  }
+  
+  /**
+   * A poor man Java closure. Works around the problem of having to return multiple objects
+   * from one function call.
+   */
+  public static interface Closure {
+    void assign(VectorExpressionWriter[] writers, ObjectInspector objectInspector);
+  }
+  
+  /**
+   * Creates the value writers for a column vector expression list. 
+   * Creates an appropriate output object inspector.
+   */
+  public static void processVectorExpressions(
+      List<ExprNodeDesc> nodesDesc,
+      List<String> outputColumnNames,
+      Closure closure) 
+      throws HiveException {
+    VectorExpressionWriter[] writers = getExpressionWriters(nodesDesc);
+    List<ObjectInspector> oids = new ArrayList<ObjectInspector>(writers.length);
+    for(int i=0; i<writers.length; ++i) {
+      oids.add(writers[i].getObjectInspector());
+    }
+    ObjectInspector objectInspector = ObjectInspectorFactory.
+        getStandardStructObjectInspector(outputColumnNames,oids);
+    closure.assign(writers, objectInspector);
+  }
+}

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgDouble.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgDouble.java Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgLong.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFAvgLong.java Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopDouble.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopDouble.java Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopLong.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdPopLong.java Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampDouble.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampDouble.java Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampLong.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFStdSampLong.java Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumDouble.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumDouble.java Mon Jun  3 18:29:44 2013
@@ -51,7 +51,7 @@ public class VectorUDAFSumDouble extends
     /** 
     /* class for storing the current aggregate value.
     */
-    static private final class Aggregation implements AggregationBuffer {
+    private static final class Aggregation implements AggregationBuffer {
       double sum;
       boolean isNull;
       

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumLong.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFSumLong.java Mon Jun  3 18:29:44 2013
@@ -51,7 +51,7 @@ public class VectorUDAFSumLong extends V
     /** 
     /* class for storing the current aggregate value.
     */
-    static private final class Aggregation implements AggregationBuffer {
+    private static final class Aggregation implements AggregationBuffer {
       long sum;
       boolean isNull;
       

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopDouble.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopDouble.java Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopLong.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarPopLong.java Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampDouble.java Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/gen/VectorUDAFVarSampLong.java Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFAvg.txt Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/VectorUDAFVar.txt Mon Jun  3 18:29:44 2013
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/FakeVectorRowBatchFromObjectIterables.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/FakeVectorRowBatchFromObjectIterables.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/FakeVectorRowBatchFromObjectIterables.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/FakeVectorRowBatchFromObjectIterables.java Mon Jun  3 18:29:44 2013
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql.exec.vector;
 
+import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -35,6 +36,10 @@ public class FakeVectorRowBatchFromObjec
   private final VectorizedRowBatch batch;
   private boolean eof;
   private final int batchSize;
+  
+  public String[] getTypes() {
+    return this.types;
+  }
 
   /**
    * Helper interface for assigning values to primitive vector column types.
@@ -58,7 +63,35 @@ public class FakeVectorRowBatchFromObjec
 
     batch = new VectorizedRowBatch(types.length, batchSize);
     for(int i=0; i< types.length; ++i) {
-      if (types[i].equalsIgnoreCase("long")) {
+      if (types[i].equalsIgnoreCase("tinyint") ||
+          types[i].equalsIgnoreCase("smallint")||
+          types[i].equalsIgnoreCase("int")||
+          types[i].equalsIgnoreCase("bigint")||
+          types[i].equalsIgnoreCase("long")) {
+        batch.cols[i] = new LongColumnVector(batchSize);
+        columnAssign[i] = new ColumnVectorAssign() {
+          @Override
+          public void assign(
+              ColumnVector columnVector,
+              int row,
+              Object value) {
+            LongColumnVector lcv = (LongColumnVector) columnVector;
+            lcv.vector[row] = Long.valueOf(value.toString());
+          }
+        };
+      } else if (types[i].equalsIgnoreCase("boolean")) {
+        batch.cols[i] = new LongColumnVector(batchSize);
+        columnAssign[i] = new ColumnVectorAssign() {
+          @Override
+          public void assign(
+              ColumnVector columnVector,
+              int row,
+              Object value) {
+            LongColumnVector lcv = (LongColumnVector) columnVector;
+            lcv.vector[row] = (Boolean) value ? 1 : 0;
+          }
+        };
+      } else if (types[i].equalsIgnoreCase("timestamp")) {
         batch.cols[i] = new LongColumnVector(batchSize);
         columnAssign[i] = new ColumnVectorAssign() {
           @Override
@@ -67,9 +100,11 @@ public class FakeVectorRowBatchFromObjec
               int row,
               Object value) {
             LongColumnVector lcv = (LongColumnVector) columnVector;
-            lcv.vector[row] = (Long) value;
+            Timestamp t = (Timestamp) value;
+            lcv.vector[row] = TimestampUtils.getTimeNanoSec(t);
           }
         };
+        
       } else if (types[i].equalsIgnoreCase("string")) {
         batch.cols[i] = new BytesColumnVector(batchSize);
         columnAssign[i] = new ColumnVectorAssign() {
@@ -86,7 +121,8 @@ public class FakeVectorRowBatchFromObjec
             bcv.length[row] = bytes.length;
           }
         };
-      } else if (types[i].equalsIgnoreCase("double")) {
+      } else if (types[i].equalsIgnoreCase("double") ||
+          types[i].equalsIgnoreCase("float")) {
         batch.cols[i] = new DoubleColumnVector(batchSize);
         columnAssign[i] = new ColumnVectorAssign() {
           @Override
@@ -95,7 +131,7 @@ public class FakeVectorRowBatchFromObjec
               int row,
               Object value) {
             DoubleColumnVector dcv = (DoubleColumnVector) columnVector;
-            dcv.vector[row] = (Double) value;
+            dcv.vector[row] = Double.valueOf(value.toString());
           }
         };
       } else {

Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java?rev=1489094&r1=1489093&r2=1489094&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java Mon Jun  3 18:29:44 2013
@@ -25,6 +25,7 @@ import static org.junit.Assert.assertNul
 import static org.junit.Assert.assertTrue;
 
 import java.lang.reflect.Constructor;
+import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -42,11 +43,18 @@ import org.apache.hadoop.hive.ql.plan.Ag
 import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.ql.plan.GroupByDesc;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.ByteWritable;
 import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -167,6 +175,102 @@ public class TestVectorGroupByOperator {
   }
 
   @Test
+  public void testTinyintKeyTypeAggregate () throws HiveException {
+    testKeyTypeAggregate(
+        "sum",
+        new FakeVectorRowBatchFromObjectIterables(
+            2,
+            new String[] {"tinyint", "bigint"},
+            Arrays.asList(new Object[]{  1,null, 1, null}),
+            Arrays.asList(new Object[]{13L,null,7L, 19L})),
+        buildHashMap((byte)1, 20L, null, 19L));
+  }
+  
+  @Test
+  public void testSmallintKeyTypeAggregate () throws HiveException {
+    testKeyTypeAggregate(
+        "sum",
+        new FakeVectorRowBatchFromObjectIterables(
+            2,
+            new String[] {"smallint", "bigint"},
+            Arrays.asList(new Object[]{  1,null, 1, null}),
+            Arrays.asList(new Object[]{13L,null,7L, 19L})),
+        buildHashMap((short)1, 20L, null, 19L));
+  }  
+  
+  @Test
+  public void testIntKeyTypeAggregate () throws HiveException {
+    testKeyTypeAggregate(
+        "sum",
+        new FakeVectorRowBatchFromObjectIterables(
+            2,
+            new String[] {"int", "bigint"},
+            Arrays.asList(new Object[]{  1,null, 1, null}),
+            Arrays.asList(new Object[]{13L,null,7L, 19L})),
+        buildHashMap((int)1, 20L, null, 19L));
+  }
+  
+  @Test
+  public void testBigintKeyTypeAggregate () throws HiveException {
+    testKeyTypeAggregate(
+        "sum",
+        new FakeVectorRowBatchFromObjectIterables(
+            2,
+            new String[] {"bigint", "bigint"},
+            Arrays.asList(new Object[]{  1,null, 1, null}),
+            Arrays.asList(new Object[]{13L,null,7L, 19L})),
+        buildHashMap((long)1L, 20L, null, 19L));
+  }
+  
+  @Test
+  public void testBooleanKeyTypeAggregate () throws HiveException {
+    testKeyTypeAggregate(
+        "sum",
+        new FakeVectorRowBatchFromObjectIterables(
+            2,
+            new String[] {"boolean", "bigint"},
+            Arrays.asList(new Object[]{  true,null, true, null}),
+            Arrays.asList(new Object[]{13L,null,7L, 19L})),
+        buildHashMap(true, 20L, null, 19L));
+  }
+  
+  @Test
+  public void testTimestampKeyTypeAggregate () throws HiveException {
+    testKeyTypeAggregate(
+        "sum",
+        new FakeVectorRowBatchFromObjectIterables(
+            2,
+            new String[] {"timestamp", "bigint"},
+            Arrays.asList(new Object[]{new Timestamp(1),null, new Timestamp(1), null}),
+            Arrays.asList(new Object[]{13L,null,7L, 19L})),
+        buildHashMap(new Timestamp(1), 20L, null, 19L));
+  }  
+  
+  @Test
+  public void testFloatKeyTypeAggregate () throws HiveException {
+    testKeyTypeAggregate(
+        "sum",
+        new FakeVectorRowBatchFromObjectIterables(
+            2,
+            new String[] {"float", "bigint"},
+            Arrays.asList(new Object[]{  1,null, 1, null}),
+            Arrays.asList(new Object[]{13L,null,7L, 19L})),
+        buildHashMap((float)1.0, 20L, null, 19L));
+  }
+  
+  @Test
+  public void testDoubleKeyTypeAggregate () throws HiveException {
+    testKeyTypeAggregate(
+        "sum",
+        new FakeVectorRowBatchFromObjectIterables(
+            2,
+            new String[] {"double", "bigint"},
+            Arrays.asList(new Object[]{  1,null, 1, null}),
+            Arrays.asList(new Object[]{13L,null,7L, 19L})),
+        buildHashMap((double)1.0, 20L, null, 19L));
+  }    
+  
+  @Test
   public void testCountStar () throws HiveException {
     testAggregateCountStar(
         2,
@@ -1014,6 +1118,112 @@ public class TestVectorGroupByOperator {
         1024,
         (double)0);
   }
+  
+  private void testKeyTypeAggregate(
+      String aggregateName,
+      FakeVectorRowBatchFromObjectIterables data,
+      Map<Object, Object> expected) throws HiveException {
+    
+    Map<String, Integer> mapColumnNames = new HashMap<String, Integer>();
+    mapColumnNames.put("Key", 0);
+    mapColumnNames.put("Value", 1);
+    VectorizationContext ctx = new VectorizationContext(mapColumnNames, 2);
+    Set<Object> keys = new HashSet<Object>();
+    
+    AggregationDesc agg = buildAggregationDesc(ctx, aggregateName,
+        "Value", TypeInfoFactory.getPrimitiveTypeInfo(data.getTypes()[1]));
+    ArrayList<AggregationDesc> aggs = new ArrayList<AggregationDesc>();
+    aggs.add(agg);
+
+    ArrayList<String> outputColumnNames = new ArrayList<String>();
+    outputColumnNames.add("_col0");
+
+    GroupByDesc desc = new GroupByDesc();
+    desc.setOutputColumnNames(outputColumnNames);
+    desc.setAggregators(aggs);
+
+    ExprNodeDesc keyExp = buildColumnDesc(ctx, "Key", 
+        TypeInfoFactory.getPrimitiveTypeInfo(data.getTypes()[0]));
+    ArrayList<ExprNodeDesc> keysDesc = new ArrayList<ExprNodeDesc>();
+    keysDesc.add(keyExp);
+    desc.setKeys(keysDesc);
+
+    VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
+
+    FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
+    vgo.initialize(null, null);
+    out.setOutputInspector(new FakeCaptureOutputOperator.OutputInspector() {
+
+      private int rowIndex;
+      private String aggregateName;
+      private Map<Object,Object> expected;
+      private Set<Object> keys;
+
+      @Override
+      public void inspectRow(Object row, int tag) throws HiveException {
+        assertTrue(row instanceof Object[]);
+        Object[] fields = (Object[]) row;
+        assertEquals(2, fields.length);
+        Object key = fields[0];
+        Object keyValue = null;
+        if (null == key) {
+          keyValue = null;
+        } else if (key instanceof ByteWritable) {
+          ByteWritable bwKey = (ByteWritable)key;
+          keyValue = bwKey.get();
+        } else if (key instanceof ShortWritable) {
+          ShortWritable swKey = (ShortWritable)key;
+          keyValue = swKey.get();
+        } else if (key instanceof IntWritable) {
+          IntWritable iwKey = (IntWritable)key;
+          keyValue = iwKey.get();
+        } else if (key instanceof LongWritable) {
+          LongWritable lwKey = (LongWritable)key;
+          keyValue = lwKey.get();
+        } else if (key instanceof TimestampWritable) {
+          TimestampWritable twKey = (TimestampWritable)key;
+          keyValue = twKey.getTimestamp();
+        } else if (key instanceof DoubleWritable) {
+          DoubleWritable dwKey = (DoubleWritable)key;
+          keyValue = dwKey.get();
+        } else if (key instanceof FloatWritable) {
+          FloatWritable fwKey = (FloatWritable)key;
+          keyValue = fwKey.get();
+        } else if (key instanceof BooleanWritable) {
+          BooleanWritable bwKey = (BooleanWritable)key;
+          keyValue = bwKey.get();
+        } else {
+          Assert.fail(String.format("Not implemented key output type %s: %s", 
+              key.getClass().getName(), key));
+        }
+        
+        assertTrue(expected.containsKey(keyValue));
+        Object expectedValue = expected.get(keyValue);
+        Object value = fields[1];
+        Validator validator = getValidator(aggregateName);
+        validator.validate(expectedValue, new Object[] {value});
+        keys.add(keyValue);
+      }
+
+      private FakeCaptureOutputOperator.OutputInspector init(
+          String aggregateName, Map<Object,Object> expected, Set<Object> keys) {
+        this.aggregateName = aggregateName;
+        this.expected = expected;
+        this.keys = keys;
+        return this;
+      }
+    }.init(aggregateName, expected, keys));
+
+    for (VectorizedRowBatch unit: data) {
+      vgo.process(unit,  0);
+    }
+    vgo.close(false);
+
+    List<Object> outBatchList = out.getCapturedRows();
+    assertNotNull(outBatchList);
+    assertEquals(expected.size(), outBatchList.size());
+    assertEquals(expected.size(), keys.size());    
+  }  
 
 
   public void testAggregateLongRepeats (



Mime
View raw message