hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1491980 - in /hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector: FakeVectorRowBatchFromObjectIterables.java TestVectorGroupByOperator.java util/FakeVectorRowBatchFromObjectIterables.java
Date Tue, 11 Jun 2013 21:46:09 GMT
Author: hashutosh
Date: Tue Jun 11 21:46:08 2013
New Revision: 1491980

URL: http://svn.apache.org/r1491980
Log:
HIVE-4704 : TestCase FakeVectorRowBatchFromObjectIterables error (Eric Hanson via Ashutosh
Chauhan)

Added:
    hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
Removed:
    hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/FakeVectorRowBatchFromObjectIterables.java
Modified:
    hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java

Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java?rev=1491980&r1=1491979&r2=1491980&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
(original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
Tue Jun 11 21:46:08 2013
@@ -37,6 +37,7 @@ import java.util.Set;
 import org.apache.hadoop.hive.ql.exec.vector.util.FakeCaptureOutputOperator;
 import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromConcat;
 import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromLongIterables;
+import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromObjectIterables;
 import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromRepeats;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.AggregationDesc;

Added: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java?rev=1491980&view=auto
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
(added)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
Tue Jun 11 21:46:08 2013
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.util;
+
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+
+/**
+ * Test helper class that creates vectorized execution batches from arbitrary type iterables.
+ */
+public class FakeVectorRowBatchFromObjectIterables extends FakeVectorRowBatchBase {
+
+  private final String[] types;
+  private final List<Iterator<Object>> iterators;
+  private final VectorizedRowBatch batch;
+  private boolean eof;
+  private final int batchSize;
+  
+  public String[] getTypes() {
+    return this.types;
+  }
+
+  /**
+   * Helper interface for assigning values to primitive vector column types.
+   */
+  private static interface ColumnVectorAssign
+  {
+    public void assign(
+        ColumnVector columnVector,
+        int row,
+        Object value);
+  }
+
+  private final ColumnVectorAssign[] columnAssign;
+
+  public FakeVectorRowBatchFromObjectIterables(int batchSize, String[] types,
+      Iterable<Object> ...iterables) throws HiveException {
+    this.types = types;
+    this.batchSize = batchSize;
+    iterators = new ArrayList<Iterator<Object>>(types.length);
+    columnAssign = new ColumnVectorAssign[types.length];
+
+    batch = new VectorizedRowBatch(types.length, batchSize);
+    for(int i=0; i< types.length; ++i) {
+      if (types[i].equalsIgnoreCase("tinyint") ||
+          types[i].equalsIgnoreCase("smallint")||
+          types[i].equalsIgnoreCase("int")||
+          types[i].equalsIgnoreCase("bigint")||
+          types[i].equalsIgnoreCase("long")) {
+        batch.cols[i] = new LongColumnVector(batchSize);
+        columnAssign[i] = new ColumnVectorAssign() {
+          @Override
+          public void assign(
+              ColumnVector columnVector,
+              int row,
+              Object value) {
+            LongColumnVector lcv = (LongColumnVector) columnVector;
+            lcv.vector[row] = Long.valueOf(value.toString());
+          }
+        };
+      } else if (types[i].equalsIgnoreCase("boolean")) {
+        batch.cols[i] = new LongColumnVector(batchSize);
+        columnAssign[i] = new ColumnVectorAssign() {
+          @Override
+          public void assign(
+              ColumnVector columnVector,
+              int row,
+              Object value) {
+            LongColumnVector lcv = (LongColumnVector) columnVector;
+            lcv.vector[row] = (Boolean) value ? 1 : 0;
+          }
+        };
+      } else if (types[i].equalsIgnoreCase("timestamp")) {
+        batch.cols[i] = new LongColumnVector(batchSize);
+        columnAssign[i] = new ColumnVectorAssign() {
+          @Override
+          public void assign(
+              ColumnVector columnVector,
+              int row,
+              Object value) {
+            LongColumnVector lcv = (LongColumnVector) columnVector;
+            Timestamp t = (Timestamp) value;
+            lcv.vector[row] = TimestampUtils.getTimeNanoSec(t);
+          }
+        };
+        
+      } else if (types[i].equalsIgnoreCase("string")) {
+        batch.cols[i] = new BytesColumnVector(batchSize);
+        columnAssign[i] = new ColumnVectorAssign() {
+          @Override
+          public void assign(
+              ColumnVector columnVector,
+              int row,
+              Object value) {
+            BytesColumnVector bcv = (BytesColumnVector) columnVector;
+            String s = (String) value;
+            byte[] bytes = s.getBytes();
+            bcv.vector[row] = bytes;
+            bcv.start[row] = 0;
+            bcv.length[row] = bytes.length;
+          }
+        };
+      } else if (types[i].equalsIgnoreCase("double") ||
+          types[i].equalsIgnoreCase("float")) {
+        batch.cols[i] = new DoubleColumnVector(batchSize);
+        columnAssign[i] = new ColumnVectorAssign() {
+          @Override
+          public void assign(
+              ColumnVector columnVector,
+              int row,
+              Object value) {
+            DoubleColumnVector dcv = (DoubleColumnVector) columnVector;
+            dcv.vector[row] = Double.valueOf(value.toString());
+          }
+        };
+      } else {
+        throw new HiveException("Unimplemented type " + types[i]);
+      }
+      iterators.add(iterables[i].iterator());
+    }
+  }
+
+  @Override
+  public VectorizedRowBatch produceNextBatch() {
+    batch.size = 0;
+    batch.selectedInUse = false;
+    for (int i=0; i < types.length; ++i) {
+      ColumnVector col = batch.cols[i];
+      col.noNulls = true;
+      col.isRepeating = false;
+    }
+    while (!eof && batch.size < this.batchSize){
+      int r = batch.size;
+      for (int i=0; i < types.length; ++i) {
+        Iterator<Object> it = iterators.get(i);
+        if (!it.hasNext()) {
+          eof = true;
+          break;
+        }
+        Object value = it.next();
+        if (null == value) {
+          batch.cols[i].isNull[batch.size] = true;
+          batch.cols[i].noNulls = false;
+        } else {
+          columnAssign[i].assign(batch.cols[i], batch.size, value);
+        }
+      }
+      if (!eof) {
+        batch.size += 1;
+      }
+    }
+    return batch;
+  }
+}
+



Mime
View raw message