hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1531391 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/optimizer/physical/ test/org/apache/hadoop/hive/ql/optimizer/physical/ test/queries/clientpositive/ test/results/clientpositive/
Date Fri, 11 Oct 2013 18:50:33 GMT
Author: hashutosh
Date: Fri Oct 11 18:50:33 2013
New Revision: 1531391

URL: http://svn.apache.org/r1531391
Log:
HIVE-5494 : Vectorization throws exception with nested UDF. (Jitendra Nath Pandey via Ashutosh
Chauhan)

Added:
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
    hive/trunk/ql/src/test/queries/clientpositive/vectorization_nested_udf.q
    hive/trunk/ql/src/test/results/clientpositive/vectorization_nested_udf.q.out
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1531391&r1=1531390&r2=1531391&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Fri
Oct 11 18:50:33 2013
@@ -435,7 +435,7 @@ public class Vectorizer implements Physi
     return pctx;
   }
 
-  private boolean validateOperator(Operator<? extends OperatorDesc> op) {
+  boolean validateOperator(Operator<? extends OperatorDesc> op) {
     boolean ret = false;
     switch (op.getType()) {
       case GROUPBY:
@@ -548,7 +548,13 @@ public class Vectorizer implements Physi
   }
 
   private boolean validateAggregationDesc(AggregationDesc aggDesc) {
-    return supportedAggregationUdfs.contains(aggDesc.getGenericUDAFName().toLowerCase());
+    if (!supportedAggregationUdfs.contains(aggDesc.getGenericUDAFName().toLowerCase())) {
+      return false;
+    }
+    if (aggDesc.getParameters() != null) {
+      return validateExprNodeDesc(aggDesc.getParameters());
+    }
+    return true;
   }
 
   private boolean validateDataType(String type) {
@@ -569,7 +575,7 @@ public class Vectorizer implements Physi
     return new VectorizationContext(cmap, columnCount);
   }
 
-  private Operator<? extends OperatorDesc> vectorizeOperator(Operator<? extends
OperatorDesc> op,
+  Operator<? extends OperatorDesc> vectorizeOperator(Operator<? extends OperatorDesc>
op,
       VectorizationContext vContext) throws HiveException {
     Operator<? extends OperatorDesc> vectorOp = null;
 

Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java?rev=1531391&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
(added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
Fri Oct 11 18:50:33 2013
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.physical;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hive.ql.exec.GroupByOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFSumLong;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncAbsLongToLong;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.GroupByDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFAbs;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestVectorizer {
+
+  static VectorizationContext vContext = null;
+
+  @Before
+  public void setUp() {
+    Map<String, Integer> columnMap = new HashMap<String, Integer>();
+    columnMap.put("col1", 0);
+    columnMap.put("col2", 1);
+
+    //Generate vectorized expression
+    vContext = new VectorizationContext(columnMap, 2);
+  }
+
+  @Test
+  public void testAggregateOnUDF() throws HiveException {
+    AggregationDesc aggDesc = new AggregationDesc();
+    aggDesc.setGenericUDAFName("sum");
+    ExprNodeGenericFuncDesc exprNodeDesc = new ExprNodeGenericFuncDesc();
+    exprNodeDesc.setTypeInfo(TypeInfoFactory.intTypeInfo);
+    ArrayList<ExprNodeDesc> params = new ArrayList<ExprNodeDesc>();
+    params.add(exprNodeDesc);
+    aggDesc.setParameters(params);
+    GenericUDFAbs absUdf = new GenericUDFAbs();
+    exprNodeDesc.setGenericUDF(absUdf);
+    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
+    ExprNodeColumnDesc colExprA = new ExprNodeColumnDesc(Integer.class, "col1", "T", false);
+    ExprNodeColumnDesc colExprB = new ExprNodeColumnDesc(Integer.class, "col2", "T", false);
+    children.add(colExprA);
+    exprNodeDesc.setChildExprs(children);
+
+    ArrayList<String> outputColumnNames = new ArrayList<String>();
+    outputColumnNames.add("_col0");
+
+    GroupByDesc desc = new GroupByDesc();
+    desc.setOutputColumnNames(outputColumnNames);
+    ArrayList<AggregationDesc> aggDescList = new ArrayList<AggregationDesc>();
+    aggDescList.add(aggDesc);
+
+    desc.setAggregators(aggDescList);
+
+    ArrayList<ExprNodeDesc> grpByKeys = new ArrayList<ExprNodeDesc>();
+    grpByKeys.add(colExprB);
+    desc.setKeys(grpByKeys);
+
+    GroupByOperator gbyOp = new GroupByOperator();
+    gbyOp.setConf(desc);
+
+    Vectorizer v = new Vectorizer();
+    Assert.assertTrue(v.validateOperator(gbyOp));
+    VectorGroupByOperator vectorOp = (VectorGroupByOperator) v.vectorizeOperator(gbyOp, vContext);
+    Assert.assertEquals(VectorUDAFSumLong.class, vectorOp.getAggregators()[0].getClass());
+    VectorUDAFSumLong udaf = (VectorUDAFSumLong) vectorOp.getAggregators()[0];
+    Assert.assertEquals(FuncAbsLongToLong.class, udaf.getInputExpression().getClass());
+  }
+}

Added: hive/trunk/ql/src/test/queries/clientpositive/vectorization_nested_udf.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/vectorization_nested_udf.q?rev=1531391&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/vectorization_nested_udf.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/vectorization_nested_udf.q Fri Oct 11 18:50:33
2013
@@ -0,0 +1,3 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT SUM(abs(ctinyint)) from alltypesorc;
+

Added: hive/trunk/ql/src/test/results/clientpositive/vectorization_nested_udf.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vectorization_nested_udf.q.out?rev=1531391&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/vectorization_nested_udf.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/vectorization_nested_udf.q.out Fri Oct 11
18:50:33 2013
@@ -0,0 +1,9 @@
+PREHOOK: query: SELECT SUM(abs(ctinyint)) from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(abs(ctinyint)) from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+261468



Mime
View raw message