hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From eh...@apache.org
Subject svn commit: r1557826 - in /hive/trunk: ant/src/org/apache/hadoop/hive/ant/ ql/src/gen/vectorization/ExpressionTemplates/ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/
Date Mon, 13 Jan 2014 19:44:07 GMT
Author: ehans
Date: Mon Jan 13 19:44:06 2014
New Revision: 1557826

URL: http://svn.apache.org/r1557826
Log:
HIVE-6067: Implement vectorized decimal comparison filters (Eric Hanson)

Added:
    hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareColumn.txt
    hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareScalar.txt
    hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalScalarCompareColumn.txt
Modified:
    hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java

Modified: hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
URL: http://svn.apache.org/viewvc/hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java?rev=1557826&r1=1557825&r2=1557826&view=diff
==============================================================================
--- hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java (original)
+++ hive/trunk/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java Mon Jan 13 19:44:06 2014
@@ -235,6 +235,27 @@ public class GenVectorCode extends Task 
       {"FilterStringScalarCompareColumn", "Greater", ">"},
       {"FilterStringScalarCompareColumn", "GreaterEqual", ">="},
 
+      {"FilterDecimalColumnCompareScalar", "Equal", "=="},
+      {"FilterDecimalColumnCompareScalar", "NotEqual", "!="},
+      {"FilterDecimalColumnCompareScalar", "Less", "<"},
+      {"FilterDecimalColumnCompareScalar", "LessEqual", "<="},
+      {"FilterDecimalColumnCompareScalar", "Greater", ">"},
+      {"FilterDecimalColumnCompareScalar", "GreaterEqual", ">="},
+
+      {"FilterDecimalScalarCompareColumn", "Equal", "=="},
+      {"FilterDecimalScalarCompareColumn", "NotEqual", "!="},
+      {"FilterDecimalScalarCompareColumn", "Less", "<"},
+      {"FilterDecimalScalarCompareColumn", "LessEqual", "<="},
+      {"FilterDecimalScalarCompareColumn", "Greater", ">"},
+      {"FilterDecimalScalarCompareColumn", "GreaterEqual", ">="},
+
+      {"FilterDecimalColumnCompareColumn", "Equal", "=="},
+      {"FilterDecimalColumnCompareColumn", "NotEqual", "!="},
+      {"FilterDecimalColumnCompareColumn", "Less", "<"},
+      {"FilterDecimalColumnCompareColumn", "LessEqual", "<="},
+      {"FilterDecimalColumnCompareColumn", "Greater", ">"},
+      {"FilterDecimalColumnCompareColumn", "GreaterEqual", ">="},
+
       {"StringScalarCompareColumn", "Equal", "=="},
       {"StringScalarCompareColumn", "NotEqual", "!="},
       {"StringScalarCompareColumn", "Less", "<"},
@@ -593,6 +614,12 @@ public class GenVectorCode extends Task 
         generateIfExprScalarColumn(tdesc);
       } else if (tdesc[0].equals("IfExprScalarScalar")) {
         generateIfExprScalarScalar(tdesc);
+      } else if (tdesc[0].equals("FilterDecimalColumnCompareScalar")) {
+        generateFilterDecimalColumnCompareScalar(tdesc);
+      } else if (tdesc[0].equals("FilterDecimalScalarCompareColumn")) {
+        generateFilterDecimalScalarCompareColumn(tdesc);
+      } else if (tdesc[0].equals("FilterDecimalColumnCompareColumn")) {
+        generateFilterDecimalColumnCompareColumn(tdesc);
       } else {
         continue;
       }
@@ -1126,6 +1153,39 @@ public class GenVectorCode extends Task 
     generateScalarBinaryOperatorColumn(tdesc, returnType, className);
   }
 
+  private void generateFilterDecimalColumnCompareScalar(String[] tdesc) throws IOException
{
+    String operatorName = tdesc[1];
+    String className = "FilterDecimalCol" + operatorName + "DecimalScalar";
+    generateDecimalColumnCompare(tdesc, className);
+  }
+
+  private void generateFilterDecimalScalarCompareColumn(String[] tdesc) throws IOException
{
+    String operatorName = tdesc[1];
+    String className = "FilterDecimalScalar" + operatorName + "DecimalColumn";
+    generateDecimalColumnCompare(tdesc, className);
+  }
+
+  private void generateFilterDecimalColumnCompareColumn(String[] tdesc) throws IOException
{
+    String operatorName = tdesc[1];
+    String className = "FilterDecimalCol" + operatorName + "DecimalColumn";
+    generateDecimalColumnCompare(tdesc, className);
+  }
+
+  private void generateDecimalColumnCompare(String[] tdesc, String className)
+      throws IOException {
+    String operatorSymbol = tdesc[2];
+
+    // Read the template into a string;
+    File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+    String templateString = readFile(templateFile);
+
+    // Expand, and write result
+    templateString = templateString.replaceAll("<ClassName>", className);
+    templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+    writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+        className, templateString);
+  }
+
   static void writeFile(long templateTime, String outputDir, String classesDir,
        String className, String str) throws IOException {
     File outputFile = new File(outputDir, className + ".java");

Added: hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareColumn.txt
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareColumn.txt?rev=1557826&view=auto
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareColumn.txt
(added)
+++ hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareColumn.txt
Mon Jan 13 19:44:06 2014
@@ -0,0 +1,461 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.common.type.Decimal128;
+
+/**
+ * Generated from template FilterDecimalColumnCompareColumn.txt, which covers binary comparison

+ * filter expressions between two columns. Output is not produced in a separate column. 
+ * The selected vector of the input {@link VectorizedRowBatch} is updated for in-place filtering.
+ */
+public class <ClassName> extends VectorExpression {
+
+  private static final long serialVersionUID = 1L;
+
+  private int colNum1;
+  private int colNum2;
+
+  public <ClassName>(int colNum1, int colNum2) { 
+    this.colNum1 = colNum1;
+    this.colNum2 = colNum2;
+  }
+
+  public <ClassName>() {
+  }
+
+  @Override
+  public void evaluate(VectorizedRowBatch batch) {
+
+    if (childExpressions != null) {
+      super.evaluateChildren(batch);
+    }
+
+    DecimalColumnVector inputColVector1 = (DecimalColumnVector) batch.cols[colNum1];
+    DecimalColumnVector inputColVector2 = (DecimalColumnVector) batch.cols[colNum2];
+    int[] sel = batch.selected;
+    boolean[] nullPos1 = inputColVector1.isNull;
+    boolean[] nullPos2 = inputColVector2.isNull;
+    int n = batch.size;
+    Decimal128[] vector1 = inputColVector1.vector;
+    Decimal128[] vector2 = inputColVector2.vector;
+
+    // return immediately if batch is empty
+    if (n == 0) {
+      return;
+    }
+    
+    // handle case where neither input has nulls
+    if (inputColVector1.noNulls && inputColVector2.noNulls) {
+      if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+      
+        /* Either all must remain selected or all will be eliminated.
+         * Repeating property will not change.
+         */
+        if (!(vector1[0].compareTo(vector2[0]) <OperatorSymbol> 0)) {
+          batch.size = 0;
+        }      
+      } else if (inputColVector1.isRepeating) {
+        if (batch.selectedInUse) {
+          int newSize = 0;
+          for(int j = 0; j != n; j++) {
+            int i = sel[j];
+            if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+              sel[newSize++] = i;
+            }
+          }
+          batch.size = newSize;
+        } else {
+          int newSize = 0;
+          for(int i = 0; i != n; i++) {
+            if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+              sel[newSize++] = i;
+            }
+          }
+          if (newSize < batch.size) {
+            batch.size = newSize;
+            batch.selectedInUse = true;
+          }
+        }
+      } else if (inputColVector2.isRepeating) {
+        if (batch.selectedInUse) {
+          int newSize = 0;
+          for(int j = 0; j != n; j++) {
+            int i = sel[j];
+            if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+              sel[newSize++] = i;
+            }
+          }
+          batch.size = newSize;
+        } else {
+          int newSize = 0;
+          for(int i = 0; i != n; i++) {
+            if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+              sel[newSize++] = i;
+            }
+          }
+          if (newSize < batch.size) {
+            batch.size = newSize;
+            batch.selectedInUse = true;
+          }
+        }
+      } else if (batch.selectedInUse) {
+        int newSize = 0;
+        for(int j = 0; j != n; j++) {
+          int i = sel[j];
+          if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+            sel[newSize++] = i;
+          }
+        }
+        batch.size = newSize;
+      } else {
+        int newSize = 0;
+        for(int i = 0; i != n; i++) {
+          if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+            sel[newSize++] = i;
+          }
+        }
+        if (newSize < batch.size) {
+          batch.size = newSize;
+          batch.selectedInUse = true;
+        }
+      }
+    
+    // handle case where only input 2 has nulls
+    } else if (inputColVector1.noNulls) { 
+      if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+        if (nullPos2[0] ||
+            !(vector1[0].compareTo(vector2[0]) <OperatorSymbol> 0)) {
+          batch.size = 0; 
+        } 
+      } else if (inputColVector1.isRepeating) {
+         
+         // no need to check for nulls in input 1
+         if (batch.selectedInUse) {
+          int newSize = 0;
+          for(int j = 0; j != n; j++) {
+            int i = sel[j];
+            if (!nullPos2[i]) {
+              if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          batch.size = newSize;
+        } else {
+          int newSize = 0;
+          for(int i = 0; i != n; i++) {
+            if (!nullPos2[i]) {
+              if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          if (newSize < batch.size) {
+            batch.size = newSize;
+            batch.selectedInUse = true;
+          }
+        }
+      } else if (inputColVector2.isRepeating) {
+        if (nullPos2[0]) {
+        
+          // no values will qualify because every comparison will be with NULL
+          batch.size = 0;
+          return;
+        }
+        if (batch.selectedInUse) {
+          int newSize = 0;
+          for(int j = 0; j != n; j++) {
+            int i = sel[j];
+            if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+              sel[newSize++] = i;
+            }
+          }
+          batch.size = newSize;
+        } else {
+          int newSize = 0;
+          for(int i = 0; i != n; i++) {
+            if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+              sel[newSize++] = i;
+            }
+          }
+          if (newSize < batch.size) {
+            batch.size = newSize;
+            batch.selectedInUse = true;
+          }
+        }
+      } else { // neither input is repeating
+        if (batch.selectedInUse) {
+          int newSize = 0;
+          for(int j = 0; j != n; j++) {
+            int i = sel[j];
+            if (!nullPos2[i]) {
+              if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          batch.size = newSize;
+        } else {
+          int newSize = 0;
+          for(int i = 0; i != n; i++) {
+            if (!nullPos2[i]) {
+              if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          if (newSize < batch.size) {
+            batch.size = newSize;
+            batch.selectedInUse = true;
+          }
+        }      
+      }
+      
+    // handle case where only input 1 has nulls
+    } else if (inputColVector2.noNulls) {
+      if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+        if (nullPos1[0] ||
+            !(vector1[0].compareTo(vector2[0]) <OperatorSymbol> 0)) {
+          batch.size = 0; 
+          return;
+        } 
+      } else if (inputColVector1.isRepeating) {
+        if (nullPos1[0]) {
+        
+          // if repeating value is null then every comparison will fail so nothing qualifies
+          batch.size = 0;
+          return; 
+        }
+        if (batch.selectedInUse) {
+          int newSize = 0;
+          for(int j = 0; j != n; j++) {
+            int i = sel[j];
+            if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+              sel[newSize++] = i;
+            }
+          }
+          batch.size = newSize;
+        } else {
+          int newSize = 0;
+          for(int i = 0; i != n; i++) {
+            if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+              sel[newSize++] = i;
+            }
+          }
+          if (newSize < batch.size) {
+            batch.size = newSize;
+            batch.selectedInUse = true;
+          }
+        }
+      } else if (inputColVector2.isRepeating) {
+         if (batch.selectedInUse) {
+          int newSize = 0;
+          for(int j = 0; j != n; j++) {
+            int i = sel[j];
+            if (!nullPos1[i]) {
+              if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          batch.size = newSize;
+        } else {
+          int newSize = 0;
+          for(int i = 0; i != n; i++) {
+            if (!nullPos1[i]) {
+              if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          if (newSize < batch.size) {
+            batch.size = newSize;
+            batch.selectedInUse = true;
+          }
+        }
+      } else { // neither input is repeating
+         if (batch.selectedInUse) {
+          int newSize = 0;
+          for(int j = 0; j != n; j++) {
+            int i = sel[j];
+            if (!nullPos1[i]) {
+              if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          batch.size = newSize;
+        } else {
+          int newSize = 0;
+          for(int i = 0; i != n; i++) {
+            if (!nullPos1[i]) {
+              if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          if (newSize < batch.size) {
+            batch.size = newSize;
+            batch.selectedInUse = true;
+          }
+        }      
+      }
+            
+    // handle case where both inputs have nulls
+    } else {
+      if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
+        if (nullPos1[0] || nullPos2[0] ||
+            !(vector1[0].compareTo(vector2[0]) <OperatorSymbol> 0)) {
+          batch.size = 0; 
+        } 
+      } else if (inputColVector1.isRepeating) {
+         if (nullPos1[0]) {
+           batch.size = 0;
+           return;
+         }
+         if (batch.selectedInUse) {
+          int newSize = 0;
+          for(int j = 0; j != n; j++) {
+            int i = sel[j];
+            if (!nullPos2[i]) {
+              if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          batch.size = newSize;
+        } else {
+          int newSize = 0;
+          for(int i = 0; i != n; i++) {
+            if (!nullPos2[i]) {
+              if (vector1[0].compareTo(vector2[i]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          if (newSize < batch.size) {
+            batch.size = newSize;
+            batch.selectedInUse = true;
+          }
+        }
+      } else if (inputColVector2.isRepeating) {
+        if (nullPos2[0]) {
+          batch.size = 0;
+          return;
+        }
+        if (batch.selectedInUse) {
+          int newSize = 0;
+          for(int j = 0; j != n; j++) {
+            int i = sel[j];
+            if (!nullPos1[i]) {
+              if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          batch.size = newSize;
+        } else {
+          int newSize = 0;
+          for(int i = 0; i != n; i++) {
+            if (!nullPos1[i]) {
+              if (vector1[i].compareTo(vector2[0]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          if (newSize < batch.size) {
+            batch.size = newSize;
+            batch.selectedInUse = true;
+          }
+        }
+      } else { // neither input is repeating
+         if (batch.selectedInUse) {
+          int newSize = 0;
+          for(int j = 0; j != n; j++) {
+            int i = sel[j];
+            if (!nullPos1[i] && !nullPos2[i]) {
+              if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          batch.size = newSize;
+        } else {
+          int newSize = 0;
+          for(int i = 0; i != n; i++) {
+            if (!nullPos1[i] && !nullPos2[i]) {
+              if (vector1[i].compareTo(vector2[i]) <OperatorSymbol> 0) {
+                sel[newSize++] = i;
+              }
+            }
+          }
+          if (newSize < batch.size) {
+            batch.size = newSize;
+            batch.selectedInUse = true;
+          }
+        }      
+      } 
+    }
+  }
+
+  @Override
+  public String getOutputType() {
+    return "boolean";
+  }
+
+  @Override
+  public int getOutputColumn() {
+    return -1;
+  }
+  
+  public int getColNum1() {
+    return colNum1;
+  }
+
+  public void setColNum1(int colNum1) {
+    this.colNum1 = colNum1;
+  }
+
+  public int getColNum2() {
+    return colNum2;
+  }
+
+  public void setColNum2(int colNum2) {
+    this.colNum2 = colNum2;
+  }
+
+  @Override
+  public VectorExpressionDescriptor.Descriptor getDescriptor() {
+    return (new VectorExpressionDescriptor.Builder())
+        .setMode(
+            VectorExpressionDescriptor.Mode.FILTER)
+        .setNumArguments(2)
+        .setArgumentTypes(
+            VectorExpressionDescriptor.ArgumentType.getType("decimal"),
+            VectorExpressionDescriptor.ArgumentType.getType("decimal"))
+        .setInputExpressionTypes(
+            VectorExpressionDescriptor.InputExpressionType.COLUMN,
+            VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
+  }
+}

Added: hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareScalar.txt
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareScalar.txt?rev=1557826&view=auto
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareScalar.txt
(added)
+++ hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalColumnCompareScalar.txt
Mon Jan 13 19:44:06 2014
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.common.type.Decimal128;
+
+/**
+ * This is a generated class to evaluate a <OperatorSymbol> comparison on a vector
of decimal
+ * values.
+ */
+public class <ClassName> extends VectorExpression {
+
+  private static final long serialVersionUID = 1L;
+
+  private int colNum;
+  private Decimal128 value;
+
+  public <ClassName>(int colNum, Decimal128 value) {
+    this.colNum = colNum;
+    this.value = value;
+  }
+
+  public <ClassName>() {
+  }
+
+  @Override
+  public void evaluate(VectorizedRowBatch batch) {
+    if (childExpressions != null) {
+      super.evaluateChildren(batch);
+    }
+    DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[colNum];
+    int[] sel = batch.selected;
+    boolean[] nullPos = inputColVector.isNull;
+    int n = batch.size;
+    Decimal128[] vector = inputColVector.vector;
+
+    // return immediately if batch is empty
+    if (n == 0) {
+      return;
+    }
+
+    if (inputColVector.noNulls) {
+      if (inputColVector.isRepeating) {
+
+        // All must be selected otherwise size would be zero. Repeating property will not
change.
+        if (!(vector[0].compareTo(value) <OperatorSymbol> 0)) {
+
+          // Entire batch is filtered out.
+          batch.size = 0;
+        }
+      } else if (batch.selectedInUse) {
+        int newSize = 0;
+        for(int j = 0; j != n; j++) {
+          int i = sel[j];
+          if (vector[i].compareTo(value) <OperatorSymbol> 0) {
+            sel[newSize++] = i;
+          }
+        }
+        batch.size = newSize;
+      } else {
+        int newSize = 0;
+        for(int i = 0; i != n; i++) {
+          if (vector[i].compareTo(value) <OperatorSymbol> 0) {
+            sel[newSize++] = i;
+          }
+        }
+        if (newSize < n) {
+          batch.size = newSize;
+          batch.selectedInUse = true;
+        }
+      }
+    } else {
+      if (inputColVector.isRepeating) {
+
+        // All must be selected otherwise size would be zero. Repeating property will not
change.
+        if (!nullPos[0]) {
+          if (!(vector[0].compareTo(value) <OperatorSymbol> 0)) {
+
+            // Entire batch is filtered out.
+            batch.size = 0;
+          }
+        } else {
+          batch.size = 0;
+        }
+      } else if (batch.selectedInUse) {
+        int newSize = 0;
+        for(int j = 0; j != n; j++) {
+          int i = sel[j];
+          if (!nullPos[i]) {
+           if (vector[i].compareTo(value) <OperatorSymbol> 0) {
+             sel[newSize++] = i;
+           }
+          }
+        }
+
+        // Change the selected vector
+        batch.size = newSize;
+      } else {
+        int newSize = 0;
+        for(int i = 0; i != n; i++) {
+          if (!nullPos[i]) {
+            if (vector[i].compareTo(value) <OperatorSymbol> 0) {
+              sel[newSize++] = i;
+            }
+          }
+        }
+        if (newSize < n) {
+          batch.size = newSize;
+          batch.selectedInUse = true;
+        }
+      }
+    }
+  }
+
+  @Override
+  public int getOutputColumn() {
+    return -1;
+  }
+
+  @Override
+  public String getOutputType() {
+    return "boolean";
+  }
+
+  public int getColNum() {
+    return colNum;
+  }
+
+  public void setColNum(int colNum) {
+    this.colNum = colNum;
+  }
+
+  public Decimal128 getValue() {
+    return value;
+  }
+
+  public void setValue(Decimal128 value) {
+    this.value = value;
+  }
+
+  @Override
+  public VectorExpressionDescriptor.Descriptor getDescriptor() {
+    return (new VectorExpressionDescriptor.Builder())
+        .setMode(
+            VectorExpressionDescriptor.Mode.FILTER)
+        .setNumArguments(2)
+        .setArgumentTypes(
+            VectorExpressionDescriptor.ArgumentType.getType("decimal"),
+            VectorExpressionDescriptor.ArgumentType.getType("decimal"))
+        .setInputExpressionTypes(
+            VectorExpressionDescriptor.InputExpressionType.COLUMN,
+            VectorExpressionDescriptor.InputExpressionType.SCALAR).build();
+  }
+}

Added: hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalScalarCompareColumn.txt
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalScalarCompareColumn.txt?rev=1557826&view=auto
==============================================================================
--- hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalScalarCompareColumn.txt
(added)
+++ hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/FilterDecimalScalarCompareColumn.txt
Mon Jan 13 19:44:06 2014
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector.expressions.gen;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor;
+import org.apache.hadoop.hive.common.type.Decimal128;
+
+/**
+ * This is a generated class to evaluate a <OperatorSymbol> comparison on a vector
of decimal
+ * values.
+ */
+public class <ClassName> extends VectorExpression {
+
+  private static final long serialVersionUID = 1L;
+
+  private int colNum;
+  private Decimal128 value;
+
+  public <ClassName>(Decimal128 value, int colNum) {
+    this.colNum = colNum;
+    this.value = value;
+  }
+
+  public <ClassName>() {
+  }
+
+  @Override
+  public void evaluate(VectorizedRowBatch batch) {
+    if (childExpressions != null) {
+      super.evaluateChildren(batch);
+    }
+    DecimalColumnVector inputColVector = (DecimalColumnVector) batch.cols[colNum];
+    int[] sel = batch.selected;
+    boolean[] nullPos = inputColVector.isNull;
+    int n = batch.size;
+    Decimal128[] vector = inputColVector.vector;
+
+    // return immediately if batch is empty
+    if (n == 0) {
+      return;
+    }
+
+    if (inputColVector.noNulls) {
+      if (inputColVector.isRepeating) {
+
+        // All must be selected otherwise size would be zero. Repeating property will not
change.
+        if (!(value.compareTo(vector[0]) <OperatorSymbol> 0)) {
+
+          // Entire batch is filtered out.
+          batch.size = 0;
+        }
+      } else if (batch.selectedInUse) {
+        int newSize = 0;
+        for(int j = 0; j != n; j++) {
+          int i = sel[j];
+          if (value.compareTo(vector[i]) <OperatorSymbol> 0) {
+            sel[newSize++] = i;
+          }
+        }
+        batch.size = newSize;
+      } else {
+        int newSize = 0;
+        for(int i = 0; i != n; i++) {
+          if (value.compareTo(vector[i]) <OperatorSymbol> 0) {
+            sel[newSize++] = i;
+          }
+        }
+        if (newSize < n) {
+          batch.size = newSize;
+          batch.selectedInUse = true;
+        }
+      }
+    } else {
+      if (inputColVector.isRepeating) {
+
+        // All must be selected otherwise size would be zero. Repeating property will not
change.
+        if (!nullPos[0]) {
+          if (!(value.compareTo(vector[0]) <OperatorSymbol> 0)) {
+
+            // Entire batch is filtered out.
+            batch.size = 0;
+          }
+        } else {
+          batch.size = 0;
+        }
+      } else if (batch.selectedInUse) {
+        int newSize = 0;
+        for(int j = 0; j != n; j++) {
+          int i = sel[j];
+          if (!nullPos[i]) {
+           if (value.compareTo(vector[i]) <OperatorSymbol> 0) {
+             sel[newSize++] = i;
+           }
+          }
+        }
+
+        // Change the selected vector
+        batch.size = newSize;
+      } else {
+        int newSize = 0;
+        for(int i = 0; i != n; i++) {
+          if (!nullPos[i]) {
+            if (value.compareTo(vector[i]) <OperatorSymbol> 0) {
+              sel[newSize++] = i;
+            }
+          }
+        }
+        if (newSize < n) {
+          batch.size = newSize;
+          batch.selectedInUse = true;
+        }
+      }
+    }
+  }
+
+  @Override
+  public int getOutputColumn() {
+    return -1;
+  }
+
+  @Override
+  public String getOutputType() {
+    return "boolean";
+  }
+
+  public int getColNum() {
+    return colNum;
+  }
+
+  public void setColNum(int colNum) {
+    this.colNum = colNum;
+  }
+
+  public Decimal128 getValue() {
+    return value;
+  }
+
+  public void setValue(Decimal128 value) {
+    this.value = value;
+  }
+
+  @Override
+  public VectorExpressionDescriptor.Descriptor getDescriptor() {
+    return (new VectorExpressionDescriptor.Builder())
+        .setMode(
+            VectorExpressionDescriptor.Mode.FILTER)
+        .setNumArguments(2)
+        .setArgumentTypes(
+            VectorExpressionDescriptor.ArgumentType.getType("decimal"),
+            VectorExpressionDescriptor.ArgumentType.getType("decimal"))
+        .setInputExpressionTypes(
+            VectorExpressionDescriptor.InputExpressionType.SCALAR,
+            VectorExpressionDescriptor.InputExpressionType.COLUMN).build();
+  }
+}

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java?rev=1557826&r1=1557825&r2=1557826&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
(original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorFilterExpressions.java
Mon Jan 13 19:44:06 2014
@@ -24,11 +24,16 @@ import static org.junit.Assert.assertTru
 
 import java.sql.Timestamp;
 
+import org.apache.hadoop.hive.common.type.Decimal128;
 import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.TimestampUtils;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColGreaterEqualDecimalColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColLessDecimalScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalScalarGreaterDecimalColumn;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnBetween;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDoubleColumnNotBetween;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColEqualLongScalar;
@@ -41,6 +46,9 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongScalarLessLongColumn;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnBetween;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColumnNotBetween;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColEqualDecimalScalar;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalColEqualDecimalColumn;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterDecimalScalarEqualDecimalColumn;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColAddLongScalar;
 import org.apache.hadoop.hive.ql.exec.vector.util.VectorizedRowGroupGenUtil;
 import org.junit.Assert;
@@ -784,4 +792,237 @@ public class TestVectorFilterExpressions
     expr.evaluate(vrb);
     assertEquals(0, vrb.size);
   }
+
+  /**
+   * This tests the template for Decimal Column-Scalar comparison filters,
+   * called FilterDecimalColumnCompareScalar.txt. Only equal is tested for
+   * multiple cases because the logic is the same for <, >, <=, >=, == and !=.
+   */
+  @Test
+  public void testFilterDecimalColEqualDecimalScalar() {
+    VectorizedRowBatch b = getVectorizedRowBatch1DecimalCol();
+    Decimal128 scalar = new Decimal128();
+    scalar.update("-3.30", (short) 2);
+    VectorExpression expr = new FilterDecimalColEqualDecimalScalar(0, scalar);
+    expr.evaluate(b);
+
+    // check that right row(s) are selected
+    assertTrue(b.selectedInUse);
+    assertEquals(1, b.selected[0]);
+    assertEquals(1, b.size);
+
+    // try again with a null value
+    b = getVectorizedRowBatch1DecimalCol();
+    b.cols[0].noNulls = false;
+    b.cols[0].isNull[1] = true;
+    expr.evaluate(b);
+
+    // verify that no rows were selected
+    assertEquals(0, b.size);
+
+    // try the repeating case
+    b = getVectorizedRowBatch1DecimalCol();
+    b.cols[0].isRepeating = true;
+    expr.evaluate(b);
+
+    // verify that no rows were selected
+    assertEquals(0, b.size);
+
+    // try the repeating null case
+    b = getVectorizedRowBatch1DecimalCol();
+    b.cols[0].isRepeating = true;
+    b.cols[0].noNulls = false;
+    b.cols[0].isNull[0] = true;
+    expr.evaluate(b);
+
+    // verify that no rows were selected
+    assertEquals(0, b.size);
+  }
+
+  /**
+   * This tests the template for Decimal Scalar-Column comparison filters,
+   * called FilterDecimalScalarCompareColumn.txt. Only equal is tested for multiple
+   * cases because the logic is the same for <, >, <=, >=, == and !=.
+   */
+  @Test
+  public void testFilterDecimalScalarEqualDecimalColumn() {
+    VectorizedRowBatch b = getVectorizedRowBatch1DecimalCol();
+    Decimal128 scalar = new Decimal128();
+    scalar.update("-3.30", (short) 2);
+    VectorExpression expr = new FilterDecimalScalarEqualDecimalColumn(scalar, 0);
+    expr.evaluate(b);
+
+    // check that right row(s) are selected
+    assertTrue(b.selectedInUse);
+    assertEquals(1, b.selected[0]);
+    assertEquals(1, b.size);
+
+    // try again with a null value
+    b = getVectorizedRowBatch1DecimalCol();
+    b.cols[0].noNulls = false;
+    b.cols[0].isNull[1] = true;
+    expr.evaluate(b);
+
+    // verify that no rows were selected
+    assertEquals(0, b.size);
+
+    // try the repeating case
+    b = getVectorizedRowBatch1DecimalCol();
+    b.cols[0].isRepeating = true;
+    expr.evaluate(b);
+
+    // verify that no rows were selected
+    assertEquals(0, b.size);
+
+    // try the repeating null case
+    b = getVectorizedRowBatch1DecimalCol();
+    b.cols[0].isRepeating = true;
+    b.cols[0].noNulls = false;
+    b.cols[0].isNull[0] = true;
+    expr.evaluate(b);
+
+    // verify that no rows were selected
+    assertEquals(0, b.size);
+  }
+
+  /**
+   * This tests the template for Decimal Column-Column comparison filters,
+   * called FilterDecimalColumnCompareColumn.txt. Only equal is tested for multiple
+   * cases because the logic is the same for <, >, <=, >=, == and !=.
+   */
+  @Test
+  public void testFilterDecimalColumnEqualDecimalColumn() {
+    VectorizedRowBatch b = getVectorizedRowBatch2DecimalCol();
+    VectorExpression expr = new FilterDecimalColEqualDecimalColumn(0, 1);
+    expr.evaluate(b);
+
+    // check that right row(s) are selected
+    assertTrue(b.selectedInUse);
+    assertEquals(1, b.selected[0]);
+    assertEquals(1, b.size);
+
+    // try again with a null value
+    b = getVectorizedRowBatch2DecimalCol();
+    b.cols[0].noNulls = false;
+    b.cols[0].isNull[1] = true;
+    expr.evaluate(b);
+
+    // verify that no rows were selected
+    assertEquals(0, b.size);
+
+    // try the repeating case
+    b = getVectorizedRowBatch2DecimalCol();
+    b.cols[0].isRepeating = true;
+    expr.evaluate(b);
+
+    // verify that no rows were selected
+    assertEquals(0, b.size);
+
+    // try the repeating null case
+    b = getVectorizedRowBatch2DecimalCol();
+    b.cols[0].isRepeating = true;
+    b.cols[0].noNulls = false;
+    b.cols[0].isNull[0] = true;
+    expr.evaluate(b);
+
+    // verify that no rows were selected
+    assertEquals(0, b.size);
+
+    // try nulls on both sides
+    b = getVectorizedRowBatch2DecimalCol();
+    b.cols[0].noNulls = false;
+    b.cols[0].isNull[0] = true;
+    b.cols[1].noNulls = false;
+    b.cols[1].isNull[2] = true;
+    expr.evaluate(b);
+    assertEquals(1, b.size);  // second of three was selected
+
+    // try repeating on both sides
+    b = getVectorizedRowBatch2DecimalCol();
+    b.cols[0].isRepeating = true;
+    b.cols[1].isRepeating = true;
+    expr.evaluate(b);
+
+    // verify that no rows were selected
+    assertEquals(0, b.size);
+  }
+
+  /**
+   * Spot check col < scalar for decimal.
+   */
+  @Test
+  public void testFilterDecimalColLessScalar() {
+    VectorizedRowBatch b = getVectorizedRowBatch1DecimalCol();
+    Decimal128 scalar = new Decimal128();
+    scalar.update("0", (short) 2);
+    VectorExpression expr = new FilterDecimalColLessDecimalScalar(0, scalar);
+    expr.evaluate(b);
+
+    // check that right row(s) are selected
+    assertTrue(b.selectedInUse);
+    assertEquals(1, b.selected[0]);
+    assertEquals(1, b.size);
+  }
+
+  /**
+   * Spot check scalar > col for decimal.
+   */
+  @Test
+  public void testFilterDecimalScalarGreaterThanColumn() {
+    VectorizedRowBatch b = getVectorizedRowBatch1DecimalCol();
+    Decimal128 scalar = new Decimal128();
+    scalar.update("0", (short) 2);
+    VectorExpression expr = new FilterDecimalScalarGreaterDecimalColumn(scalar, 0);
+    expr.evaluate(b);
+
+    // check that right row(s) are selected
+    assertTrue(b.selectedInUse);
+    assertEquals(1, b.selected[0]);
+    assertEquals(1, b.size);
+  }
+
+  /**
+   * Spot check col >= col for decimal.
+   */
+  @Test
+  public void testFilterDecimalColGreaterEqualCol() {
+    VectorizedRowBatch b = getVectorizedRowBatch2DecimalCol();
+    VectorExpression expr = new FilterDecimalColGreaterEqualDecimalColumn(0, 1);
+    expr.evaluate(b);
+
+    // check that right row(s) are selected
+    assertTrue(b.selectedInUse);
+    assertEquals(0, b.selected[0]);
+    assertEquals(1, b.selected[1]);
+    assertEquals(2, b.size);
+  }
+
+  private VectorizedRowBatch getVectorizedRowBatch1DecimalCol() {
+    VectorizedRowBatch b = new VectorizedRowBatch(1);
+    DecimalColumnVector v0;
+    b.cols[0] = v0 = new DecimalColumnVector(18, 2);
+    v0.vector[0].update("1.20", (short) 2);
+    v0.vector[1].update("-3.30", (short) 2);
+    v0.vector[2].update("0", (short) 2);
+
+    b.size = 3;
+    return b;
+  }
+
+  private VectorizedRowBatch getVectorizedRowBatch2DecimalCol() {
+    VectorizedRowBatch b = new VectorizedRowBatch(2);
+    DecimalColumnVector v0, v1;
+    b.cols[0] = v0 = new DecimalColumnVector(18, 2);
+    v0.vector[0].update("1.20", (short) 2);
+    v0.vector[1].update("-3.30", (short) 2);
+    v0.vector[2].update("0", (short) 2);
+
+    b.cols[1] = v1 = new DecimalColumnVector(18, 2);
+    v1.vector[0].update("-1", (short) 2);
+    v1.vector[1].update("-3.30", (short) 2);
+    v1.vector[2].update("10", (short) 2);
+
+    b.size = 3;
+    return b;
+  }
 }



Mime
View raw message