hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jiten...@apache.org
Subject svn commit: r1575633 - in /hive/branches/branch-0.13: common/src/java/org/apache/hadoop/hive/common/type/ ql/src/gen/vectorization/UDAFTemplates/ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/ ql/src/test/org/apache/hadoop/hi...
Date Sat, 08 Mar 2014 23:31:13 GMT
Author: jitendra
Date: Sat Mar  8 23:31:13 2014
New Revision: 1575633

URL: http://svn.apache.org/r1575633
Log:
HIVE-6508 : Mismatched aggregation results between vector and non-vector mode with decimal
field (Remus Rusanu via jitendra)

Modified:
    hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java
    hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
    hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
    hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
    hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java

Modified: hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
--- hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java
(original)
+++ hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java
Sat Mar  8 23:31:13 2014
@@ -261,8 +261,19 @@ public final class Decimal128 extends Nu
    *          object to copy from
    */
   public Decimal128 update(Decimal128 o) {
+    update(o, o.scale);
+    return this;
+  }
+
+  /**
+   * Copy the value of given object and assigns a custom scale.
+   *
+   * @param o
+   *          object to copy from
+   */
+  public Decimal128 update(Decimal128 o, short scale) {
     this.unscaledValue.update(o.unscaledValue);
-    this.scale = o.scale;
+    this.scale = scale;
     this.signum = o.signum;
     return this;
   }

Modified: hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
(original)
+++ hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt
Sat Mar  8 23:31:13 2014
@@ -60,12 +60,12 @@ public class <ClassName> extends VectorA
         value = new Decimal128();
       }
 
-      public void checkValue(Decimal128 value) {
+      public void checkValue(Decimal128 value, short scale) {
         if (isNull) {
           isNull = false;
           this.value.update(value);
         } else if (this.value.compareTo(value) <OperatorSymbol> 0) {
-          this.value.update(value);
+          this.value.update(value, scale);
         }
       }
 
@@ -124,16 +124,16 @@ public class <ClassName> extends VectorA
         if (inputVector.isRepeating) {
           iterateNoNullsRepeatingWithAggregationSelection(
             aggregationBufferSets, aggregrateIndex,
-            vector[0], batchSize);
+            vector[0], inputVector.scale, batchSize);
         } else {
           if (batch.selectedInUse) {
             iterateNoNullsSelectionWithAggregationSelection(
               aggregationBufferSets, aggregrateIndex,
-              vector, batch.selected, batchSize);
+              vector, inputVector.scale, batch.selected, batchSize);
           } else {
             iterateNoNullsWithAggregationSelection(
               aggregationBufferSets, aggregrateIndex,
-              vector, batchSize);
+              vector, inputVector.scale, batchSize);
           }
         }
       } else {
@@ -141,21 +141,21 @@ public class <ClassName> extends VectorA
           if (batch.selectedInUse) {
             iterateHasNullsRepeatingSelectionWithAggregationSelection(
               aggregationBufferSets, aggregrateIndex,
-              vector[0], batchSize, batch.selected, inputVector.isNull);
+              vector[0], inputVector.scale, batchSize, batch.selected, inputVector.isNull);
           } else {
             iterateHasNullsRepeatingWithAggregationSelection(
               aggregationBufferSets, aggregrateIndex,
-              vector[0], batchSize, inputVector.isNull);
+              vector[0], inputVector.scale, batchSize, inputVector.isNull);
           }
         } else {
           if (batch.selectedInUse) {
             iterateHasNullsSelectionWithAggregationSelection(
               aggregationBufferSets, aggregrateIndex,
-              vector, batchSize, batch.selected, inputVector.isNull);
+              vector, inputVector.scale, batchSize, batch.selected, inputVector.isNull);
           } else {
             iterateHasNullsWithAggregationSelection(
               aggregationBufferSets, aggregrateIndex,
-              vector, batchSize, inputVector.isNull);
+              vector, inputVector.scale, batchSize, inputVector.isNull);
           }
         }
       }
@@ -165,6 +165,7 @@ public class <ClassName> extends VectorA
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregrateIndex,
       Decimal128 value,
+      short scale,
       int batchSize) {
 
       for (int i=0; i < batchSize; ++i) {
@@ -172,7 +173,7 @@ public class <ClassName> extends VectorA
           aggregationBufferSets,
           aggregrateIndex,
           i);
-        myagg.checkValue(value);
+        myagg.checkValue(value, scale);
       }
     }
 
@@ -180,6 +181,7 @@ public class <ClassName> extends VectorA
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregrateIndex,
       Decimal128[] values,
+      short scale,
       int[] selection,
       int batchSize) {
 
@@ -188,7 +190,7 @@ public class <ClassName> extends VectorA
           aggregationBufferSets,
           aggregrateIndex,
           i);
-        myagg.checkValue(values[selection[i]]);
+        myagg.checkValue(values[selection[i]], scale);
       }
     }
 
@@ -196,13 +198,14 @@ public class <ClassName> extends VectorA
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregrateIndex,
       Decimal128[] values,
+      short scale,
       int batchSize) {
       for (int i=0; i < batchSize; ++i) {
         Aggregation myagg = getCurrentAggregationBuffer(
           aggregationBufferSets,
           aggregrateIndex,
           i);
-        myagg.checkValue(values[i]);
+        myagg.checkValue(values[i], scale);
       }
     }
 
@@ -210,6 +213,7 @@ public class <ClassName> extends VectorA
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregrateIndex,
       Decimal128 value,
+      short scale,
       int batchSize,
       int[] selection,
       boolean[] isNull) {
@@ -220,7 +224,7 @@ public class <ClassName> extends VectorA
             aggregationBufferSets,
             aggregrateIndex,
             i);
-          myagg.checkValue(value);
+          myagg.checkValue(value, scale);
         }
       }
 
@@ -230,6 +234,7 @@ public class <ClassName> extends VectorA
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregrateIndex,
       Decimal128 value,
+      short scale,
       int batchSize,
       boolean[] isNull) {
 
@@ -239,7 +244,7 @@ public class <ClassName> extends VectorA
             aggregationBufferSets,
             aggregrateIndex,
             i);
-          myagg.checkValue(value);
+          myagg.checkValue(value, scale);
         }
       }
     }
@@ -248,6 +253,7 @@ public class <ClassName> extends VectorA
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregrateIndex,
       Decimal128[] values,
+      short scale,
       int batchSize,
       int[] selection,
       boolean[] isNull) {
@@ -259,7 +265,7 @@ public class <ClassName> extends VectorA
             aggregationBufferSets,
             aggregrateIndex,
             j);
-          myagg.checkValue(values[i]);
+          myagg.checkValue(values[i], scale);
         }
       }
    }
@@ -268,6 +274,7 @@ public class <ClassName> extends VectorA
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregrateIndex,
       Decimal128[] values,
+      short scale,
       int batchSize,
       boolean[] isNull) {
 
@@ -277,7 +284,7 @@ public class <ClassName> extends VectorA
             aggregationBufferSets,
             aggregrateIndex,
             i);
-          myagg.checkValue(values[i]);
+          myagg.checkValue(values[i], scale);
         }
       }
    }
@@ -305,28 +312,31 @@ public class <ClassName> extends VectorA
           if (inputVector.noNulls &&
             (myagg.isNull || (myagg.value.compareTo(vector[0]) <OperatorSymbol> 0)))
{
             myagg.isNull = false;
-            myagg.value.update(vector[0]);
+            myagg.value.update(vector[0], inputVector.scale);
           }
           return;
         }
 
         if (!batch.selectedInUse && inputVector.noNulls) {
-          iterateNoSelectionNoNulls(myagg, vector, batchSize);
+          iterateNoSelectionNoNulls(myagg, vector, inputVector.scale, batchSize);
         }
         else if (!batch.selectedInUse) {
-          iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull);
+          iterateNoSelectionHasNulls(myagg, vector, inputVector.scale, 
+            batchSize, inputVector.isNull);
         }
         else if (inputVector.noNulls){
-          iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected);
+          iterateSelectionNoNulls(myagg, vector, inputVector.scale, batchSize, batch.selected);
         }
         else {
-          iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected);
+          iterateSelectionHasNulls(myagg, vector, inputVector.scale, 
+            batchSize, inputVector.isNull, batch.selected);
         }
     }
 
     private void iterateSelectionHasNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         boolean[] isNull,
         int[] selected) {
@@ -340,7 +350,7 @@ public class <ClassName> extends VectorA
             myagg.value.update(value);
           }
           else if (myagg.value.compareTo(value) <OperatorSymbol> 0) {
-            myagg.value.update(value);
+            myagg.value.update(value, scale);
           }
         }
       }
@@ -349,6 +359,7 @@ public class <ClassName> extends VectorA
     private void iterateSelectionNoNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         int[] selected) {
 
@@ -360,7 +371,7 @@ public class <ClassName> extends VectorA
       for (int i=0; i< batchSize; ++i) {
         Decimal128 value = vector[selected[i]];
         if (myagg.value.compareTo(value) <OperatorSymbol> 0) {
-          myagg.value.update(value);
+          myagg.value.update(value, scale);
         }
       }
     }
@@ -368,6 +379,7 @@ public class <ClassName> extends VectorA
     private void iterateNoSelectionHasNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         boolean[] isNull) {
 
@@ -375,11 +387,11 @@ public class <ClassName> extends VectorA
         if (!isNull[i]) {
           Decimal128 value = vector[i];
           if (myagg.isNull) {
-            myagg.value.update(value);
+            myagg.value.update(value, scale);
             myagg.isNull = false;
           }
           else if (myagg.value.compareTo(value) <OperatorSymbol> 0) {
-            myagg.value.update(value);
+            myagg.value.update(value, scale);
           }
         }
       }
@@ -388,6 +400,7 @@ public class <ClassName> extends VectorA
     private void iterateNoSelectionNoNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize) {
       if (myagg.isNull) {
         myagg.value.update(vector[0]);
@@ -397,7 +410,7 @@ public class <ClassName> extends VectorA
       for (int i=0;i<batchSize;++i) {
         Decimal128 value = vector[i];
         if (myagg.value.compareTo(value) <OperatorSymbol> 0) {
-          myagg.value.update(value);
+          myagg.value.update(value, scale);
         }
       }
     }

Modified: hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
(original)
+++ hive/branches/branch-0.13/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt
Sat Mar  8 23:31:13 2014
@@ -79,27 +79,27 @@ public class <ClassName> extends VectorA
         throw new UnsupportedOperationException();
       }
 
-      public void updateValueWithCheckAndInit(Decimal128 scratch, Decimal128 value) {
+      public void updateValueWithCheckAndInit(Decimal128 scratch, Decimal128 value, short
scale) {
         if (this.isNull) {
           this.init();
         }
-        this.sum.addDestructive(value, value.getScale());
+        this.sum.addDestructive(value, scale);
         this.count += 1;
         if(this.count > 1) {
            scratch.update(count);
-           scratch.multiplyDestructive(value, value.getScale());
-           scratch.subtractDestructive(sum, sum.getScale());
+           scratch.multiplyDestructive(value, scale);
+           scratch.subtractDestructive(sum, scale);
            double t = scratch.doubleValue();
            this.variance += (t*t) / ((double)this.count*(this.count-1));
         }
       }
 
-      public void updateValueNoCheck(Decimal128 scratch, Decimal128 value) {
-        this.sum.addDestructive(value, value.getScale());
+      public void updateValueNoCheck(Decimal128 scratch, Decimal128 value, short scale) {
+        this.sum.addDestructive(value, scale);
         this.count += 1;
         scratch.update(count);
-        scratch.multiplyDestructive(value, value.getScale());
-        scratch.subtractDestructive(sum, sum.getScale());
+        scratch.multiplyDestructive(value, scale);
+        scratch.subtractDestructive(sum, scale);
         double t = scratch.doubleValue();
         this.variance += (t*t) / ((double)this.count*(this.count-1));
       }
@@ -181,24 +181,26 @@ public class <ClassName> extends VectorA
       if (inputVector.isRepeating) {
         if (inputVector.noNulls || !inputVector.isNull[0]) {
           iterateRepeatingNoNullsWithAggregationSelection(
-            aggregationBufferSets, aggregateIndex, vector[0], batchSize);
+            aggregationBufferSets, aggregateIndex, vector[0], inputVector.scale, batchSize);
         }
       }
       else if (!batch.selectedInUse && inputVector.noNulls) {
         iterateNoSelectionNoNullsWithAggregationSelection(
-            aggregationBufferSets, aggregateIndex, vector, batchSize);
+            aggregationBufferSets, aggregateIndex, vector, inputVector.scale, batchSize);
       }
       else if (!batch.selectedInUse) {
         iterateNoSelectionHasNullsWithAggregationSelection(
-            aggregationBufferSets, aggregateIndex, vector, batchSize, inputVector.isNull);
+            aggregationBufferSets, aggregateIndex, vector, inputVector.scale, 
+            batchSize, inputVector.isNull);
       }
       else if (inputVector.noNulls){
         iterateSelectionNoNullsWithAggregationSelection(
-            aggregationBufferSets, aggregateIndex, vector, batchSize, batch.selected);
+            aggregationBufferSets, aggregateIndex, vector, inputVector.scale, 
+            batchSize, batch.selected);
       }
       else {
         iterateSelectionHasNullsWithAggregationSelection(
-            aggregationBufferSets, aggregateIndex, vector, batchSize,
+            aggregationBufferSets, aggregateIndex, vector, inputVector.scale, batchSize,
             inputVector.isNull, batch.selected);
       }
 
@@ -208,6 +210,7 @@ public class <ClassName> extends VectorA
         VectorAggregationBufferRow[] aggregationBufferSets,
         int aggregateIndex,
         Decimal128 value,
+        short scale,
         int batchSize) {
 
       for (int i=0; i<batchSize; ++i) {
@@ -215,7 +218,7 @@ public class <ClassName> extends VectorA
           aggregationBufferSets,
           aggregateIndex,
           i);
-        myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+        myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
       }
     }
 
@@ -223,6 +226,7 @@ public class <ClassName> extends VectorA
         VectorAggregationBufferRow[] aggregationBufferSets,
         int aggregateIndex,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         boolean[] isNull,
         int[] selected) {
@@ -235,7 +239,7 @@ public class <ClassName> extends VectorA
         int i = selected[j];
         if (!isNull[i]) {
           Decimal128 value = vector[i];
-          myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+          myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
         }
       }
     }
@@ -244,6 +248,7 @@ public class <ClassName> extends VectorA
         VectorAggregationBufferRow[] aggregationBufferSets,
         int aggregateIndex,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         int[] selected) {
 
@@ -253,7 +258,7 @@ public class <ClassName> extends VectorA
           aggregateIndex,
           i);
         Decimal128 value = vector[selected[i]];
-        myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+        myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
       }
     }
 
@@ -261,6 +266,7 @@ public class <ClassName> extends VectorA
         VectorAggregationBufferRow[] aggregationBufferSets,
         int aggregateIndex,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         boolean[] isNull) {
 
@@ -271,7 +277,7 @@ public class <ClassName> extends VectorA
             aggregateIndex,
           i);
           Decimal128 value = vector[i];
-          myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+          myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
         }
       }
     }
@@ -280,6 +286,7 @@ public class <ClassName> extends VectorA
         VectorAggregationBufferRow[] aggregationBufferSets,
         int aggregateIndex,
         Decimal128[] vector,
+        short scale,
         int batchSize) {
 
       for (int i=0; i<batchSize; ++i) {
@@ -288,7 +295,7 @@ public class <ClassName> extends VectorA
           aggregateIndex,
           i);
         Decimal128 value = vector[i];
-        myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+        myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
       }
     }
 
@@ -313,42 +320,45 @@ public class <ClassName> extends VectorA
 
       if (inputVector.isRepeating) {
         if (inputVector.noNulls) {
-          iterateRepeatingNoNulls(myagg, vector[0], batchSize);
+          iterateRepeatingNoNulls(myagg, vector[0], inputVector.scale, batchSize);
         }
       }
       else if (!batch.selectedInUse && inputVector.noNulls) {
-        iterateNoSelectionNoNulls(myagg, vector, batchSize);
+        iterateNoSelectionNoNulls(myagg, vector, inputVector.scale, batchSize);
       }
       else if (!batch.selectedInUse) {
-        iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull);
+        iterateNoSelectionHasNulls(myagg, vector, inputVector.scale, batchSize, inputVector.isNull);
       }
       else if (inputVector.noNulls){
-        iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected);
+        iterateSelectionNoNulls(myagg, vector, inputVector.scale, batchSize, batch.selected);
       }
       else {
-        iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected);
+        iterateSelectionHasNulls(myagg, vector, inputVector.scale, 
+          batchSize, inputVector.isNull, batch.selected);
       }
     }
 
     private void  iterateRepeatingNoNulls(
         Aggregation myagg,
         Decimal128 value,
+        short scale,
         int batchSize) {
 
       // TODO: conjure a formula w/o iterating
       //
 
-      myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+      myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
 
       // We pulled out i=0 so we can remove the count > 1 check in the loop
       for (int i=1; i<batchSize; ++i) {
-        myagg.updateValueNoCheck(scratchDecimal, value);
+        myagg.updateValueNoCheck(scratchDecimal, value, scale);
       }
     }
 
     private void iterateSelectionHasNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         boolean[] isNull,
         int[] selected) {
@@ -357,7 +367,7 @@ public class <ClassName> extends VectorA
         int i = selected[j];
         if (!isNull[i]) {
           Decimal128 value = vector[i];
-          myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+          myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
         }
       }
     }
@@ -365,6 +375,7 @@ public class <ClassName> extends VectorA
     private void iterateSelectionNoNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         int[] selected) {
 
@@ -373,26 +384,27 @@ public class <ClassName> extends VectorA
       }
 
       Decimal128 value = vector[selected[0]];
-      myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+      myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
 
       // i=0 was pulled out to remove the count > 1 check in the loop
       //
       for (int i=1; i< batchSize; ++i) {
         value = vector[selected[i]];
-        myagg.updateValueNoCheck(scratchDecimal, value);
+        myagg.updateValueNoCheck(scratchDecimal, value, scale);
       }
     }
 
     private void iterateNoSelectionHasNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         boolean[] isNull) {
 
       for(int i=0;i<batchSize;++i) {
         if (!isNull[i]) {
           Decimal128 value = vector[i];
-          myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+          myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
         }
       }
     }
@@ -400,6 +412,7 @@ public class <ClassName> extends VectorA
     private void iterateNoSelectionNoNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize) {
 
       if (myagg.isNull) {
@@ -407,12 +420,12 @@ public class <ClassName> extends VectorA
       }
 
       Decimal128 value = vector[0];
-      myagg.updateValueWithCheckAndInit(scratchDecimal, value);
+      myagg.updateValueWithCheckAndInit(scratchDecimal, value, scale);
 
       // i=0 was pulled out to remove count > 1 check
       for (int i=1; i<batchSize; ++i) {
         value = vector[i];
-        myagg.updateValueNoCheck(scratchDecimal, value);
+        myagg.updateValueNoCheck(scratchDecimal, value, scale);
       }
     }
 

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java
Sat Mar  8 23:31:13 2014
@@ -51,12 +51,12 @@ public class VectorUDAFSumDecimal extend
       transient private Decimal128 sum = new Decimal128();
       transient private boolean isNull;
 
-      public void sumValue(Decimal128 value) {
+      public void sumValue(Decimal128 value, short scale) {
         if (isNull) {
-          sum.update(value);
+          sum.update(value, scale);
           isNull = false;
         } else {
-          sum.addDestructive(value, value.getScale());
+          sum.addDestructive(value, scale);
         }
       }
 
@@ -110,16 +110,19 @@ public class VectorUDAFSumDecimal extend
         if (inputVector.isRepeating) {
           iterateNoNullsRepeatingWithAggregationSelection(
             aggregationBufferSets, aggregateIndex,
-            vector[0], batchSize);
+            vector[0], inputVector.scale,
+            batchSize);
         } else {
           if (batch.selectedInUse) {
             iterateNoNullsSelectionWithAggregationSelection(
               aggregationBufferSets, aggregateIndex,
-              vector, batch.selected, batchSize);
+              vector, inputVector.scale,
+              batch.selected, batchSize);
           } else {
             iterateNoNullsWithAggregationSelection(
               aggregationBufferSets, aggregateIndex,
-              vector, batchSize);
+              vector, inputVector.scale,
+              batchSize);
           }
         }
       } else {
@@ -127,21 +130,25 @@ public class VectorUDAFSumDecimal extend
           if (batch.selectedInUse) {
             iterateHasNullsRepeatingSelectionWithAggregationSelection(
               aggregationBufferSets, aggregateIndex,
-              vector[0], batchSize, batch.selected, inputVector.isNull);
+              vector[0], inputVector.scale,
+              batchSize, batch.selected, inputVector.isNull);
           } else {
             iterateHasNullsRepeatingWithAggregationSelection(
               aggregationBufferSets, aggregateIndex,
-              vector[0], batchSize, inputVector.isNull);
+              vector[0], inputVector.scale,
+              batchSize, inputVector.isNull);
           }
         } else {
           if (batch.selectedInUse) {
             iterateHasNullsSelectionWithAggregationSelection(
               aggregationBufferSets, aggregateIndex,
-              vector, batchSize, batch.selected, inputVector.isNull);
+              vector, inputVector.scale,
+              batchSize, batch.selected, inputVector.isNull);
           } else {
             iterateHasNullsWithAggregationSelection(
               aggregationBufferSets, aggregateIndex,
-              vector, batchSize, inputVector.isNull);
+              vector,inputVector.scale,
+              batchSize, inputVector.isNull);
           }
         }
       }
@@ -151,6 +158,7 @@ public class VectorUDAFSumDecimal extend
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregateIndex,
       Decimal128 value,
+      short scale,
       int batchSize) {
 
       for (int i=0; i < batchSize; ++i) {
@@ -158,7 +166,7 @@ public class VectorUDAFSumDecimal extend
           aggregationBufferSets,
           aggregateIndex,
           i);
-        myagg.sumValue(value);
+        myagg.sumValue(value, scale);
       }
     }
 
@@ -166,6 +174,7 @@ public class VectorUDAFSumDecimal extend
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregateIndex,
       Decimal128[] values,
+      short scale,
       int[] selection,
       int batchSize) {
 
@@ -174,7 +183,7 @@ public class VectorUDAFSumDecimal extend
           aggregationBufferSets,
           aggregateIndex,
           i);
-        myagg.sumValue(values[selection[i]]);
+        myagg.sumValue(values[selection[i]], scale);
       }
     }
 
@@ -182,13 +191,14 @@ public class VectorUDAFSumDecimal extend
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregateIndex,
       Decimal128[] values,
+      short scale,
       int batchSize) {
       for (int i=0; i < batchSize; ++i) {
         Aggregation myagg = getCurrentAggregationBuffer(
           aggregationBufferSets,
           aggregateIndex,
           i);
-        myagg.sumValue(values[i]);
+        myagg.sumValue(values[i], scale);
       }
     }
 
@@ -196,6 +206,7 @@ public class VectorUDAFSumDecimal extend
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregateIndex,
       Decimal128 value,
+      short scale,
       int batchSize,
       int[] selection,
       boolean[] isNull) {
@@ -206,7 +217,7 @@ public class VectorUDAFSumDecimal extend
             aggregationBufferSets,
             aggregateIndex,
             i);
-          myagg.sumValue(value);
+          myagg.sumValue(value, scale);
         }
       }
 
@@ -216,6 +227,7 @@ public class VectorUDAFSumDecimal extend
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregateIndex,
       Decimal128 value,
+      short scale,
       int batchSize,
       boolean[] isNull) {
 
@@ -225,7 +237,7 @@ public class VectorUDAFSumDecimal extend
             aggregationBufferSets,
             aggregateIndex,
             i);
-          myagg.sumValue(value);
+          myagg.sumValue(value, scale);
         }
       }
     }
@@ -234,6 +246,7 @@ public class VectorUDAFSumDecimal extend
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregateIndex,
       Decimal128[] values,
+      short scale,
       int batchSize,
       int[] selection,
       boolean[] isNull) {
@@ -245,7 +258,7 @@ public class VectorUDAFSumDecimal extend
             aggregationBufferSets,
             aggregateIndex,
             j);
-          myagg.sumValue(values[i]);
+          myagg.sumValue(values[i], scale);
         }
       }
    }
@@ -254,6 +267,7 @@ public class VectorUDAFSumDecimal extend
       VectorAggregationBufferRow[] aggregationBufferSets,
       int aggregateIndex,
       Decimal128[] values,
+      short scale,
       int batchSize,
       boolean[] isNull) {
 
@@ -263,7 +277,7 @@ public class VectorUDAFSumDecimal extend
             aggregationBufferSets,
             aggregateIndex,
             i);
-          myagg.sumValue(values[i]);
+          myagg.sumValue(values[i], scale);
         }
       }
    }
@@ -295,29 +309,30 @@ public class VectorUDAFSumDecimal extend
           myagg.sum.zeroClear();
         }
         scratchDecimal.update(batchSize);
-        scratchDecimal.multiplyDestructive(vector[0], vector[0].getScale());
+        scratchDecimal.multiplyDestructive(vector[0], inputVector.scale);
         myagg.sum.update(scratchDecimal);
       }
         return;
       }
 
       if (!batch.selectedInUse && inputVector.noNulls) {
-        iterateNoSelectionNoNulls(myagg, vector, batchSize);
+        iterateNoSelectionNoNulls(myagg, vector, inputVector.scale, batchSize);
       }
       else if (!batch.selectedInUse) {
-        iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull);
+        iterateNoSelectionHasNulls(myagg, vector, inputVector.scale, batchSize, inputVector.isNull);
       }
       else if (inputVector.noNulls){
-        iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected);
+        iterateSelectionNoNulls(myagg, vector, inputVector.scale, batchSize, batch.selected);
       }
       else {
-        iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected);
+        iterateSelectionHasNulls(myagg, vector, inputVector.scale, batchSize, inputVector.isNull,
batch.selected);
       }
     }
 
     private void iterateSelectionHasNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         boolean[] isNull,
         int[] selected) {
@@ -330,7 +345,7 @@ public class VectorUDAFSumDecimal extend
             myagg.isNull = false;
             myagg.sum.zeroClear();
           }
-          myagg.sum.addDestructive(value, value.getScale());
+          myagg.sum.addDestructive(value, scale);
         }
       }
     }
@@ -338,6 +353,7 @@ public class VectorUDAFSumDecimal extend
     private void iterateSelectionNoNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         int[] selected) {
 
@@ -348,13 +364,14 @@ public class VectorUDAFSumDecimal extend
 
       for (int i=0; i< batchSize; ++i) {
         Decimal128 value = vector[selected[i]];
-        myagg.sum.addDestructive(value, value.getScale());
+        myagg.sum.addDestructive(value, scale);
       }
     }
 
     private void iterateNoSelectionHasNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize,
         boolean[] isNull) {
 
@@ -365,7 +382,7 @@ public class VectorUDAFSumDecimal extend
             myagg.sum.zeroClear();
             myagg.isNull = false;
           }
-          myagg.sum.addDestructive(value, value.getScale());
+          myagg.sum.addDestructive(value, scale);
         }
       }
     }
@@ -373,6 +390,7 @@ public class VectorUDAFSumDecimal extend
     private void iterateNoSelectionNoNulls(
         Aggregation myagg,
         Decimal128[] vector,
+        short scale,
         int batchSize) {
       if (myagg.isNull) {
         myagg.sum.zeroClear();
@@ -381,7 +399,7 @@ public class VectorUDAFSumDecimal extend
 
       for (int i=0;i<batchSize;++i) {
         Decimal128 value = vector[i];
-        myagg.sum.addDestructive(value, value.getScale());
+        myagg.sum.addDestructive(value, scale);
       }
     }
 

Modified: hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
(original)
+++ hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
Sat Mar  8 23:31:13 2014
@@ -603,6 +603,7 @@ public class TestVectorGroupByOperator {
   @Test
   public void testCountDecimal() throws HiveException {
     testAggregateDecimal(
+        "Decimal",
         "count",
         2,
         Arrays.asList(new Object[]{
@@ -615,6 +616,7 @@ public class TestVectorGroupByOperator {
   @Test
   public void testMaxDecimal() throws HiveException {
     testAggregateDecimal(
+        "Decimal",
         "max",
         2,
         Arrays.asList(new Object[]{
@@ -623,26 +625,29 @@ public class TestVectorGroupByOperator {
                 new Decimal128(3)}),
        new Decimal128(3));
     testAggregateDecimal(
-            "max",
-            2,
-            Arrays.asList(new Object[]{
-                    new Decimal128(3),
-                    new Decimal128(2),
-                    new Decimal128(1)}),
-           new Decimal128(3));
+        "Decimal",
+        "max",
+        2,
+        Arrays.asList(new Object[]{
+                new Decimal128(3),
+                new Decimal128(2),
+                new Decimal128(1)}),
+        new Decimal128(3));
     testAggregateDecimal(
-            "max",
-            2,
-            Arrays.asList(new Object[]{
-                    new Decimal128(2),
-                    new Decimal128(3),
-                    new Decimal128(1)}),
-           new Decimal128(3));
+        "Decimal",
+        "max",
+        2,
+        Arrays.asList(new Object[]{
+                new Decimal128(2),
+                new Decimal128(3),
+                new Decimal128(1)}),
+        new Decimal128(3));
   }
 
   @Test
   public void testMinDecimal() throws HiveException {
     testAggregateDecimal(
+        "Decimal",
         "min",
         2,
         Arrays.asList(new Object[]{
@@ -651,28 +656,31 @@ public class TestVectorGroupByOperator {
                 new Decimal128(3)}),
        new Decimal128(1));
     testAggregateDecimal(
-            "min",
-            2,
-            Arrays.asList(new Object[]{
-                    new Decimal128(3),
-                    new Decimal128(2),
-                    new Decimal128(1)}),
-           new Decimal128(1));
+        "Decimal",
+        "min",
+        2,
+        Arrays.asList(new Object[]{
+                new Decimal128(3),
+                new Decimal128(2),
+                new Decimal128(1)}),
+        new Decimal128(1));
 
     testAggregateDecimal(
-          "min",
-          2,
-          Arrays.asList(new Object[]{
-                  new Decimal128(2),
-                  new Decimal128(1),
-                  new Decimal128(3)}),
-         new Decimal128(1));
+        "Decimal",
+        "min",
+        2,
+        Arrays.asList(new Object[]{
+                new Decimal128(2),
+                new Decimal128(1),
+                new Decimal128(3)}),
+        new Decimal128(1));
   }
 
   @Test
   public void testSumDecimal() throws HiveException {
     testAggregateDecimal(
-        "sum",
+        "Decimal",
+       "sum",
         2,
         Arrays.asList(new Object[]{
                 new Decimal128(1),
@@ -682,8 +690,25 @@ public class TestVectorGroupByOperator {
   }
 
   @Test
+  public void testSumDecimalHive6508() throws HiveException {
+    short scale = 4;
+    testAggregateDecimal(
+        "Decimal(10,4)",
+        "sum",
+        4,
+        Arrays.asList(new Object[]{
+                new Decimal128("1234.2401", scale),
+                new Decimal128("1868.52", scale),
+                new Decimal128(0L, (short) 0),
+                new Decimal128("456.84", scale),
+                new Decimal128("121.89", scale)}),
+       new Decimal128("3681.4901", scale));
+  }
+
+  @Test
   public void testAvgDecimal() throws HiveException {
     testAggregateDecimal(
+        "Decimal",
         "avg",
         2,
         Arrays.asList(new Object[]{
@@ -696,18 +721,20 @@ public class TestVectorGroupByOperator {
   @Test
   public void testAvgDecimalNegative() throws HiveException {
     testAggregateDecimal(
-            "avg",
-            2,
-            Arrays.asList(new Object[]{
-                    new Decimal128(-1),
-                    new Decimal128(-2),
-                    new Decimal128(-3)}),
-           HiveDecimal.create((-1-2-3)/3));
+        "Decimal",
+        "avg",
+        2,
+        Arrays.asList(new Object[]{
+                new Decimal128(-1),
+                new Decimal128(-2),
+                new Decimal128(-3)}),
+        HiveDecimal.create((-1-2-3)/3));
   }
 
   @Test
   public void testVarianceDecimal () throws HiveException {
       testAggregateDecimal(
+        "Decimal",
         "variance",
         2,
         Arrays.asList(new Object[]{
@@ -721,6 +748,7 @@ public class TestVectorGroupByOperator {
   @Test
   public void testVarSampDecimal () throws HiveException {
       testAggregateDecimal(
+        "Decimal",
         "var_samp",
         2,
         Arrays.asList(new Object[]{
@@ -734,6 +762,7 @@ public class TestVectorGroupByOperator {
   @Test
   public void testStdPopDecimal () throws HiveException {
       testAggregateDecimal(
+        "Decimal",
         "stddev_pop",
         2,
         Arrays.asList(new Object[]{
@@ -747,6 +776,7 @@ public class TestVectorGroupByOperator {
   @Test
   public void testStdSampDecimal () throws HiveException {
       testAggregateDecimal(
+        "Decimal",
         "stddev_samp",
         2,
         Arrays.asList(new Object[]{
@@ -1936,14 +1966,15 @@ public class TestVectorGroupByOperator {
   }
 
   public void testAggregateDecimal (
-          String aggregateName,
-          int batchSize,
-          Iterable<Object> values,
-          Object expected) throws HiveException {
+      String typeName,
+      String aggregateName,
+      int batchSize,
+      Iterable<Object> values,
+      Object expected) throws HiveException {
 
         @SuppressWarnings("unchecked")
         FakeVectorRowBatchFromObjectIterables fdr = new FakeVectorRowBatchFromObjectIterables(
-            batchSize, new String[] {"Decimal"}, values);
+            batchSize, new String[] {typeName}, values);
         testAggregateDecimalIterable (aggregateName, fdr, expected);
       }
 

Modified: hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java?rev=1575633&r1=1575632&r2=1575633&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
(original)
+++ hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorRowBatchFromObjectIterables.java
Sat Mar  8 23:31:13 2014
@@ -22,6 +22,9 @@ import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.regex.MatchResult;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.hadoop.hive.common.type.Decimal128;
 import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
@@ -141,7 +144,23 @@ public class FakeVectorRowBatchFromObjec
           }
         };
       } else if (types[i].toLowerCase().startsWith("decimal")) {
-            batch.cols[i] = new DecimalColumnVector(batchSize, 38, 0);
+            Pattern decimalPattern = Pattern.compile(
+                "decimal(?:\\((\\d+)(?:\\,(\\d+))?\\))?", Pattern.CASE_INSENSITIVE);
+            Matcher mr = decimalPattern.matcher(types[i]);
+            int precission = 38;
+            int scale = 0;
+            if (mr.matches()) {
+              String typePrecission = mr.group(1);
+              if (typePrecission != null) {
+                precission = Integer.parseInt(typePrecission);
+              }
+              String typeScale = mr.group(2);
+              if (typeScale != null) {
+                scale = Integer.parseInt(typeScale);
+              }
+            }
+
+            batch.cols[i] = new DecimalColumnVector(batchSize, precission, scale);
             columnAssign[i] = new ColumnVectorAssign() {
                 @Override
                 public void assign(



Mime
View raw message