carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [29/47] incubator-carbondata git commit: [CARBONDATA-113][Bug]fix precision of DecimalType for CarbonAggregates(#875)
Date Mon, 01 Aug 2016 10:05:27 GMT
[CARBONDATA-113][Bug]fix precision of DecimalType for CarbonAggregates(#875)

[CARBONDATA-113][Bug]fix precision of DecimalType for CarbonAggregates(#875)


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4d2f6842
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4d2f6842
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4d2f6842

Branch: refs/heads/master
Commit: 4d2f68427c6b3b2fc95175ebab16952e0999b939
Parents: adba597
Author: Gin-zhj <zhujin2@huawei.com>
Authored: Thu Jul 28 19:51:28 2016 +0800
Committer: david <qiangcai@qq.com>
Committed: Thu Jul 28 19:51:28 2016 +0800

----------------------------------------------------------------------
 .../carbondata/spark/agg/CarbonAggregates.scala | 28 +++++++-------------
 .../carbondata/spark/util/CarbonScalaUtil.scala | 16 +++++------
 .../testsuite/bigdecimal/TestBigDecimal.scala   | 23 +++++++++++++++-
 3 files changed, 39 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4d2f6842/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
b/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
index de685ab..50872ff 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/agg/CarbonAggregates.scala
@@ -116,7 +116,7 @@ case class AverageCarbon(child: Expression, castedDataType: DataType =
null)
       AverageCarbonFinal(partialSum.toAttribute,
         child.dataType match {
           case IntegerType | StringType | LongType | TimestampType => DoubleType
-          case _ => child.dataType
+          case _ => CarbonScalaUtil.updateDataType(child.dataType)
         }),
       partialSum :: Nil)
   }
@@ -153,7 +153,11 @@ case class SumCarbon(child: Expression, castedDataType: DataType = null)
     val partialSum = Alias(SumCarbon(child), "PartialSum")()
     SplitEvaluation(
       SumCarbonFinal(partialSum.toAttribute,
-        if (castedDataType != null) castedDataType else child.dataType),
+        if (castedDataType != null) {
+          castedDataType
+        } else {
+          CarbonScalaUtil.updateDataType(child.dataType)
+        }),
       partialSum :: Nil)
   }
 
@@ -265,7 +269,7 @@ case class SumDistinctCarbon(child: Expression, castedDataType: DataType
= null)
         if (castedDataType != null) {
           castedDataType
         } else {
-          child.dataType
+          CarbonScalaUtil.updateDataType(child.dataType)
         }),
       partialSum :: Nil)
   }
@@ -369,10 +373,7 @@ case class AverageFunctionCarbon(expr: Expression, base: AggregateExpression1,
f
       } else {
         avg match {
           case avg: AvgBigDecimalAggregator =>
-            val decimalValue: BigDecimal = avg.getBigDecimalValue
-            val updatedDataType = CarbonScalaUtil
-              .getDecimalDataTypeWithUpdatedPrecision(decimalValue, base.dataType)
-            Cast(Literal(decimalValue), updatedDataType).eval(null)
+            Cast(Literal(avg.getBigDecimalValue), base.dataType).eval(null)
           case avg: AvgLongAggregator =>
             Cast(Literal(avg.getDoubleValue), base.dataType).eval(null)
           case avg: AvgTimestampAggregator =>
@@ -489,10 +490,7 @@ case class SumFunctionCarbon(expr: Expression, base: AggregateExpression1,
final
       } else {
         sum match {
           case s: SumBigDecimalAggregator =>
-            val decimalValue: BigDecimal = sum.getBigDecimalValue
-            val updatedDataType = CarbonScalaUtil
-              .getDecimalDataTypeWithUpdatedPrecision(decimalValue, base.dataType)
-            Cast(Literal(decimalValue), updatedDataType).eval(input)
+            Cast(Literal(sum.getBigDecimalValue), base.dataType).eval(input)
           case s: SumLongAggregator =>
             Cast(Literal(sum.getLongValue), base.dataType).eval(input)
           case _ =>
@@ -689,13 +687,7 @@ case class SumDisctinctFunctionCarbon(expr: Expression, base: AggregateExpressio
         null
       }
       else {
-        val updatedDataType = base.dataType match {
-          case decimal: DecimalType =>
-            CarbonScalaUtil
-              .getDecimalDataTypeWithUpdatedPrecision(distinct.getBigDecimalValue, base.dataType)
-          case _ => base.dataType
-        }
-        Cast(Literal(distinct.getValueObject), updatedDataType).eval(null)
+        Cast(Literal(distinct.getValueObject), base.dataType).eval(null)
       }
     }
     else {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4d2f6842/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
index 20d45dc..b3effd3 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
@@ -105,19 +105,17 @@ object CarbonScalaUtil {
     }
   }
 
-  def getDecimalDataTypeWithUpdatedPrecision(decimalValue: java.math.BigDecimal,
+  def updateDataType(
       currentDataType: org.apache.spark.sql.types.DataType): org.apache.spark.sql.types.DataType
= {
-    var newDataType: org.apache.spark.sql.types.DataType = currentDataType
-    if (null != decimalValue) {
-      val precision = decimalValue.precision
-      if (precision <= DecimalType.MAX_PRECISION) {
-        newDataType = DecimalType(precision, decimalValue.scale())
-      }
+    currentDataType match {
+      case decimal: DecimalType =>
+        val scale = currentDataType.asInstanceOf[DecimalType].scale
+        DecimalType(DecimalType.MAX_PRECISION, scale)
+      case _ =>
+        currentDataType
     }
-    newDataType
   }
 
-
   case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
 
   object CarbonSparkUtil {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4d2f6842/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
index b07705d..fb28568 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/bigdecimal/TestBigDecimal.scala
@@ -136,7 +136,28 @@ class TestBigDecimal extends QueryTest with BeforeAndAfterAll {
 
     sql("drop table if exists decimalDictLookUp")
   }
-  
+
+  test("test sum aggregation on big decimal column with high precision") {
+    sql("drop table if exists carbonBigDecimal")
+    sql("create table if not exists carbonBigDecimal (ID Int, date Timestamp, country String,
name String, phonetype String, serialname String, salary decimal(30, 10)) STORED BY 'org.apache.carbondata.format'")
+    sql("LOAD DATA LOCAL INPATH './src/test/resources/decimalBoundaryDataCarbon.csv' into
table carbonBigDecimal")
+
+    checkAnswer(sql("select sum(salary)+10 from carbonBigDecimal"),
+      sql("select sum(salary)+10 from hiveBigDecimal"))
+
+    sql("drop table if exists carbonBigDecimal")
+  }
+
+  test("test sum-distinct aggregation on big decimal column with high precision") {
+    sql("drop table if exists carbonBigDecimal")
+    sql("create table if not exists carbonBigDecimal (ID Int, date Timestamp, country String,
name String, phonetype String, serialname String, salary decimal(30, 10)) STORED BY 'org.apache.carbondata.format'")
+    sql("LOAD DATA LOCAL INPATH './src/test/resources/decimalBoundaryDataCarbon.csv' into
table carbonBigDecimal")
+
+    checkAnswer(sql("select sum(distinct(salary))+10 from carbonBigDecimal"),
+      sql("select sum(distinct(salary))+10 from hiveBigDecimal"))
+
+    sql("drop table if exists carbonBigDecimal")
+  }
 
   override def afterAll {
     sql("drop table if exists carbonTable")


Mime
View raw message