carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [3/7] carbondata git commit: [CARBONDATA-2029] Fixed Pre Aggregate table issue with expression
Date Tue, 16 Jan 2018 15:36:28 GMT
[CARBONDATA-2029] Fixed Pre Aggregate table issue with expression

Query with expression is giving wrong result

This closes #1803


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/04d40ccb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/04d40ccb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/04d40ccb

Branch: refs/heads/carbonstore
Commit: 04d40ccb911199bad4825292214c5ba34910f12a
Parents: 0d23461
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Mon Jan 15 16:26:36 2018 +0530
Committer: kunal642 <kunalkapoor642@gmail.com>
Committed: Tue Jan 16 14:48:22 2018 +0530

----------------------------------------------------------------------
 .../TestPreAggregateTableSelection.scala         |  5 +++++
 .../spark/sql/hive/CarbonPreAggregateRules.scala | 19 ++++++++++++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/04d40ccb/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
index 9bbba3a..559e91f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
@@ -262,6 +262,11 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll
{
     preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
   }
 
+  test("Test query with math operation hitting fact table") {
+    val df =  sql("select sum(id)+count(id) from maintable")
+    preAggTableValidator(df.queryExecution.analyzed, "maintable")
+  }
+
   override def afterAll: Unit = {
     sql("drop table if exists mainTable")
     sql("drop table if exists lineitem")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/04d40ccb/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
index de554c5..299ed4d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
@@ -1013,6 +1013,7 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession)
extends Rule
       carbonTable: CarbonTable,
       queryColumns: scala.collection.mutable.HashSet[QueryColumn],
       aggreagteExps: scala.collection.mutable.HashSet[AggregateExpression]): Boolean = {
+    var isValid = true
     groupByExpression foreach  { expression =>
       extractColumnFromExpression(expression, queryColumns, carbonTable)
     }
@@ -1025,13 +1026,13 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession)
extends Rule
           carbonTable);
       case Alias(attr: AggregateExpression, _) =>
         if (attr.isDistinct) {
-          return false
+          isValid = false
         }
         val aggExp = PreAggregateUtil.validateAggregateFunctionAndGetFields(attr)
         if (aggExp.nonEmpty) {
           aggreagteExps ++= aggExp
         } else {
-          return false
+          isValid = false
         }
       case Alias(expression: Expression, _) =>
         if (expression.isInstanceOf[ScalaUDF] &&
@@ -1049,10 +1050,22 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession)
extends Rule
               queryColumns += getQueryColumn(attr.name,
                 carbonTable)
               attr
+            case attr: AggregateExpression =>
+              if (attr.isDistinct) {
+                isValid = false
+              }
+              val aggExp = PreAggregateUtil.validateAggregateFunctionAndGetFields(attr)
+              if (aggExp.nonEmpty) {
+                aggreagteExps ++= aggExp
+              } else {
+                isValid = false
+              }
+              attr
+
           }
         }
     }
-    true
+    isValid
   }
 
   /**


Mime
View raw message