carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kumarvisha...@apache.org
Subject carbondata git commit: [CARBONDATA-1930] Added condition to refer to parent dictionary if filter is given in aggregate table query
Date Fri, 22 Dec 2017 16:11:48 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 406d75442 -> 1381c9f19


[CARBONDATA-1930] Added condition to refer to parent dictionary if filter is given in aggregate
table query

When filter is applied to aggregate query then the query is not accessing the parent table
dictionary instead it is trying to search for aggregate table dictionary files.Add check to
access parent table dictionary files if aggregate column has parent columns.

This closes #1710


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1381c9f1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1381c9f1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1381c9f1

Branch: refs/heads/master
Commit: 1381c9f19508f68f95953c48126e27ca239818d1
Parents: 406d754
Author: kunal642 <kunalkapoor642@gmail.com>
Authored: Fri Dec 22 13:23:11 2017 +0530
Committer: kumarvishal <kumarvishal.1802@gmail.com>
Committed: Fri Dec 22 21:41:31 2017 +0530

----------------------------------------------------------------------
 .../carbondata/core/scan/filter/FilterUtil.java     | 16 +++++++++++++++-
 .../TestPreAggregateTableSelection.scala            | 11 +++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/1381c9f1/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index a8fa8ec..3268ca3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -53,6 +53,7 @@ import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
 import org.apache.carbondata.core.keygenerator.KeyGenException;
 import org.apache.carbondata.core.keygenerator.KeyGenerator;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -61,6 +62,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
 import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
 import org.apache.carbondata.core.scan.expression.ColumnExpression;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.ExpressionResult;
@@ -1241,15 +1243,27 @@ public final class FilterUtil {
       AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier,
       CarbonDimension carbonDimension, TableProvider tableProvider) throws IOException {
     String dictionaryPath = null;
+    ColumnIdentifier columnIdentifier = carbonDimension.getColumnIdentifier();
     if (null != tableProvider) {
       CarbonTable carbonTable = tableProvider
           .getCarbonTable(dictionarySourceAbsoluteTableIdentifier.getCarbonTableIdentifier());
       dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties()
           .get(CarbonCommonConstants.DICTIONARY_PATH);
+      if (null != carbonDimension.getColumnSchema().getParentColumnTableRelations() &&
+          carbonDimension.getColumnSchema().getParentColumnTableRelations().size() == 1)
{
+        dictionarySourceAbsoluteTableIdentifier =
+            QueryUtil.getTableIdentifierForColumn(carbonDimension,
+                carbonTable.getAbsoluteTableIdentifier());
+        columnIdentifier = new ColumnIdentifier(
+            carbonDimension.getColumnSchema().getParentColumnTableRelations().get(0).getColumnId(),
+            carbonDimension.getColumnProperties(), carbonDimension.getDataType());
+      } else {
+        dictionarySourceAbsoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier();
+      }
     }
     DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
         new DictionaryColumnUniqueIdentifier(dictionarySourceAbsoluteTableIdentifier,
-            carbonDimension.getColumnIdentifier(), carbonDimension.getDataType(), dictionaryPath);
+            columnIdentifier, carbonDimension.getDataType(), dictionaryPath);
     CacheProvider cacheProvider = CacheProvider.getInstance();
     Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache =
         cacheProvider.createCache(CacheType.FORWARD_DICTIONARY);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1381c9f1/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
index 7157b8a..fdeb2bd 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
@@ -219,6 +219,17 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll
{
     }
   }
 
+  test("test if pre-agg table is hit with filter condition") {
+    sql("drop table if exists filtertable")
+    sql("CREATE TABLE filtertable(id int, name string, city string, age string) STORED BY"
+
+        " 'org.apache.carbondata.format' TBLPROPERTIES('dictionary_include'='name,age')")
+    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/measureinsertintotest.csv' into table filtertable")
+    sql("create datamap agg9 on table filtertable using 'preaggregate' as select name, age,
sum(age) from filtertable group by name, age")
+    val df = sql("select name, sum(age) from filtertable where age = '29' group by name,
age")
+    preAggTableValidator(df.queryExecution.analyzed, "filtertable_agg9")
+    checkAnswer(df, Row("vishal", 29))
+  }
+
   override def afterAll: Unit = {
     sql("drop table if exists mainTable")
     sql("drop table if exists lineitem")


Mime
View raw message