carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kunalkap...@apache.org
Subject carbondata git commit: [CARBONDATA-2606][Complex DataType Enhancements]Fix for ComplexDataType Projection PushDown
Date Tue, 03 Jul 2018 09:28:34 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 12c28c946 -> 133ec17e5


[CARBONDATA-2606][Complex DataType Enhancements]Fix for ComplexDataType Projection PushDown

Problem1: Fix for ComplexDataType Projection PushDown when Table Schema contains
ColumnName in UpperCase

Solution: Change ColumnName to Lowercase

Problem2: If Struct contains Array, pushdown only parent column

Solution: Check for ArrayType or GetArrayItem in the Complex Column,
if any ArrayType is found, then pushdown parent column

This closes #2421


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/133ec17e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/133ec17e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/133ec17e

Branch: refs/heads/master
Commit: 133ec17e527c22d8de1353f723fb6d77cd404eeb
Parents: 12c28c9
Author: Indhumathi27 <indhumathim27@gmail.com>
Authored: Wed Jun 27 13:01:28 2018 +0530
Committer: kunal642 <kunalkapoor642@gmail.com>
Committed: Tue Jul 3 14:49:31 2018 +0530

----------------------------------------------------------------------
 .../core/scan/model/QueryModelBuilder.java      |  6 ++--
 .../sql/CarbonDatasourceHadoopRelation.scala    | 37 +++++++++++++-------
 2 files changed, 29 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/133ec17e/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java
b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java
index c75f2bc..ced80f2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModelBuilder.java
@@ -82,13 +82,15 @@ public class QueryModelBuilder {
       }
     }
     projection = optimizeProjectionForComplexColumns(projection, projectionColumns, factTableName);
+    List<String> projectionDimensionAndMeasures = new ArrayList<>();
     this.projection = projection;
     for (ProjectionDimension projectionDimension : projection.getDimensions()) {
-      LOGGER.info("Project Columns: " + projectionDimension.getColumnName());
+      projectionDimensionAndMeasures.add(projectionDimension.getColumnName());
     }
     for (ProjectionMeasure projectionMeasure : projection.getMeasures()) {
-      LOGGER.info("Project Columns: " + projectionMeasure.getColumnName());
+      projectionDimensionAndMeasures.add(projectionMeasure.getColumnName());
     }
+    LOGGER.info("Projection Columns: " + projectionDimensionAndMeasures);
     return this;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/133ec17e/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index fad944c..86e9ace 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql
 
 import scala.collection.mutable.{ArrayBuffer, ListBuffer}
+import scala.util.control.Breaks._
 
 import org.apache.spark.CarbonInputMetrics
 import org.apache.spark.rdd.RDD
@@ -75,24 +76,34 @@ case class CarbonDatasourceHadoopRelation(
       CarbonFilters.createCarbonFilter(schema, filter)
     }.reduceOption(new AndExpression(_, _))
 
+    var parentColumn = new ListBuffer[String]
     // In case of Struct or StructofStruct Complex type, get the project column for given
     // parent/child field and pushdown the corresponding project column. In case of Array,
     // ArrayofStruct or StructofArray, pushdown parent column
     var reqColumns = projects.map {
       case a@Alias(s: GetStructField, name) =>
-        val arrayTypeExists = s.childSchema.map(x => x.dataType)
-          .filter(dataType => dataType.isInstanceOf[ArrayType])
-        val ifGetArrayItem = s.child.map(x => x.isInstanceOf[GetArrayItem])
-        var ifGetArrayItemExists = false
-        ifGetArrayItem.foreach(ifexists =>
-          if (ifexists.equals(true)) {
-            ifGetArrayItemExists = true
+        var arrayTypeExists = false
+        var ifGetArrayItemExists = s
+        breakable({
+          while (ifGetArrayItemExists.containsChild != null) {
+            if (ifGetArrayItemExists.child.isInstanceOf[AttributeReference]) {
+              arrayTypeExists = s.childSchema.toString().contains("ArrayType")
+              break
+            } else {
+              if (ifGetArrayItemExists.child.isInstanceOf[GetArrayItem]) {
+                arrayTypeExists = true
+                break
+              }
+              else {
+                ifGetArrayItemExists = ifGetArrayItemExists.child.asInstanceOf[GetStructField]
+              }
+            }
           }
-        )
-        if (0 == arrayTypeExists.length && ifGetArrayItemExists.equals(false)) {
+        })
+        if (!arrayTypeExists) {
+          parentColumn += s.toString().split("\\.")(0).replaceAll("#.*", "").toLowerCase
+          parentColumn = parentColumn.distinct
           s.toString().replaceAll("#[0-9]*", "").toLowerCase
-        } else if (ifGetArrayItemExists.equals(true)) {
-          s.toString().split("\\.")(0).replaceAll("#.*", "").toLowerCase
         } else {
           s.toString().split("\\.")(0).replaceAll("#.*", "").toLowerCase
         }
@@ -117,7 +128,9 @@ case class CarbonDatasourceHadoopRelation(
 
         if (null != reqColumns && reqColumns.nonEmpty) {
           reqColumns.foreach(reqCol => {
-            if (!reqCol.toString.equalsIgnoreCase(col) && !reqCol.toString.startsWith(col
+ ".")) {
+            if (!reqCol.toString.equalsIgnoreCase(col) &&
+                !reqCol.toString.startsWith(col.toLowerCase + ".") &&
+                !parentColumn.contains(col.toLowerCase)) {
               output += col
             } else {
               output += reqCol.toString


Mime
View raw message