carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [33/50] carbondata git commit: [CARBONDATA-1013] Changes to adhere Join without dictionary column
Date Thu, 11 May 2017 13:53:23 GMT
[CARBONDATA-1013] Changes to adhere Join without dictionary column

[CARBONDATA-1013] Changes to adhere join condition on dictionary columns

Changes to adhere Join without Dictionary Columns


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8024a49b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8024a49b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8024a49b

Branch: refs/heads/branch-1.1
Commit: 8024a49b475041f14c82be91a16889e2e39e7a63
Parents: b16ab63
Author: SRIGOPALMOHANTY <SRIGOPALMOHANTY@GMAIL.COM>
Authored: Mon May 8 13:13:16 2017 +0530
Committer: chenliang613 <chenliang613@huawei.com>
Committed: Tue May 9 15:20:25 2017 +0800

----------------------------------------------------------------------
 .../joinquery/JoinWithoutDictionaryColumn.scala | 100 +++++++++++++++++++
 .../sql/optimizer/CarbonLateDecodeRule.scala    |  23 +++--
 2 files changed, 112 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/8024a49b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/joinquery/JoinWithoutDictionaryColumn.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/joinquery/JoinWithoutDictionaryColumn.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/joinquery/JoinWithoutDictionaryColumn.scala
new file mode 100644
index 0000000..d726858
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/joinquery/JoinWithoutDictionaryColumn.scala
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.joinquery
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test cases for testing columns having \N or \null values for non numeric columns
+ */
+class JoinWithoutDictionaryColumn extends QueryTest with BeforeAndAfterAll {
+
+  override def beforeAll {
+    sql("drop table if exists mobile")
+    sql("drop table if exists emp")
+
+    sql("drop table if exists mobile_d")
+    sql("drop table if exists emp_d")
+
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+        CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT
+      )
+
+    sql(
+      """
+        create table mobile (mid String,Mobileid String, Color String, id int) STORED BY
'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='Color')
+      """)
+    sql(
+      """
+        create table emp (eid String,ename String, Mobileid String,Color String, id int)
STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='Color')
+      """)
+
+    sql(
+      """
+        create table mobile_d (mid String,Mobileid String, Color String, id int) STORED BY
'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='Color','DICTIONARY_INCLUDE'='Mobileid')
+      """)
+    sql(
+      """
+        create table emp_d (eid String,ename String, Mobileid String,Color String, id int)
STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='Color','DICTIONARY_INCLUDE'='Mobileid')
+      """)
+
+    sql(
+      s"""
+           LOAD DATA LOCAL INPATH '$resourcesPath/join/mobile.csv' into table
+           mobile
+           OPTIONS('FILEHEADER'='mid,Mobileid,Color,id')
+           """)
+    sql(
+      s"""
+           LOAD DATA LOCAL INPATH '$resourcesPath/join/employee.csv' into table
+           emp
+           OPTIONS('FILEHEADER'='eid,ename,Mobileid,Color,id')
+           """)
+
+    sql(
+      s"""
+           LOAD DATA LOCAL INPATH '$resourcesPath/join/mobile.csv' into table
+           mobile_d
+           OPTIONS('FILEHEADER'='mid,Mobileid,Color,id')
+           """)
+    sql(
+      s"""
+           LOAD DATA LOCAL INPATH '$resourcesPath/join/employee.csv' into table
+           emp_d
+           OPTIONS('FILEHEADER'='eid,ename,Mobileid,Color,id')
+           """)
+  }
+
+  test("select * from emp join mobile on emp.Mobileid=mobile.Mobileid") {
+    checkAnswer(
+      sql("select * from emp join mobile on emp.Mobileid=mobile.Mobileid"),
+      sql("select * from emp_d join mobile_d on emp_d.Mobileid=mobile_d.Mobileid")
+    )
+  }
+
+  override def afterAll {
+    sql("drop table if exists emp")
+    sql("drop table if exists mobile")
+    sql("drop table if exists emp_d")
+    sql("drop table if exists mobile_d")
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/8024a49b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
index 45cc330..aff34ea 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
@@ -321,7 +321,7 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with PredicateHelper
{
             Filter(filter.condition, child)
           }
 
-        case j: Join
+         case j: Join
           if !(j.left.isInstanceOf[CarbonDictionaryTempDecoder] ||
                j.right.isInstanceOf[CarbonDictionaryTempDecoder]) =>
           val attrsOnJoin = new util.HashSet[Attribute]
@@ -337,7 +337,7 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with PredicateHelper
{
 
           val leftCondAttrs = new util.HashSet[AttributeReferenceWrapper]
           val rightCondAttrs = new util.HashSet[AttributeReferenceWrapper]
-          if (attrsOnJoin.size() > 0) {
+          val join = if (attrsOnJoin.size() > 0) {
 
             attrsOnJoin.asScala.map { attr =>
               if (qualifierPresence(j.left, attr)) {
@@ -361,18 +361,19 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with PredicateHelper
{
                 new util.HashSet[AttributeReferenceWrapper](),
                 j.right)
             }
-            if (!decoder) {
-              decoder = true
-              CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
-                new util.HashSet[AttributeReferenceWrapper](),
-                Join(leftPlan, rightPlan, j.joinType, j.condition),
-                isOuter = true)
-            } else {
-              Join(leftPlan, rightPlan, j.joinType, j.condition)
-            }
+            Join(leftPlan, rightPlan, j.joinType, j.condition)
           } else {
             j
           }
+          if (!decoder) {
+            decoder = true
+            CarbonDictionaryTempDecoder(new util.HashSet[AttributeReferenceWrapper](),
+              new util.HashSet[AttributeReferenceWrapper](),
+              join,
+              isOuter = true)
+          } else {
+            join
+          }
 
         case p: Project
           if relations.nonEmpty && !p.child.isInstanceOf[CarbonDictionaryTempDecoder]
=>


Mime
View raw message