carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [1/2] carbondata git commit: resolved bug for dictionary_exclude not working using carbondatasource
Date Tue, 09 May 2017 07:08:35 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master b3b24f2a0 -> 77dab1d7f


resolved bug for dictionary_exclude not working using carbondatasource


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/65b0803f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/65b0803f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/65b0803f

Branch: refs/heads/master
Commit: 65b0803f34e82f143d449c67fb68c79bf411ab0a
Parents: b3b24f2
Author: anubhav100 <anubhav.tarar@knoldus.in>
Authored: Sat Apr 8 23:28:06 2017 +0530
Committer: chenliang613 <chenliang613@huawei.com>
Committed: Tue May 9 15:06:46 2017 +0800

----------------------------------------------------------------------
 .../org/apache/spark/sql/TableCreator.scala     | 38 ++++++++++----------
 .../carbondata/CarbonDataSourceSuite.scala      | 16 +++++++++
 2 files changed, 35 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/65b0803f/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index f8bdcf8..c4f9fc5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -33,22 +33,22 @@ object TableCreator {
 
   // detects whether complex dimension is part of dictionary_exclude
   def isComplexDimDictionaryExclude(dimensionDataType: String): Boolean = {
-    val dimensionType = Array("array", "struct")
-    dimensionType.exists(x => x.equalsIgnoreCase(dimensionDataType))
+    val dimensionTypes = Array("array", "arraytype", "struct", "structtype")
+    dimensionTypes.exists(dimensionType => dimensionType.equalsIgnoreCase(dimensionDataType))
   }
 
   // detects whether datatype is part of dictionary_exclude
   def isDataTypeSupportedForDictionary_Exclude(columnDataType: String): Boolean = {
-    val dataTypes = Array("string")
-    dataTypes.exists(x => x.equalsIgnoreCase(columnDataType))
+    val dataTypes = Array("string", "stringtype")
+    dataTypes.exists(dataType => dataType.equalsIgnoreCase(columnDataType))
   }
 
   // detect dimention data type
   def isDetectAsDimentionDatatype(dimensionDatatype: String): Boolean = {
-    val dimensionType =
+    val dimensionTypes =
       Array("string", "stringtype", "array", "arraytype", "struct",
         "structtype", "timestamp", "timestamptype", "date", "datetype")
-    dimensionType.exists(x => x.equalsIgnoreCase(dimensionDatatype))
+    dimensionTypes.exists(dimensionType => dimensionType.equalsIgnoreCase(dimensionDatatype))
   }
 
   protected def extractDimColsAndNoDictionaryFields(fields: Seq[Field],
@@ -65,13 +65,13 @@ object TableCreator {
         tableProperties(CarbonCommonConstants.DICTIONARY_EXCLUDE).split(',').map(_.trim)
       dictExcludeCols
         .foreach { dictExcludeCol =>
-          if (!fields.exists(x => x.column.equalsIgnoreCase(dictExcludeCol))) {
+          if (!fields.exists(field => field.column.equalsIgnoreCase(dictExcludeCol)))
{
             val errormsg = "DICTIONARY_EXCLUDE column: " + dictExcludeCol +
               " does not exist in table. Please check create table statement."
             throw new MalformedCarbonCommandException(errormsg)
           } else {
-            val dataType = fields.find(x =>
-              x.column.equalsIgnoreCase(dictExcludeCol)).get.dataType.get
+            val dataType = fields.find(field =>
+              field.column.equalsIgnoreCase(dictExcludeCol)).get.dataType.get
             if (isComplexDimDictionaryExclude(dataType)) {
               val errormsg = "DICTIONARY_EXCLUDE is unsupported for complex datatype column:
" +
                 dictExcludeCol
@@ -89,7 +89,7 @@ object TableCreator {
       dictIncludeCols =
         tableProperties(CarbonCommonConstants.DICTIONARY_INCLUDE).split(",").map(_.trim)
       dictIncludeCols.foreach { distIncludeCol =>
-        if (!fields.exists(x => x.column.equalsIgnoreCase(distIncludeCol.trim))) {
+        if (!fields.exists(field => field.column.equalsIgnoreCase(distIncludeCol.trim)))
{
           val errormsg = "DICTIONARY_INCLUDE column: " + distIncludeCol.trim +
             " does not exist in table. Please check create table statement."
           throw new MalformedCarbonCommandException(errormsg)
@@ -99,7 +99,7 @@ object TableCreator {
 
     // include cols should contain exclude cols
     dictExcludeCols.foreach { dicExcludeCol =>
-      if (dictIncludeCols.exists(x => x.equalsIgnoreCase(dicExcludeCol))) {
+      if (dictIncludeCols.exists(col => col.equalsIgnoreCase(dicExcludeCol))) {
         val errormsg = "DICTIONARY_EXCLUDE can not contain the same column: " + dicExcludeCol
+
           " with DICTIONARY_INCLUDE. Please check create table statement."
         throw new MalformedCarbonCommandException(errormsg)
@@ -110,13 +110,13 @@ object TableCreator {
     // add it to noDictionaryDims list. consider all dictionary excludes/include cols as
dims
     fields.foreach(field => {
 
-      if (dictExcludeCols.toSeq.exists(x => x.equalsIgnoreCase(field.column))) {
+      if (dictExcludeCols.toSeq.exists(col => col.equalsIgnoreCase(field.column))) {
         if (DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.TIMESTAMP
&&
             DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.DATE)
{
           noDictionaryDims :+= field.column
         }
         dimFields += field
-      } else if (dictIncludeCols.exists(x => x.equalsIgnoreCase(field.column))) {
+      } else if (dictIncludeCols.exists(col => col.equalsIgnoreCase(field.column))) {
         dimFields += field
       } else if (isDetectAsDimentionDatatype(field.dataType.get)) {
         dimFields += field
@@ -155,8 +155,8 @@ object TableCreator {
     // by default consider all non string cols as msrs. consider all include/ exclude cols
as dims
     fields.foreach(field => {
       if (!isDetectAsDimentionDatatype(field.dataType.get)) {
-        if (!dictIncludedCols.exists(x => x.equalsIgnoreCase(field.column)) &&
-          !dictExcludedCols.exists(x => x.equalsIgnoreCase(field.column))) {
+        if (!dictIncludedCols.exists(col => col.equalsIgnoreCase(field.column)) &&
+          !dictExcludedCols.exists(col => col.equalsIgnoreCase(field.column))) {
           msrFields :+= field
         }
       }
@@ -218,9 +218,9 @@ object TableCreator {
   def rearrangedColumnGroup(colGroup: String, dims: Seq[Field]): String = {
     // if columns in column group is not in schema order than arrange it in schema order
     var colGrpFieldIndx: Seq[Int] = Seq[Int]()
-    colGroup.split(',').map(_.trim).foreach { x =>
+    colGroup.split(',').map(_.trim).foreach { col =>
       dims.zipWithIndex.foreach { dim =>
-        if (dim._1.column.equalsIgnoreCase(x)) {
+        if (dim._1.column.equalsIgnoreCase(col)) {
           colGrpFieldIndx :+= dim._2
         }
       }
@@ -315,7 +315,7 @@ object TableCreator {
       noInvertedIdxColsProps =
         tableProperties("NO_INVERTED_INDEX").split(',').map(_.trim)
       noInvertedIdxColsProps.foreach { noInvertedIdxColProp =>
-          if (!fields.exists(x => x.column.equalsIgnoreCase(noInvertedIdxColProp))) {
+          if (!fields.exists(field => field.column.equalsIgnoreCase(noInvertedIdxColProp)))
{
             val errormsg = "NO_INVERTED_INDEX column: " + noInvertedIdxColProp +
               " does not exist in table. Please check create table statement."
             throw new MalformedCarbonCommandException(errormsg)
@@ -326,7 +326,7 @@ object TableCreator {
     val distinctCols = noInvertedIdxColsProps.toSet
     // extract the no inverted index columns
     fields.foreach(field => {
-      if (distinctCols.exists(x => x.equalsIgnoreCase(field.column))) {
+      if (distinctCols.exists(col => col.equalsIgnoreCase(field.column))) {
         noInvertedIdxCols :+= field.column
       }
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/65b0803f/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
index bd70d85..3d814a1 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
@@ -29,6 +29,7 @@ class CarbonDataSourceSuite extends QueryTest with BeforeAndAfterAll {
     // Drop table
     sql("DROP TABLE IF EXISTS carbon_testtable")
     sql("DROP TABLE IF EXISTS csv_table")
+    sql("DROP TABLE IF EXISTS car")
 
     // Create table
     sql(
@@ -176,4 +177,19 @@ class CarbonDataSourceSuite extends QueryTest with BeforeAndAfterAll
{
     sql("drop database testdb")
   }
 
+  test("test carbon source table with string type in dictionary_exclude") {
+    try {
+      sql("create table car( \nL_SHIPDATE string,\nL_SHIPMODE string,\nL_SHIPINSTRUCT string,"
+
+          "\nL_RETURNFLAG string,\nL_RECEIPTDATE string,\nL_ORDERKEY string,\nL_PARTKEY string,"
+
+          "\nL_SUPPKEY string,\nL_LINENUMBER int,\nL_QUANTITY decimal,\nL_EXTENDEDPRICE decimal,"
+
+          "\nL_DISCOUNT decimal,\nL_TAX decimal,\nL_LINESTATUS string,\nL_COMMITDATE string,"
+
+          "\nL_COMMENT string \n) \nUSING org.apache.spark.sql.CarbonSource\nOPTIONS (tableName
" +
+          "\"car\", DICTIONARY_EXCLUDE \"L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_COMMENT\")")
+      assert(true)
+    }
+    catch {
+      case exception: Exception => assert(false)
+    }
+  }
+
 }


Mime
View raw message