carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-1491] Dictionary_exclude columns are not going into no_dictionary flow
Date Thu, 21 Sep 2017 05:55:21 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master d4eabbe56 -> 1e7da59b4


[CARBONDATA-1491] Dictionary_exclude columns are not going into no_dictionary flow

(1) DICTIONARY_EXCLUDE columns are not considered as no_dictionary columns - while parsing
we are not setting nodictionary columns.
(2) For reconstructing defaultValue for newly added no dictionary measure column, logic is
changed, as the previous logic can throw IllegalArgumentException for wrong length.
(3) Test cases are added for the same.

This closes #1374


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1e7da59b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1e7da59b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1e7da59b

Branch: refs/heads/master
Commit: 1e7da59b466ae4f33e3c184db02f218f91a6a2ae
Parents: d4eabbe
Author: dhatchayani <dhatcha.official@gmail.com>
Authored: Wed Sep 20 12:03:18 2017 +0530
Committer: Ravindra Pesala <ravi.pesala@gmail.com>
Committed: Thu Sep 21 11:24:39 2017 +0530

----------------------------------------------------------------------
 .../scan/executor/util/RestructureUtil.java     |  7 ++++--
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |  5 +++-
 .../vectorreader/AddColumnTestCases.scala       | 24 ++++++++++++++++++++
 3 files changed, 33 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/1e7da59b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
index 5e78741..6a281dd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -215,13 +215,16 @@ public class RestructureUtil {
    */
   private static Object getNoDictionaryDefaultValue(DataType datatype, byte[] defaultValue)
{
     Object noDictionaryDefaultValue = null;
+    String value = null;
     if (!isDefaultValueNull(defaultValue)) {
       switch (datatype) {
         case INT:
-          noDictionaryDefaultValue = ByteUtil.toInt(defaultValue, 0, defaultValue.length);
+          value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+          noDictionaryDefaultValue = Integer.parseInt(value);
           break;
         case LONG:
-          noDictionaryDefaultValue = ByteUtil.toLong(defaultValue, 0, defaultValue.length);
+          value = new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+          noDictionaryDefaultValue = Long.parseLong(value);
           break;
         case TIMESTAMP:
           long timestampValue = ByteUtil.toLong(defaultValue, 0, defaultValue.length);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1e7da59b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index acdec91..42070c4 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -617,7 +617,10 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser
{
     // by default consider all String cols as dims and if any dictionary include isn't present
then
     // add it to noDictionaryDims list. consider all dictionary excludes/include cols as
dims
     fields.foreach { field =>
-      if (dictIncludeCols.exists(x => x.equalsIgnoreCase(field.column))) {
+      if (dictExcludeCols.exists(x => x.equalsIgnoreCase(field.column))) {
+        noDictionaryDims :+= field.column
+        dimFields += field
+      } else if (dictIncludeCols.exists(x => x.equalsIgnoreCase(field.column))) {
         dimFields += field
       } else if (DataTypeUtil.getDataType(field.dataType.get.toUpperCase) == DataType.TIMESTAMP
&&
                  !dictIncludeCols.exists(x => x.equalsIgnoreCase(field.column))) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1e7da59b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
index 30485d1..f5ff2e3 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala
@@ -418,6 +418,30 @@ class AddColumnTestCases extends Spark2QueryTest with BeforeAndAfterAll
{
     sql("DROP TABLE IF EXISTS carbon_table")
   }
 
+  test("test to check if intField returns correct result - dictionary exclude") {
+    sqlContext.setConf("carbon.enable.vector.reader", "true")
+    sql("DROP TABLE IF EXISTS carbon_table")
+    sql("CREATE TABLE carbon_table(intField INT,stringField STRING,charField STRING,timestampField
TIMESTAMP, decimalField DECIMAL(6,2)) STORED BY 'carbondata'")
+    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO TABLE carbon_table
OPTIONS('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+    sql(
+      "ALTER TABLE carbon_table ADD COLUMNS(newField INT) TBLPROPERTIES" +
+      "('DEFAULT.VALUE.newField'='67890', 'DICTIONARY_EXCLUDE'='newField')")
+    checkAnswer(sql("SELECT DISTINCT(newField) FROM carbon_table"), Row(67890))
+    sql("DROP TABLE IF EXISTS carbon_table")
+  }
+
+  test("test to check if bigintField returns correct result - dictionary exclude") {
+    sqlContext.setConf("carbon.enable.vector.reader", "true")
+    sql("DROP TABLE IF EXISTS carbon_table")
+    sql("CREATE TABLE carbon_table(intField INT,stringField STRING,charField STRING,timestampField
TIMESTAMP, decimalField DECIMAL(6,2)) STORED BY 'carbondata'")
+    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/restructure/data1.csv' INTO TABLE carbon_table
OPTIONS('FILEHEADER'='intField,stringField,charField,timestampField,decimalField')")
+    sql(
+      "ALTER TABLE carbon_table ADD COLUMNS(newField bigint) TBLPROPERTIES" +
+      "('DEFAULT.VALUE.newField'='67890', 'DICTIONARY_EXCLUDE'='newField')")
+    checkAnswer(sql("SELECT DISTINCT(newField) FROM carbon_table"), Row(67890))
+    sql("DROP TABLE IF EXISTS carbon_table")
+  }
+
   test("test to check if shortField returns correct result") {
     sql("DROP TABLE IF EXISTS carbon_table")
     sql("CREATE TABLE carbon_table(intField INT,stringField STRING,charField STRING,timestampField
TIMESTAMP, decimalField DECIMAL(6,2)) STORED BY 'carbondata'")


Mime
View raw message