carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] carbondata git commit: Convert decimal to byte at the end of sort step when using GLOBAL_SORT.
Date Wed, 14 Jun 2017 12:02:06 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 4fb38bfb3 -> f5b08de13


Convert decimal to byte at the end of sort step when using GLOBAL_SORT.

Convert decimal to byte at the end of sort step when using GLOBAL_SORT.


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b24f1228
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b24f1228
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b24f1228

Branch: refs/heads/master
Commit: b24f1228f8694e55792e2e90072e80c08e1cef7b
Parents: 4fb38bf
Author: Yadong Qi <qiyadong2010@gmail.com>
Authored: Wed Jun 14 14:22:23 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Wed Jun 14 20:01:16 2017 +0800

----------------------------------------------------------------------
 .../dataload/TestGlobalSortDataLoad.scala       | 56 ++++++++++++++++++++
 .../newflow/sort/SortStepRowUtil.java           | 19 ++++++-
 2 files changed, 74 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b24f1228/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
index c00aaa3..190b7df 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestGlobalSortDataLoad.scala
@@ -24,6 +24,7 @@ import org.apache.carbondata.core.util.CarbonProperties
 import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.common.util.QueryTest
+import org.apache.spark.sql.test.TestQueryExecutor.projectPath
 import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 
 class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll
{
@@ -62,8 +63,10 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach
with Befo
     sql("DROP TABLE IF EXISTS carbon_localsort_triple")
     sql("DROP TABLE IF EXISTS carbon_localsort_delete")
     sql("DROP TABLE IF EXISTS carbon_localsort_update")
+    sql("DROP TABLE IF EXISTS carbon_localsort_difftypes")
     sql("DROP TABLE IF EXISTS carbon_globalsort")
     sql("DROP TABLE IF EXISTS carbon_globalsort_partitioned")
+    sql("DROP TABLE IF EXISTS carbon_globalsort_difftypes")
   }
 
   // ----------------------------------- Compare Result -----------------------------------
@@ -274,6 +277,59 @@ class TestGlobalSortDataLoad extends QueryTest with BeforeAndAfterEach
with Befo
       sql("SELECT * FROM carbon_localsort_once ORDER BY name"))
   }
 
+  test("Test with different date types") {
+    val path = s"$projectPath/examples/spark2/src/main/resources/data.csv"
+
+    sql(
+      s"""
+         | CREATE TABLE carbon_localsort_difftypes(
+         | shortField SHORT,
+         | intField INT,
+         | bigintField LONG,
+         | doubleField DOUBLE,
+         | stringField STRING,
+         | timestampField TIMESTAMP,
+         | decimalField DECIMAL(18,2),
+         | dateField DATE,
+         | charField CHAR(5),
+         | floatField FLOAT
+         | )
+         | STORED BY 'org.apache.carbondata.format'
+       """.stripMargin)
+    sql(
+      s"""
+         | LOAD DATA LOCAL INPATH '$path' INTO TABLE carbon_localsort_difftypes
+         | OPTIONS('SORT_SCOPE'='GLOBAL_SORT',
+         | 'FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField')
+       """.stripMargin)
+
+    sql(
+      s"""
+         | CREATE TABLE carbon_globalsort_difftypes(
+         | shortField SHORT,
+         | intField INT,
+         | bigintField LONG,
+         | doubleField DOUBLE,
+         | stringField STRING,
+         | timestampField TIMESTAMP,
+         | decimalField DECIMAL(18,2),
+         | dateField DATE,
+         | charField CHAR(5),
+         | floatField FLOAT
+         | )
+         | STORED BY 'org.apache.carbondata.format'
+       """.stripMargin)
+    sql(
+      s"""
+         | LOAD DATA LOCAL INPATH '$path' INTO TABLE carbon_globalsort_difftypes
+         | OPTIONS('SORT_SCOPE'='GLOBAL_SORT',
+         | 'FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField')
+       """.stripMargin)
+
+    checkAnswer(sql("SELECT * FROM carbon_globalsort_difftypes ORDER BY shortField"),
+      sql("SELECT * FROM carbon_localsort_difftypes ORDER BY shortField"))
+  }
+
   private def resetConf() {
     CarbonProperties.getInstance()
       .addProperty(CarbonCommonConstants.LOAD_SORT_SCOPE, CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT)

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b24f1228/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
index 474df4d..53f1497 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
@@ -17,6 +17,10 @@
 
 package org.apache.carbondata.processing.newflow.sort;
 
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.util.DataTypeUtil;
 import org.apache.carbondata.processing.sortandgroupby.sortdata.SortParameters;
 import org.apache.carbondata.processing.util.NonDictionaryUtil;
 
@@ -54,9 +58,22 @@ public class SortStepRowUtil {
       }
 
       index = 0;
+
+      DataType[] measureDataType = parameters.getMeasureDataType();
       // read measure values
       for (int i = 0; i < measureCount; i++) {
-        measures[index++] = data[allCount];
+        Object value = data[allCount];
+        if (null != value) {
+          if (measureDataType[i] == DataType.DECIMAL) {
+            BigDecimal decimal = (BigDecimal) value;
+            measures[index++] = DataTypeUtil.bigDecimalToByte(decimal);
+          } else {
+            measures[index++] = value;
+          }
+        } else {
+          measures[index++] = null;
+        }
+
         allCount++;
       }
 


Mime
View raw message