carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [09/12] carbondata git commit: fixed codec for UpscaleFloatingCodec
Date Tue, 27 Jun 2017 14:08:15 GMT
fixed codec for UpscaleFloatingCodec


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1d8254b8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1d8254b8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1d8254b8

Branch: refs/heads/encoding_override
Commit: 1d8254b85c7a0613e2fc2698d2a642fc52b6335e
Parents: 3dec25b
Author: kunal642 <kunal.kapoor@knoldus.in>
Authored: Sat Jun 17 13:37:08 2017 +0530
Committer: jackylk <jacky.likun@huawei.com>
Committed: Mon Jun 26 21:08:53 2017 +0800

----------------------------------------------------------------------
 .../datastore/page/encoding/UpscaleFloatingCodec.java    |  2 +-
 .../datastore/page/statistics/ColumnPageStatsVO.java     |  2 +-
 .../spark-common-test/src/test/resources/double.csv      |  3 +++
 .../TestLoadDataWithHiveSyntaxDefaultFormat.scala        | 11 +++++++++++
 4 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d8254b8/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/UpscaleFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/UpscaleFloatingCodec.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/UpscaleFloatingCodec.java
index 19c6bd7..56c4508 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/UpscaleFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/UpscaleFloatingCodec.java
@@ -137,7 +137,7 @@ public class UpscaleFloatingCodec extends AdaptiveCompressionCodec {
           encodedPage.putLong(rowId, (long)(Math.round(factor * value)));
           break;
         case DOUBLE:
-          encodedPage.putDouble(rowId, (Math.round(factor * value)));
+          encodedPage.putDouble(rowId, value);
           break;
         default:
           throw new RuntimeException("internal error: " + debugInfo());

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d8254b8/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
index 642e6b2..058699a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
@@ -96,7 +96,7 @@ public class ColumnPageStatsVO {
       case DOUBLE:
         max = ((double) max > (double) value) ? max : value;
         min = ((double) min < (double) value) ? min : value;
-        int num = getDecimalCount((double) value);
+        int num = Math.abs(getDecimalCount((double) value));
         decimal = decimal > num ? decimal : num;
         nonExistValue = (double) min - 1;
         break;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d8254b8/integration/spark-common-test/src/test/resources/double.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/double.csv b/integration/spark-common-test/src/test/resources/double.csv
new file mode 100644
index 0000000..267392e
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/double.csv
@@ -0,0 +1,3 @@
+empno, salary
+'abc', 775678765456789098765432.789
+'def', 876567898743456785232.44431

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1d8254b8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala
index f8e11de..b8713e5 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithHiveSyntaxDefaultFormat.scala
@@ -670,6 +670,17 @@ class TestLoadDataWithHiveSyntaxDefaultFormat extends QueryTest with
BeforeAndAf
       Row("~carbon,")))
   }
 
+  test("test data load with double datatype") {
+    sql("drop table if exists double_test")
+    sql(
+      "CREATE table double_test (empno string, salary double) STORED BY 'carbondata' TBLPROPERTIES"
+
+      "('DICTIONARY_EXCLUDE'='empno')"
+    )
+    sql(
+      s"load data local inpath '$resourcesPath/double.csv' into table double_test options"
+
+      "('FILEHEADER'='empno,salary')")
+    checkAnswer(sql("select salary from double_test limit 1"),Row(7.756787654567891E23))
+  }
 
   override def afterAll {
     sql("drop table if exists escapechar1")


Mime
View raw message