carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From manishgupt...@apache.org
Subject carbondata git commit: [CARBONDATA-3098] Fix for negative exponents value giving wrong results in Float datatype
Date Wed, 14 Nov 2018 08:29:19 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master cd0ce4187 -> ceb135175


[CARBONDATA-3098] Fix for negative exponents value giving wrong results in Float datatype

Problem: When the value of exponent is a negative number then the data is incorrect due to
loss of precision of Floating point values and wrong
calculation of the count of decimal points.

Solution: Handled floating point precision by converting it to double and counted the decimal
count values as done in double datatype(using Big Decimal).

This closes #2918


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ceb13517
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ceb13517
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ceb13517

Branch: refs/heads/master
Commit: ceb13517553e729ccb408d95ae7ae401f7aebcb8
Parents: cd0ce41
Author: Manish Nalla <manishnalla1994@gmail.com>
Authored: Wed Nov 14 10:57:49 2018 +0530
Committer: manishgupta88 <tomanishgupta18@gmail.com>
Committed: Wed Nov 14 14:03:55 2018 +0530

----------------------------------------------------------------------
 .../encoding/adaptive/AdaptiveFloatingCodec.java     | 14 +-------------
 .../page/statistics/PrimitivePageStatsCollector.java | 14 +-------------
 .../datasource/SparkCarbonDataSourceTest.scala       | 15 +++++++++++++++
 3 files changed, 17 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/ceb13517/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
index 49696eb..b04c9df 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
@@ -167,19 +167,7 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
 
     @Override
     public void encode(int rowId, float value) {
-      if (targetDataType == DataTypes.BYTE) {
-        encodedPage.putByte(rowId, (byte) (value * floatFactor));
-      } else if (targetDataType == DataTypes.SHORT) {
-        encodedPage.putShort(rowId, (short) (value * floatFactor));
-      } else if (targetDataType == DataTypes.SHORT_INT) {
-        encodedPage.putShortInt(rowId, (int) (value * floatFactor));
-      } else if (targetDataType == DataTypes.INT) {
-        encodedPage.putInt(rowId, (int) (value * floatFactor));
-      } else if (targetDataType == DataTypes.LONG) {
-        encodedPage.putLong(rowId, (long) (value * floatFactor));
-      } else {
-        throw new RuntimeException("internal error: " + debugInfo());
-      }
+      encode(rowId, (double) value);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ceb13517/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
index 9be5a58..e604057 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
@@ -253,19 +253,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector,
Si
   }
 
   private int getDecimalCount(float value) {
-    int decimalPlaces = 0;
-    try {
-      String strValue = Float.valueOf(Math.abs(value)).toString();
-      int integerPlaces = strValue.indexOf('.');
-      if (-1 != integerPlaces) {
-        decimalPlaces = strValue.length() - integerPlaces - 1;
-      }
-    } catch (NumberFormatException e) {
-      if (!Double.isInfinite(value)) {
-        throw e;
-      }
-    }
-    return decimalPlaces;
+    return getDecimalCount((double) value);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ceb13517/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
index 1e58a9e..5eb103a 100644
--- a/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
+++ b/integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala
@@ -1310,6 +1310,21 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll
{
     checkAnswer(spark.sql("select * from t_carbn01b_hive"), spark.sql("select * from t_carbn01b"))
     spark.sql("drop table if exists t_carbn01b_hive")
     spark.sql(s"drop table if exists t_carbn01b")
+    }
+
+  test("Test Float value by having negative exponents") {
+    spark.sql("DROP TABLE IF EXISTS float_p")
+    spark.sql("DROP TABLE IF EXISTS float_c")
+    spark.sql("CREATE TABLE float_p(f float) using parquet")
+    spark.sql("CREATE TABLE float_c(f float) using carbon")
+    spark.sql("INSERT INTO float_p select \"1.4E-3\"")
+    spark.sql("INSERT INTO float_p select \"1.4E-38\"")
+    spark.sql("INSERT INTO float_c select \"1.4E-3\"")
+    spark.sql("INSERT INTO float_c select \"1.4E-38\"")
+    checkAnswer(spark.sql("SELECT * FROM float_p"),
+      spark.sql("SELECT * FROM float_c"))
+    spark.sql("DROP TABLE float_p")
+    spark.sql("DROP TABLE float_c")
   }
 
   test("test fileformat flow with drop and query on same table") {


Mime
View raw message