carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [37/50] [abbrv] carbondata git commit: [CARBONDATA-1984][Compression Codec] Double Compression Codec Rectification.
Date Tue, 09 Jan 2018 04:02:05 GMT
[CARBONDATA-1984][Compression Codec] Double Compression Codec Rectification.

This closes #1763


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/08b8af7b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/08b8af7b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/08b8af7b

Branch: refs/heads/branch-1.3
Commit: 08b8af7b044fa761caf58da2875935d40d4ca13b
Parents: c9e5842
Author: sounakr <sounakr@gmail.com>
Authored: Thu Jan 4 15:15:42 2018 +0530
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Fri Jan 5 14:36:13 2018 +0800

----------------------------------------------------------------------
 .../adaptive/AdaptiveDeltaFloatingCodec.java    |   8 +-
 .../adaptive/AdaptiveFloatingCodec.java         |  10 +-
 .../data_notitle_AdaptiveFloating_byte.csv      |   4 +
 .../data_notitle_AdaptiveFloating_int.csv       |   4 +
 .../data_notitle_AdaptiveFloating_short.csv     |   4 +
 .../data_notitle_AdaptiveFloating_short_int.csv |   4 +
 .../test/resources/double/data_notitle_byte.csv |   4 +
 .../test/resources/double/data_notitle_int.csv  |   4 +
 .../test/resources/double/data_notitle_long.csv |   4 +
 .../resources/double/data_notitle_short.csv     |   4 +
 .../resources/double/data_notitle_short_int.csv |   4 +
 .../allqueries/DoubleDataTypeTest.scala         | 132 +++++++++++++++++++
 12 files changed, 177 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
index 1f00feb..421a17a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
@@ -154,13 +154,13 @@ public class AdaptiveDeltaFloatingCodec extends AdaptiveCodec {
     @Override
     public void encode(int rowId, double value) {
       if (targetDataType.equals(DataTypes.BYTE)) {
-        encodedPage.putByte(rowId, (byte) (max - (value * factor)));
+        encodedPage.putByte(rowId, (byte) (max - Math.round(value * factor)));
       } else if (targetDataType.equals(DataTypes.SHORT)) {
-        encodedPage.putShort(rowId, (short) (max - (value * factor)));
+        encodedPage.putShort(rowId, (short) (max - Math.round(value * factor)));
       } else if (targetDataType.equals(DataTypes.SHORT_INT)) {
-        encodedPage.putShortInt(rowId, (int) (max - (value * factor)));
+        encodedPage.putShortInt(rowId, (int) (max - Math.round(value * factor)));
       } else if (targetDataType.equals(DataTypes.INT)) {
-        encodedPage.putInt(rowId, (int) (max - (value * factor)));
+        encodedPage.putInt(rowId, (int) (max - Math.round(value * factor)));
       } else if (targetDataType.equals(DataTypes.DOUBLE)) {
         encodedPage.putDouble(rowId, value);
       } else {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
index fb3e248..7adeff0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
@@ -154,15 +154,15 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
     @Override
     public void encode(int rowId, double value) {
       if (targetDataType == DataTypes.BYTE) {
-        encodedPage.putByte(rowId, (byte) (value * factor));
+        encodedPage.putByte(rowId, (byte) Math.round(value * factor));
       } else if (targetDataType == DataTypes.SHORT) {
-        encodedPage.putShort(rowId, (short) (value * factor));
+        encodedPage.putShort(rowId, (short) Math.round(value * factor));
       } else if (targetDataType == DataTypes.SHORT_INT) {
-        encodedPage.putShortInt(rowId, (int) (value * factor));
+        encodedPage.putShortInt(rowId, (int) Math.round(value * factor));
       } else if (targetDataType == DataTypes.INT) {
-        encodedPage.putInt(rowId, (int) (value * factor));
+        encodedPage.putInt(rowId, (int) Math.round(value * factor));
       } else if (targetDataType == DataTypes.LONG) {
-        encodedPage.putLong(rowId, (long) (value * factor));
+        encodedPage.putLong(rowId, (long) Math.round(value * factor));
       } else if (targetDataType == DataTypes.DOUBLE) {
         encodedPage.putDouble(rowId, value);
       } else {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_byte.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_byte.csv
b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_byte.csv
new file mode 100644
index 0000000..f4fc5b8
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_byte.csv
@@ -0,0 +1,4 @@
+12345|"test1"|922337203687|11.11|0.0012|2016-8-1 11:45:15
+12346|"test2"|922337203688|11.12|0.0013|2016-8-2 11:45:15
+12347|"test2"|922337203689|11.13|0.0014|2016-8-3 11:45:15
+12348|"test3"|922337203680|11.14|0.0015|2016-8-4 11:45:15

http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_int.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_int.csv
b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_int.csv
new file mode 100644
index 0000000..985369e
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_int.csv
@@ -0,0 +1,4 @@
+12345|"test1"|922337203687|11.11|100000.2812|2016-8-1 11:45:15
+12346|"test2"|922337203688|11.12|100000.2813|2016-8-2 11:45:15
+12347|"test2"|922337203689|11.13|100000.2814|2016-8-3 11:45:15
+12348|"test3"|922337203680|11.14|200000.1415|2016-8-4 11:45:15

http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_short.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_short.csv
b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_short.csv
new file mode 100644
index 0000000..0c43fd0
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_short.csv
@@ -0,0 +1,4 @@
+12345|"test1"|922337203687|11.11|0.1412|2016-8-1 11:45:15
+12346|"test2"|922337203688|11.12|0.1413|2016-8-2 11:45:15
+12347|"test2"|922337203689|11.13|0.1414|2016-8-3 11:45:15
+12348|"test3"|922337203680|11.14|0.1415|2016-8-4 11:45:15

http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_short_int.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_short_int.csv
b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_short_int.csv
new file mode 100644
index 0000000..9ff2485
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/double/data_notitle_AdaptiveFloating_short_int.csv
@@ -0,0 +1,4 @@
+12345|"test1"|922337203687|11.11|300.1412|2016-8-1 11:45:15
+12346|"test2"|922337203688|11.12|300.1413|2016-8-2 11:45:15
+12347|"test2"|922337203689|11.13|300.1414|2016-8-3 11:45:15
+12348|"test3"|922337203680|11.14|800.1415|2016-8-4 11:45:15

http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/integration/spark-common-test/src/test/resources/double/data_notitle_byte.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/double/data_notitle_byte.csv
b/integration/spark-common-test/src/test/resources/double/data_notitle_byte.csv
new file mode 100644
index 0000000..63da041
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/double/data_notitle_byte.csv
@@ -0,0 +1,4 @@
+12345|"test1"|922337203687|11.11|3.1412|2016-8-1 11:45:15
+12346|"test2"|922337203688|11.12|3.1413|2016-8-2 11:45:15
+12347|"test2"|922337203689|11.13|3.1414|2016-8-3 11:45:15
+12348|"test3"|922337203680|11.14|3.1415|2016-8-4 11:45:15

http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/integration/spark-common-test/src/test/resources/double/data_notitle_int.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/double/data_notitle_int.csv
b/integration/spark-common-test/src/test/resources/double/data_notitle_int.csv
new file mode 100644
index 0000000..b0eaf3d
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/double/data_notitle_int.csv
@@ -0,0 +1,4 @@
+12345|"test1"|922337203687|11.11|199161.2812|2016-8-1 11:45:15
+12346|"test2"|922337203688|11.12|199161.2813|2016-8-2 11:45:15
+12347|"test2"|922337203689|11.13|199161.2814|2016-8-3 11:45:15
+12348|"test3"|922337203680|11.14|200000.1415|2016-8-4 11:45:15

http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/integration/spark-common-test/src/test/resources/double/data_notitle_long.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/double/data_notitle_long.csv
b/integration/spark-common-test/src/test/resources/double/data_notitle_long.csv
new file mode 100644
index 0000000..5509d2c
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/double/data_notitle_long.csv
@@ -0,0 +1,4 @@
+12345|"test1"|922337203687|11.11|100000.1412|2016-8-1 11:45:15
+12346|"test2"|922337203688|11.12|100000.1413|2016-8-2 11:45:15
+12347|"test2"|922337203689|11.13|100000.1414|2016-8-3 11:45:15
+12348|"test3"|922337203680|11.14|200000.1415|2016-8-4 11:45:15

http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/integration/spark-common-test/src/test/resources/double/data_notitle_short.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/double/data_notitle_short.csv
b/integration/spark-common-test/src/test/resources/double/data_notitle_short.csv
new file mode 100644
index 0000000..d14ff04
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/double/data_notitle_short.csv
@@ -0,0 +1,4 @@
+12345|"test1"|922337203687|11.11|3.1412|2016-8-1 11:45:15
+12346|"test2"|922337203688|11.12|3.1413|2016-8-2 11:45:15
+12347|"test2"|922337203689|11.13|3.1414|2016-8-3 11:45:15
+12348|"test3"|922337203680|11.14|5.1415|2016-8-4 11:45:15

http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/integration/spark-common-test/src/test/resources/double/data_notitle_short_int.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/double/data_notitle_short_int.csv
b/integration/spark-common-test/src/test/resources/double/data_notitle_short_int.csv
new file mode 100644
index 0000000..760d58e
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/double/data_notitle_short_int.csv
@@ -0,0 +1,4 @@
+12345|"test1"|922337203687|11.11|300.1412|2016-8-1 11:45:15
+12346|"test2"|922337203688|11.12|300.1413|2016-8-2 11:45:15
+12347|"test2"|922337203689|11.13|300.1414|2016-8-3 11:45:15
+12348|"test3"|922337203680|11.14|900.1415|2016-8-4 11:45:15

http://git-wip-us.apache.org/repos/asf/carbondata/blob/08b8af7b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/DoubleDataTypeTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/DoubleDataTypeTest.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/DoubleDataTypeTest.scala
index d7a96eb..28ff91a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/DoubleDataTypeTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/DoubleDataTypeTest.scala
@@ -17,6 +17,7 @@
 
 package org.apache.carbondata.spark.testsuite.allqueries
 
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.test.util.QueryTest
 import org.scalatest.BeforeAndAfterAll
 
@@ -24,6 +25,16 @@ class DoubleDataTypeTest extends QueryTest with BeforeAndAfterAll {
 
   override def beforeAll: Unit = {
     sql("DROP TABLE IF EXISTS oscon_carbon_old")
+    sql("drop table if exists carbon_datatype_double_byte")
+    sql("drop table if exists carbon_datatype_double_short")
+    sql("drop table if exists carbon_datatype_double_sint")
+    sql("drop table if exists carbon_datatype_double_int")
+    sql("drop table if exists carbon_datatype_double_long")
+    sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_byte")
+    sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_short")
+    sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_sint")
+    sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_int")
+    sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_long")
     sql("""create table oscon_carbon_old (CUST_PRFRD_FLG String,PROD_BRAND_NAME String,PROD_COLOR
String,CUST_LAST_RVW_DATE String,CUST_COUNTRY String,CUST_CITY String,PRODUCT_NAME String,CUST_JOB_TITLE
String,CUST_STATE String,CUST_BUY_POTENTIAL String,PRODUCT_MODEL String,ITM_ID String,ITM_NAME
String,PRMTION_ID String,PRMTION_NAME String,SHP_MODE_ID String,SHP_MODE String,DELIVERY_COUNTRY
String,DELIVERY_STATE String,DELIVERY_CITY String,DELIVERY_DISTRICT String,ACTIVE_EMUI_VERSION
String,WH_NAME String,STR_ORDER_DATE String,OL_ORDER_NO String,OL_ORDER_DATE String,OL_SITE
String,CUST_FIRST_NAME String,CUST_LAST_NAME String,CUST_BIRTH_DY String,CUST_BIRTH_MM String,CUST_BIRTH_YR
String,CUST_BIRTH_COUNTRY String,CUST_SEX String,CUST_ADDRESS_ID String,CUST_STREET_NO String,CUST_STREET_NAME
String,CUST_AGE String,CUST_SUITE_NO String,CUST_ZIP String,CUST_COUNTY String,PRODUCT_ID
String,PROD_SHELL_COLOR String,DEVICE_NAME String,PROD_SHORT_DESC String,PROD_LONG_DESC String,PROD_THUMB

 String,PROD_IMAGE String,PROD_UPDATE_DATE String,PROD_LIVE String,PROD_LOC String,PROD_RAM
String,PROD_ROM String,PROD_CPU_CLOCK String,PROD_SERIES String,ITM_REC_START_DATE String,ITM_REC_END_DATE
String,ITM_BRAND_ID String,ITM_BRAND String,ITM_CLASS_ID String,ITM_CLASS String,ITM_CATEGORY_ID
String,ITM_CATEGORY String,ITM_MANUFACT_ID String,ITM_MANUFACT String,ITM_FORMULATION String,ITM_COLOR
String,ITM_CONTAINER String,ITM_MANAGER_ID String,PRM_START_DATE String,PRM_END_DATE String,PRM_CHANNEL_DMAIL
String,PRM_CHANNEL_EMAIL String,PRM_CHANNEL_CAT String,PRM_CHANNEL_TV String,PRM_CHANNEL_RADIO
String,PRM_CHANNEL_PRESS String,PRM_CHANNEL_EVENT String,PRM_CHANNEL_DEMO String,PRM_CHANNEL_DETAILS
String,PRM_PURPOSE String,PRM_DSCNT_ACTIVE String,SHP_CODE String,SHP_CARRIER String,SHP_CONTRACT
String,CHECK_DATE String,CHECK_YR String,CHECK_MM String,CHECK_DY String,CHECK_HOUR String,BOM
String,INSIDE_NAME String,PACKING_DATE String,PACKING_YR String,PACKING_MM String,PACKING_DY
String,
 PACKING_HOUR String,DELIVERY_PROVINCE String,PACKING_LIST_NO String,ACTIVE_CHECK_TIME String,ACTIVE_CHECK_YR
String,ACTIVE_CHECK_MM String,ACTIVE_CHECK_DY String,ACTIVE_CHECK_HOUR String,ACTIVE_AREA_ID
String,ACTIVE_COUNTRY String,ACTIVE_PROVINCE String,ACTIVE_CITY String,ACTIVE_DISTRICT String,ACTIVE_NETWORK
String,ACTIVE_FIRMWARE_VER String,ACTIVE_OS_VERSION String,LATEST_CHECK_TIME String,LATEST_CHECK_YR
String,LATEST_CHECK_MM String,LATEST_CHECK_DY String,LATEST_CHECK_HOUR String,LATEST_AREAID
String,LATEST_COUNTRY String,LATEST_PROVINCE String,LATEST_CITY String,LATEST_DISTRICT String,LATEST_FIRMWARE_VER
String,LATEST_EMUI_VERSION String,LATEST_OS_VERSION String,LATEST_NETWORK String,WH_ID String,WH_STREET_NO
String,WH_STREET_NAME String,WH_STREET_TYPE String,WH_SUITE_NO String,WH_CITY String,WH_COUNTY
String,WH_STATE String,WH_ZIP String,WH_COUNTRY String,OL_SITE_DESC String,OL_RET_ORDER_NO
String,OL_RET_DATE String,PROD_MODEL_ID String,CUST_ID String,PROD_UNQ_MDL_ID String,CU
 ST_NICK_NAME String,CUST_LOGIN String,CUST_EMAIL_ADDR String,PROD_UNQ_DEVICE_ADDR String,PROD_UQ_UUID
String,PROD_BAR_CODE String,TRACKING_NO String,STR_ORDER_NO String,CUST_DEP_COUNT double,CUST_VEHICLE_COUNT
double,CUST_ADDRESS_CNT double,CUST_CRNT_CDEMO_CNT double,CUST_CRNT_HDEMO_CNT double,CUST_CRNT_ADDR_DM
double,CUST_FIRST_SHIPTO_CNT double,CUST_FIRST_SALES_CNT double,CUST_GMT_OFFSET double,CUST_DEMO_CNT
double,CUST_INCOME double,PROD_UNLIMITED double,PROD_OFF_PRICE double,PROD_UNITS double,TOTAL_PRD_COST
double,TOTAL_PRD_DISC double,PROD_WEIGHT double,REG_UNIT_PRICE double,EXTENDED_AMT double,UNIT_PRICE_DSCNT_PCT
double,DSCNT_AMT double,PROD_STD_CST double,TOTAL_TX_AMT double,FREIGHT_CHRG double,WAITING_PERIOD
double,DELIVERY_PERIOD double,ITM_CRNT_PRICE double,ITM_UNITS double,ITM_WSLE_CST double,ITM_SIZE
double,PRM_CST double,PRM_RESPONSE_TARGET double,PRM_ITM_DM double,SHP_MODE_CNT double,WH_GMT_OFFSET
double,WH_SQ_FT double,STR_ORD_QTY double,STR_WSLE_CST double,STR_LIST_
 PRICE double,STR_SALES_PRICE double,STR_EXT_DSCNT_AMT double,STR_EXT_SALES_PRICE double,STR_EXT_WSLE_CST
double,STR_EXT_LIST_PRICE double,STR_EXT_TX double,STR_COUPON_AMT double,STR_NET_PAID double,STR_NET_PAID_INC_TX
double,STR_NET_PRFT double,STR_SOLD_YR_CNT double,STR_SOLD_MM_CNT double,STR_SOLD_ITM_CNT
double,STR_TOTAL_CUST_CNT double,STR_AREA_CNT double,STR_DEMO_CNT double,STR_OFFER_CNT double,STR_PRM_CNT
double,STR_TICKET_CNT double,STR_NET_PRFT_DM_A double,STR_NET_PRFT_DM_B double,STR_NET_PRFT_DM_C
double,STR_NET_PRFT_DM_D double,STR_NET_PRFT_DM_E double,STR_RET_STR_ID double,STR_RET_REASON_CNT
double,STR_RET_TICKET_NO double,STR_RTRN_QTY double,STR_RTRN_AMT double,STR_RTRN_TX double,STR_RTRN_AMT_INC_TX
double,STR_RET_FEE double,STR_RTRN_SHIP_CST double,STR_RFNDD_CSH double,STR_REVERSED_CHRG
double,STR_STR_CREDIT double,STR_RET_NET_LOSS double,STR_RTRNED_YR_CNT double,STR_RTRN_MM_CNT
double,STR_RET_ITM_CNT double,STR_RET_CUST_CNT double,STR_RET_AREA_CNT double,STR_RET_OFFER_C
 NT double,STR_RET_PRM_CNT double,STR_RET_NET_LOSS_DM_A double,STR_RET_NET_LOSS_DM_B double,STR_RET_NET_LOSS_DM_C
double,STR_RET_NET_LOSS_DM_D double,OL_ORD_QTY double,OL_WSLE_CST double,OL_LIST_PRICE double,OL_SALES_PRICE
double,OL_EXT_DSCNT_AMT double,OL_EXT_SALES_PRICE double,OL_EXT_WSLE_CST double,OL_EXT_LIST_PRICE
double,OL_EXT_TX double,OL_COUPON_AMT double,OL_EXT_SHIP_CST double,OL_NET_PAID double,OL_NET_PAID_INC_TX
double,OL_NET_PAID_INC_SHIP double,OL_NET_PAID_INC_SHIP_TX double,OL_NET_PRFT double,OL_SOLD_YR_CNT
double,OL_SOLD_MM_CNT double,OL_SHIP_DATE_CNT double,OL_ITM_CNT double,OL_BILL_CUST_CNT double,OL_BILL_AREA_CNT
double,OL_BILL_DEMO_CNT double,OL_BILL_OFFER_CNT double,OL_SHIP_CUST_CNT double,OL_SHIP_AREA_CNT
double,OL_SHIP_DEMO_CNT double,OL_SHIP_OFFER_CNT double,OL_WEB_PAGE_CNT double,OL_WEB_SITE_CNT
double,OL_SHIP_MODE_CNT double,OL_WH_CNT double,OL_PRM_CNT double,OL_NET_PRFT_DM_A double,OL_NET_PRFT_DM_B
double,OL_NET_PRFT_DM_C double,OL_NET_PRFT_DM_D double,OL_RE
 T_RTRN_QTY double,OL_RTRN_AMT double,OL_RTRN_TX double,OL_RTRN_AMT_INC_TX double,OL_RET_FEE
double,OL_RTRN_SHIP_CST double,OL_RFNDD_CSH double,OL_REVERSED_CHRG double,OL_ACCOUNT_CREDIT
double,OL_RTRNED_YR_CNT double,OL_RTRNED_MM_CNT double,OL_RTRITM_CNT double,OL_RFNDD_CUST_CNT
double,OL_RFNDD_AREA_CNT double,OL_RFNDD_DEMO_CNT double,OL_RFNDD_OFFER_CNT double,OL_RTRNING_CUST_CNT
double,OL_RTRNING_AREA_CNT double,OL_RTRNING_DEMO_CNT double,OL_RTRNING_OFFER_CNT double,OL_RTRWEB_PAGE_CNT
double,OL_REASON_CNT double,OL_NET_LOSS double,OL_NET_LOSS_DM_A double,OL_NET_LOSS_DM_B double,OL_NET_LOSS_DM_C
double) STORED BY 'org.apache.carbondata.format' tblproperties('DICTIONARY_EXCLUDE'='CUST_ID,CUST_NICK_NAME,CUST_FIRST_NAME,CUST_LAST_NAME,CUST_LOGIN,CUST_EMAIL_ADDR,PROD_UNQ_MDL_ID,PROD_UNQ_DEVICE_ADDR,PROD_UQ_UUID,DEVICE_NAME,PROD_BAR_CODE,ITM_ID,ITM_NAME,ITM_BRAND_ID,ITM_BRAND,BOM,PACKING_LIST_NO,TRACKING_NO,ACTIVE_FIRMWARE_VER,LATEST_FIRMWARE_VER,LATEST_EMUI_VERSION,LATEST_NETWORK,STR_ORD
 ER_NO','table_blocksize'='256')""")
     sql(s"""load data LOCAL inpath '$resourcesPath/oscon_10.csv' into table oscon_carbon_old
options('DELIMITER'=',', 'QUOTECHAR'='\"','FILEHEADER'='ACTIVE_AREA_ID, ACTIVE_CHECK_DY, ACTIVE_CHECK_HOUR,
ACTIVE_CHECK_MM, ACTIVE_CHECK_TIME, ACTIVE_CHECK_YR, ACTIVE_CITY, ACTIVE_COUNTRY, ACTIVE_DISTRICT,
ACTIVE_EMUI_VERSION, ACTIVE_FIRMWARE_VER, ACTIVE_NETWORK, ACTIVE_OS_VERSION, ACTIVE_PROVINCE,
BOM, CHECK_DATE, CHECK_DY, CHECK_HOUR, CHECK_MM, CHECK_YR, CUST_ADDRESS_ID, CUST_AGE, CUST_BIRTH_COUNTRY,
CUST_BIRTH_DY, CUST_BIRTH_MM, CUST_BIRTH_YR, CUST_BUY_POTENTIAL, CUST_CITY, CUST_STATE, CUST_COUNTRY,
CUST_COUNTY, CUST_EMAIL_ADDR, CUST_LAST_RVW_DATE, CUST_FIRST_NAME, CUST_ID, CUST_JOB_TITLE,
CUST_LAST_NAME, CUST_LOGIN, CUST_NICK_NAME, CUST_PRFRD_FLG, CUST_SEX, CUST_STREET_NAME, CUST_STREET_NO,
CUST_SUITE_NO, CUST_ZIP, DELIVERY_CITY, DELIVERY_STATE, DELIVERY_COUNTRY, DELIVERY_DISTRICT,
DELIVERY_PROVINCE, DEVICE_NAME, INSIDE_NAME, ITM_BRAND, ITM_BRAND_ID, ITM_CATEGORY, ITM_CATEGORY_ID,
ITM_C
 LASS, ITM_CLASS_ID, ITM_COLOR, ITM_CONTAINER, ITM_FORMULATION, ITM_MANAGER_ID, ITM_MANUFACT,
ITM_MANUFACT_ID, ITM_ID, ITM_NAME, ITM_REC_END_DATE, ITM_REC_START_DATE, LATEST_AREAID, LATEST_CHECK_DY,
LATEST_CHECK_HOUR, LATEST_CHECK_MM, LATEST_CHECK_TIME, LATEST_CHECK_YR, LATEST_CITY, LATEST_COUNTRY,
LATEST_DISTRICT, LATEST_EMUI_VERSION, LATEST_FIRMWARE_VER, LATEST_NETWORK, LATEST_OS_VERSION,
LATEST_PROVINCE, OL_ORDER_DATE, OL_ORDER_NO, OL_RET_ORDER_NO, OL_RET_DATE, OL_SITE, OL_SITE_DESC,
PACKING_DATE, PACKING_DY, PACKING_HOUR, PACKING_LIST_NO, PACKING_MM, PACKING_YR, PRMTION_ID,
PRMTION_NAME, PRM_CHANNEL_CAT, PRM_CHANNEL_DEMO, PRM_CHANNEL_DETAILS, PRM_CHANNEL_DMAIL, PRM_CHANNEL_EMAIL,
PRM_CHANNEL_EVENT, PRM_CHANNEL_PRESS, PRM_CHANNEL_RADIO, PRM_CHANNEL_TV, PRM_DSCNT_ACTIVE,
PRM_END_DATE, PRM_PURPOSE, PRM_START_DATE, PRODUCT_ID, PROD_BAR_CODE, PROD_BRAND_NAME, PRODUCT_NAME,
PRODUCT_MODEL, PROD_MODEL_ID, PROD_COLOR, PROD_SHELL_COLOR, PROD_CPU_CLOCK, PROD_IMAGE, PROD_LIVE,
PROD_LOC, PROD
 _LONG_DESC, PROD_RAM, PROD_ROM, PROD_SERIES, PROD_SHORT_DESC, PROD_THUMB, PROD_UNQ_DEVICE_ADDR,
PROD_UNQ_MDL_ID, PROD_UPDATE_DATE, PROD_UQ_UUID, SHP_CARRIER, SHP_CODE, SHP_CONTRACT, SHP_MODE_ID,
SHP_MODE, STR_ORDER_DATE, STR_ORDER_NO, TRACKING_NO, WH_CITY, WH_COUNTRY, WH_COUNTY, WH_ID,
WH_NAME, WH_STATE, WH_STREET_NAME, WH_STREET_NO, WH_STREET_TYPE, WH_SUITE_NO, WH_ZIP, CUST_DEP_COUNT,
CUST_VEHICLE_COUNT, CUST_ADDRESS_CNT, CUST_CRNT_CDEMO_CNT, CUST_CRNT_HDEMO_CNT, CUST_CRNT_ADDR_DM,
CUST_FIRST_SHIPTO_CNT, CUST_FIRST_SALES_CNT, CUST_GMT_OFFSET, CUST_DEMO_CNT, CUST_INCOME,
PROD_UNLIMITED, PROD_OFF_PRICE, PROD_UNITS, TOTAL_PRD_COST, TOTAL_PRD_DISC, PROD_WEIGHT, REG_UNIT_PRICE,
EXTENDED_AMT, UNIT_PRICE_DSCNT_PCT, DSCNT_AMT, PROD_STD_CST, TOTAL_TX_AMT, FREIGHT_CHRG, WAITING_PERIOD,
DELIVERY_PERIOD, ITM_CRNT_PRICE, ITM_UNITS, ITM_WSLE_CST, ITM_SIZE, PRM_CST, PRM_RESPONSE_TARGET,
PRM_ITM_DM, SHP_MODE_CNT, WH_GMT_OFFSET, WH_SQ_FT, STR_ORD_QTY, STR_WSLE_CST, STR_LIST_PRICE,
STR_SALES_PRICE, 
 STR_EXT_DSCNT_AMT, STR_EXT_SALES_PRICE, STR_EXT_WSLE_CST, STR_EXT_LIST_PRICE, STR_EXT_TX,
STR_COUPON_AMT, STR_NET_PAID, STR_NET_PAID_INC_TX, STR_NET_PRFT, STR_SOLD_YR_CNT, STR_SOLD_MM_CNT,
STR_SOLD_ITM_CNT, STR_TOTAL_CUST_CNT, STR_AREA_CNT, STR_DEMO_CNT, STR_OFFER_CNT, STR_PRM_CNT,
STR_TICKET_CNT, STR_NET_PRFT_DM_A, STR_NET_PRFT_DM_B, STR_NET_PRFT_DM_C, STR_NET_PRFT_DM_D,
STR_NET_PRFT_DM_E, STR_RET_STR_ID, STR_RET_REASON_CNT, STR_RET_TICKET_NO, STR_RTRN_QTY, STR_RTRN_AMT,
STR_RTRN_TX, STR_RTRN_AMT_INC_TX, STR_RET_FEE, STR_RTRN_SHIP_CST, STR_RFNDD_CSH, STR_REVERSED_CHRG,
STR_STR_CREDIT, STR_RET_NET_LOSS, STR_RTRNED_YR_CNT, STR_RTRN_MM_CNT, STR_RET_ITM_CNT, STR_RET_CUST_CNT,
STR_RET_AREA_CNT, STR_RET_OFFER_CNT, STR_RET_PRM_CNT, STR_RET_NET_LOSS_DM_A, STR_RET_NET_LOSS_DM_B,
STR_RET_NET_LOSS_DM_C, STR_RET_NET_LOSS_DM_D, OL_ORD_QTY, OL_WSLE_CST, OL_LIST_PRICE, OL_SALES_PRICE,
OL_EXT_DSCNT_AMT, OL_EXT_SALES_PRICE, OL_EXT_WSLE_CST, OL_EXT_LIST_PRICE, OL_EXT_TX, OL_COUPON_AMT,
OL_EXT_SHIP_C
 ST, OL_NET_PAID, OL_NET_PAID_INC_TX, OL_NET_PAID_INC_SHIP, OL_NET_PAID_INC_SHIP_TX, OL_NET_PRFT,
OL_SOLD_YR_CNT, OL_SOLD_MM_CNT, OL_SHIP_DATE_CNT, OL_ITM_CNT, OL_BILL_CUST_CNT, OL_BILL_AREA_CNT,
OL_BILL_DEMO_CNT, OL_BILL_OFFER_CNT, OL_SHIP_CUST_CNT, OL_SHIP_AREA_CNT, OL_SHIP_DEMO_CNT,
OL_SHIP_OFFER_CNT, OL_WEB_PAGE_CNT, OL_WEB_SITE_CNT, OL_SHIP_MODE_CNT, OL_WH_CNT, OL_PRM_CNT,
OL_NET_PRFT_DM_A, OL_NET_PRFT_DM_B, OL_NET_PRFT_DM_C, OL_NET_PRFT_DM_D, OL_RET_RTRN_QTY, OL_RTRN_AMT,
OL_RTRN_TX, OL_RTRN_AMT_INC_TX, OL_RET_FEE, OL_RTRN_SHIP_CST, OL_RFNDD_CSH, OL_REVERSED_CHRG,
OL_ACCOUNT_CREDIT, OL_RTRNED_YR_CNT, OL_RTRNED_MM_CNT, OL_RTRITM_CNT, OL_RFNDD_CUST_CNT, OL_RFNDD_AREA_CNT,
OL_RFNDD_DEMO_CNT, OL_RFNDD_OFFER_CNT, OL_RTRNING_CUST_CNT, OL_RTRNING_AREA_CNT, OL_RTRNING_DEMO_CNT,
OL_RTRNING_OFFER_CNT, OL_RTRWEB_PAGE_CNT, OL_REASON_CNT, OL_NET_LOSS, OL_NET_LOSS_DM_A, OL_NET_LOSS_DM_B,
OL_NET_LOSS_DM_C','BAD_RECORDS_ACTION'='FORCE','BAD_RECORDS_LOGGER_ENABLE'='FALSE')""")
   }
@@ -33,7 +44,128 @@ class DoubleDataTypeTest extends QueryTest with BeforeAndAfterAll {
     assert(result.equals(10L))
   }
 
+  test("Double Datatype Check with AdaptiveDeltaFloating Codec (BYTE)") {
+
+    sql(
+      """create table carbon_datatype_double_byte (c_double double) stored by 'org.apache.carbondata.format'""".stripMargin)
+    sql(s"load data inpath '$resourcesPath/double/data_notitle_byte.csv' into table " +
+        s"carbon_datatype_double_byte options('delimiter'='|', 'fileheader'='id, c_string,
" +
+        s"c_bigint, c_decimal, c_double, c_timestamp')")
+    checkAnswer(sql("select * from carbon_datatype_double_byte"),
+      Seq(Row(3.1412), Row(3.1413), Row(3.1414), Row(3.1415)))
+  }
+
+  test("Double Datatype Check with AdaptiveDeltaFloating Codec (Short)") {
+
+    sql(
+      """create table carbon_datatype_double_short (c_double double) stored by 'org.apache.carbondata.format'""")
+    sql(s"load data inpath '$resourcesPath/double/data_notitle_short.csv' into table " +
+        s"carbon_datatype_double_short options('delimiter'='|', 'fileheader'='id, c_string,
" +
+        s"c_bigint, c_decimal, c_double, c_timestamp')")
+    checkAnswer(sql("select * from carbon_datatype_double_short"),
+      Seq(Row(3.1412), Row(3.1413), Row(3.1414), Row(5.1415)))
+  }
+
+  test("Double Datatype Check with AdaptiveDeltaFloating Codec (Short_Int)") {
+
+    sql(
+      """create table carbon_datatype_double_sint (c_double double) stored by 'org.apache.carbondata.format'""")
+    sql(s"load data inpath '$resourcesPath/double/data_notitle_short_int.csv' into table
" +
+        s"carbon_datatype_double_sint options('delimiter'='|', 'fileheader'='id, c_string,
" +
+        s"c_bigint, c_decimal, c_double, c_timestamp')")
+    checkAnswer(sql("select * from carbon_datatype_double_sint"),
+      Seq(Row(300.1412), Row(300.1413), Row(300.1414), Row(900.1415)))
+  }
+
+  test("Double Datatype Check with AdaptiveDeltaFloating Codec (Int)") {
+
+    sql(
+      """create table carbon_datatype_double_int (c_double double) stored by 'org.apache.carbondata.format'""")
+    sql(s"load data inpath '$resourcesPath/double/data_notitle_int.csv' into table " +
+        s"carbon_datatype_double_int options('delimiter'='|', 'fileheader'='id, c_string,
" +
+        s"c_bigint, c_decimal, c_double, c_timestamp')")
+    checkAnswer(sql("select * from carbon_datatype_double_int"),
+      Seq(Row(199161.2812), Row(199161.2813), Row(199161.2814), Row(200000.1415)))
+  }
+
+  test("Double Datatype Check with AdaptiveDeltaFloating Codec (Long)") {
+
+    sql(
+      """create table carbon_datatype_double_long (c_double double) stored by 'org.apache.carbondata.format'""")
+    sql(s"load data inpath '$resourcesPath/double/data_notitle_long.csv' into table " +
+        s"carbon_datatype_double_long options('delimiter'='|', 'fileheader'='id, c_string,
" +
+        s"c_bigint, c_decimal, c_double, c_timestamp')")
+    checkAnswer(sql("select * from carbon_datatype_double_long"),
+      Seq(Row(100000.1412), Row(100000.1413), Row(100000.1414), Row(200000.1415)))
+  }
+
+  test("Double Datatype Check with AdaptiveFloatingCodec Codec (BYTE)") {
+
+    sql(
+      """create table carbon_datatype_AdaptiveFloatingCodec_double_byte (c_double double)
stored by 'org.apache.carbondata.format'""".stripMargin)
+    sql(s"load data inpath '$resourcesPath/double/data_notitle_AdaptiveFloating_byte.csv'
into table " +
+        s"carbon_datatype_AdaptiveFloatingCodec_double_byte options('delimiter'='|', 'fileheader'='id,
c_string, " +
+        s"c_bigint, c_decimal, c_double, c_timestamp')")
+    checkAnswer(sql("select * from carbon_datatype_AdaptiveFloatingCodec_double_byte"),
+      Seq(Row(0.0012), Row(0.0013), Row(0.0014), Row(0.0015)))
+  }
+
+  test("Double Datatype Check with AdaptiveFloatingCodec Codec (Short)") {
+
+    sql(
+      """create table carbon_datatype_AdaptiveFloatingCodec_double_short (c_double double)
stored by 'org.apache.carbondata.format'""")
+    sql(s"load data inpath '$resourcesPath/double/data_notitle_AdaptiveFloating_short.csv'
into table " +
+        s"carbon_datatype_AdaptiveFloatingCodec_double_short options('delimiter'='|', 'fileheader'='id,
c_string, " +
+        s"c_bigint, c_decimal, c_double, c_timestamp')")
+    checkAnswer(sql("select * from carbon_datatype_AdaptiveFloatingCodec_double_short"),
+      Seq(Row(0.1412), Row(0.1413), Row(0.1414), Row(0.1415)))
+  }
+
+  test("Double Datatype Check with AdaptiveFloatingCodec Codec (Short_Int)") {
+
+    sql(
+      """create table carbon_datatype_AdaptiveFloatingCodec_double_sint (c_double double)
stored by 'org.apache.carbondata.format'""")
+    sql(s"load data inpath '$resourcesPath/double/data_notitle_AdaptiveFloating_short_int.csv'
into table " +
+        s"carbon_datatype_AdaptiveFloatingCodec_double_sint options('delimiter'='|', 'fileheader'='id,
c_string, " +
+        s"c_bigint, c_decimal, c_double, c_timestamp')")
+    checkAnswer(sql("select * from carbon_datatype_AdaptiveFloatingCodec_double_sint"),
+      Seq(Row(300.1412), Row(300.1413), Row(300.1414), Row(800.1415)))
+  }
+
+  test("Double Datatype Check with AdaptiveFloatingCodec Codec (Int)") {
+
+    sql(
+      """create table carbon_datatype_AdaptiveFloatingCodec_double_int (c_double double)
stored by 'org.apache.carbondata.format'""")
+    sql(s"load data inpath '$resourcesPath/double/data_notitle_AdaptiveFloating_int.csv'
into table " +
+        s"carbon_datatype_AdaptiveFloatingCodec_double_int options('delimiter'='|', 'fileheader'='id,
c_string, " +
+        s"c_bigint, c_decimal, c_double, c_timestamp')")
+    checkAnswer(sql("select * from carbon_datatype_AdaptiveFloatingCodec_double_int"),
+      Seq(Row(100000.2812), Row(100000.2813), Row(100000.2814), Row(200000.1415)))
+  }
+
+  test("Double Datatype Check with AdaptiveFloatingCodec Codec (Long)") {
+
+    sql(
+      """create table carbon_datatype_AdaptiveFloatingCodec_double_long (c_double double)
stored by 'org.apache.carbondata.format'""")
+    sql(s"load data inpath '$resourcesPath/double/data_notitle_long.csv' into table " +
+        s"carbon_datatype_AdaptiveFloatingCodec_double_long options('delimiter'='|', 'fileheader'='id,
c_string, " +
+        s"c_bigint, c_decimal, c_double, c_timestamp')")
+    checkAnswer(sql("select * from carbon_datatype_AdaptiveFloatingCodec_double_long"),
+      Seq(Row(100000.1412), Row(100000.1413), Row(100000.1414), Row(200000.1415)))
+  }
+
+
   override def afterAll: Unit = {
+    sql("drop table if exists carbon_datatype_double_byte")
+    sql("drop table if exists carbon_datatype_double_short")
+    sql("drop table if exists carbon_datatype_double_sint")
+    sql("drop table if exists carbon_datatype_double_int")
+    sql("drop table if exists carbon_datatype_double_long")
+    sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_byte")
+    sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_short")
+    sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_sint")
+    sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_int")
+    sql("drop table if exists carbon_datatype_AdaptiveFloatingCodec_double_long")
     sql("DROP TABLE IF EXISTS oscon_carbon_old")
   }
 


Mime
View raw message