carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [5/7] carbondata git commit: [CARBONDATA-1015] Refactory write step and add ColumnPage in data load This closes #852
Date Fri, 12 May 2017 14:29:35 GMT
[CARBONDATA-1015] Refactory write step and add ColumnPage in data load This closes #852


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d9ecfb50
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d9ecfb50
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d9ecfb50

Branch: refs/heads/12-dev
Commit: d9ecfb503dd955e22727f837ee5dffa7af3a90fc
Parents: 663176a
Author: jackylk <jacky.likun@huawei.com>
Authored: Thu May 4 23:32:07 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Fri May 12 21:52:31 2017 +0800

----------------------------------------------------------------------
 .../core/compression/BigIntCompressor.java      |   9 +-
 .../core/compression/DoubleCompressor.java      |  43 +-
 .../core/compression/ValueCompressor.java       |   3 +-
 .../core/constants/CarbonCommonConstants.java   |   6 -
 .../chunk/store/MeasureChunkStoreFactory.java   |  26 +-
 .../BlockIndexerStorageForNoInvertedIndex.java  |   2 +-
 ...ndexerStorageForNoInvertedIndexForShort.java |   3 +-
 .../compression/MeasureMetaDataModel.java       |   8 +-
 .../compression/ReaderCompressModel.java        |   8 +-
 .../compression/ValueCompressionHolder.java     |  26 +-
 .../compression/WriterCompressModel.java        |  41 +-
 .../compression/decimal/CompressByteArray.java  |   4 +-
 .../decimal/CompressionMaxMinByte.java          |   6 +-
 .../decimal/CompressionMaxMinDefault.java       |   6 +-
 .../decimal/CompressionMaxMinInt.java           |   6 +-
 .../decimal/CompressionMaxMinLong.java          |   6 +-
 .../decimal/CompressionMaxMinShort.java         |   6 +-
 .../nondecimal/CompressionNonDecimalByte.java   |   6 +-
 .../CompressionNonDecimalDefault.java           |   6 +-
 .../nondecimal/CompressionNonDecimalInt.java    |   6 +-
 .../nondecimal/CompressionNonDecimalLong.java   |   6 +-
 .../CompressionNonDecimalMaxMinByte.java        |   6 +-
 .../CompressionNonDecimalMaxMinDefault.java     |   6 +-
 .../CompressionNonDecimalMaxMinInt.java         |   6 +-
 .../CompressionNonDecimalMaxMinLong.java        |   6 +-
 .../CompressionNonDecimalMaxMinShort.java       |   6 +-
 .../nondecimal/CompressionNonDecimalShort.java  |   6 +-
 .../compression/none/CompressionNoneByte.java   |   6 +-
 .../none/CompressionNoneDefault.java            |   6 +-
 .../compression/none/CompressionNoneInt.java    |   6 +-
 .../compression/none/CompressionNoneLong.java   |   6 +-
 .../compression/none/CompressionNoneShort.java  |   6 +-
 .../dataholder/CarbonWriteDataHolder.java       | 156 +--
 .../HeavyCompressedDoubleArrayDataStore.java    |  57 --
 .../core/datastore/page/ColumnPage.java         |  42 +
 .../core/datastore/page/ComplexColumnPage.java  |  77 ++
 .../datastore/page/FixLengthColumnPage.java     | 155 +++
 .../datastore/page/VarLengthColumnPage.java     |  42 +
 .../datastore/page/compression/Compression.java |  23 +
 .../datastore/page/encoding/ColumnCodec.java    |  35 +
 .../datastore/page/encoding/DummyCodec.java     |  37 +
 .../page/statistics/PageStatistics.java         | 124 +++
 .../page/statistics/StatisticsCollector.java    |  66 ++
 .../core/metadata/ValueEncoderMeta.java         |  18 +-
 .../core/metadata/datatype/DataType.java        |   6 +-
 .../apache/carbondata/core/util/ByteUtil.java   |   2 +
 .../core/util/CarbonMetadataUtil.java           |  44 +-
 .../carbondata/core/util/CarbonProperties.java  |   1 -
 .../apache/carbondata/core/util/CarbonUtil.java |   2 +-
 .../carbondata/core/util/CompressionFinder.java |  10 +-
 .../carbondata/core/util/DataTypeUtil.java      |   3 -
 .../apache/carbondata/core/util/NodeHolder.java |  31 +-
 .../core/util/ValueCompressionUtil.java         | 190 ++--
 .../core/util/CarbonMetadataUtilTest.java       |  11 +-
 .../core/util/ValueCompressionUtilTest.java     | 114 +--
 .../core/writer/CarbonFooterWriterTest.java     |   2 +-
 .../examples/CarbonSessionExample.scala         |   1 +
 format/src/main/thrift/carbondata.thrift        |   8 +-
 .../testsuite/emptyrow/TestEmptyRows.scala      |   2 +-
 .../dataload/TestLoadDataWithNoMeasure.scala    |   8 +-
 .../ColumnGroupDataTypesTestCase.scala          |   6 +
 .../spark/sql/parser/CarbonSparkSqlParser.scala |   2 +-
 .../merger/CompactionResultSortProcessor.java   |  17 +-
 .../sort/impl/ParallelReadMergeSorterImpl.java  |   2 +-
 ...arallelReadMergeSorterWithBucketingImpl.java |   2 +-
 .../sort/unsafe/UnsafeCarbonRowPage.java        | 133 +--
 .../newflow/sort/unsafe/UnsafeSortDataRows.java |  10 +-
 .../holder/UnsafeSortTempFileChunkHolder.java   |  29 +-
 .../merger/UnsafeIntermediateFileMerger.java    |  38 +-
 .../CarbonRowDataWriterProcessorStepImpl.java   |  10 +-
 .../sortdata/IntermediateFileMerger.java        |  37 +-
 .../sortandgroupby/sortdata/SortDataRows.java   |  29 +-
 .../sortandgroupby/sortdata/SortParameters.java |  25 +-
 .../sortdata/SortTempFileChunkHolder.java       |  29 +-
 .../store/CarbonFactDataHandlerColumnar.java    | 975 ++++++++-----------
 .../store/CarbonFactDataHandlerModel.java       |  27 +-
 .../store/SingleThreadFinalSortFilesMerger.java |  11 +-
 .../store/colgroup/ColGroupBlockStorage.java    |  29 +-
 .../writer/v3/CarbonFactDataWriterImplV3.java   |   4 +-
 .../util/CarbonDataProcessorUtil.java           |  50 +-
 .../store/colgroup/ColGroupMinMaxTest.java      |   2 +
 81 files changed, 1727 insertions(+), 1306 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
index 8e43684..8360a68 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
@@ -17,7 +17,7 @@
 package org.apache.carbondata.core.compression;
 
 import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 
 /**
  * It compresses big int data
@@ -63,6 +63,7 @@ public class BigIntCompressor extends ValueCompressor {
    */
   @Override
   protected Object compressAdaptive(DataType convertedDataType, CarbonWriteDataHolder dataHolder) {
+
     long[] value = dataHolder.getWritableLongValues();
     return compressValue(convertedDataType, value, 0, false);
   }
@@ -82,7 +83,7 @@ public class BigIntCompressor extends ValueCompressor {
   protected Object compressValue(DataType convertedDataType, long[] value, long maxValue,
       boolean isMinMax) {
     switch (convertedDataType) {
-      case DATA_BYTE:
+      case BYTE:
         byte[] result = new byte[value.length];
         if (isMinMax) {
           for (int j = 0; j < value.length; j++) {
@@ -94,7 +95,7 @@ public class BigIntCompressor extends ValueCompressor {
           }
         }
         return result;
-      case DATA_SHORT:
+      case SHORT:
         short[] shortResult = new short[value.length];
         if (isMinMax) {
           for (int j = 0; j < value.length; j++) {
@@ -106,7 +107,7 @@ public class BigIntCompressor extends ValueCompressor {
           }
         }
         return shortResult;
-      case DATA_INT:
+      case INT:
         int[] intResult = new int[value.length];
         if (isMinMax) {
           for (int j = 0; j < value.length; j++) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
index 840709d..bc2d6f1 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.compression;
 import java.math.BigDecimal;
 
 import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 
 /**
  * Double compressor
@@ -33,7 +33,7 @@ public class DoubleCompressor extends ValueCompressor {
     BigDecimal max = BigDecimal.valueOf((double)maxValue);
     double[] value = dataHolder.getWritableDoubleValues();
     switch (convertedDataType) {
-      case DATA_BYTE:
+      case BYTE:
         byte[] result = new byte[value.length];
         for (int j = 0; j < value.length; j++) {
           BigDecimal val = BigDecimal.valueOf(value[j]);
@@ -42,7 +42,7 @@ public class DoubleCompressor extends ValueCompressor {
           i++;
         }
         return result;
-      case DATA_SHORT:
+      case SHORT:
         short[] shortResult = new short[value.length];
         for (int j = 0; j < value.length; j++) {
           BigDecimal val = BigDecimal.valueOf(value[j]);
@@ -51,7 +51,7 @@ public class DoubleCompressor extends ValueCompressor {
           i++;
         }
         return shortResult;
-      case DATA_INT:
+      case INT:
         int[] intResult = new int[value.length];
         for (int j = 0; j < value.length; j++) {
           BigDecimal val = BigDecimal.valueOf(value[j]);
@@ -60,7 +60,7 @@ public class DoubleCompressor extends ValueCompressor {
           i++;
         }
         return intResult;
-      case DATA_LONG:
+      case LONG:
         long[] longResult = new long[value.length];
         for (int j = 0; j < value.length; j++) {
           BigDecimal val = BigDecimal.valueOf(value[j]);
@@ -69,7 +69,7 @@ public class DoubleCompressor extends ValueCompressor {
           i++;
         }
         return longResult;
-      case DATA_FLOAT:
+      case FLOAT:
         float[] floatResult = new float[value.length];
         for (int j = 0; j < value.length; j++) {
           BigDecimal val = BigDecimal.valueOf(value[j]);
@@ -96,35 +96,35 @@ public class DoubleCompressor extends ValueCompressor {
     int i = 0;
     double[] value = dataHolder.getWritableDoubleValues();
     switch (convertedDataType) {
-      case DATA_BYTE:
+      case BYTE:
         byte[] result = new byte[value.length];
         for (int j = 0; j < value.length; j++) {
           result[i] = (byte) (Math.round(Math.pow(10, decimal) * value[j]));
           i++;
         }
         return result;
-      case DATA_SHORT:
+      case SHORT:
         short[] shortResult = new short[value.length];
         for (int j = 0; j < value.length; j++) {
           shortResult[i] = (short) (Math.round(Math.pow(10, decimal) * value[j]));
           i++;
         }
         return shortResult;
-      case DATA_INT:
+      case INT:
         int[] intResult = new int[value.length];
         for (int j = 0; j < value.length; j++) {
           intResult[i] = (int) (Math.round(Math.pow(10, decimal) * value[j]));
           i++;
         }
         return intResult;
-      case DATA_LONG:
+      case LONG:
         long[] longResult = new long[value.length];
         for (int j = 0; j < value.length; j++) {
           longResult[i] = Math.round(Math.pow(10, decimal) * value[j]);
           i++;
         }
         return longResult;
-      case DATA_FLOAT:
+      case FLOAT:
         float[] floatResult = new float[value.length];
         for (int j = 0; j < value.length; j++) {
           floatResult[i] = (float) (Math.round(Math.pow(10, decimal) * value[j]));
@@ -148,35 +148,35 @@ public class DoubleCompressor extends ValueCompressor {
     double[] value = dataHolder.getWritableDoubleValues();
     int i = 0;
     switch (convertedDataType) {
-      case DATA_BYTE:
+      case BYTE:
         byte[] result = new byte[value.length];
         for (int j = 0; j < value.length; j++) {
           result[i] = (byte) (maxValue - value[j]);
           i++;
         }
         return result;
-      case DATA_SHORT:
+      case SHORT:
         short[] shortResult = new short[value.length];
         for (int j = 0; j < value.length; j++) {
           shortResult[i] = (short) (maxValue - value[j]);
           i++;
         }
         return shortResult;
-      case DATA_INT:
+      case INT:
         int[] intResult = new int[value.length];
         for (int j = 0; j < value.length; j++) {
           intResult[i] = (int) (maxValue - value[j]);
           i++;
         }
         return intResult;
-      case DATA_LONG:
+      case LONG:
         long[] longResult = new long[value.length];
         for (int j = 0; j < value.length; j++) {
           longResult[i] = (long) (maxValue - value[j]);
           i++;
         }
         return longResult;
-      case DATA_FLOAT:
+      case FLOAT:
         float[] floatResult = new float[value.length];
         for (int j = 0; j < value.length; j++) {
           floatResult[i] = (float) (maxValue - value[j]);
@@ -198,36 +198,35 @@ public class DoubleCompressor extends ValueCompressor {
     double[] value = dataHolder.getWritableDoubleValues();
     int i = 0;
     switch (changedDataType) {
-      case DATA_BYTE:
+      case BYTE:
         byte[] result = new byte[value.length];
         for (int j = 0; j < value.length; j++) {
           result[i] = (byte) value[j];
           i++;
         }
         return result;
-      case DATA_SHORT:
+      case SHORT:
         short[] shortResult = new short[value.length];
         for (int j = 0; j < value.length; j++) {
           shortResult[i] = (short) value[j];
           i++;
         }
         return shortResult;
-      case DATA_INT:
+      case INT:
         int[] intResult = new int[value.length];
         for (int j = 0; j < value.length; j++) {
           intResult[i] = (int) value[j];
           i++;
         }
         return intResult;
-      case DATA_LONG:
-      case DATA_BIGINT:
+      case LONG:
         long[] longResult = new long[value.length];
         for (int j = 0; j < value.length; j++) {
           longResult[i] = (long) value[j];
           i++;
         }
         return longResult;
-      case DATA_FLOAT:
+      case FLOAT:
         float[] floatResult = new float[value.length];
         for (int j = 0; j < value.length; j++) {
           floatResult[i] = (float) value[j];

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
index 230507f..16f8ac1 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
@@ -17,10 +17,9 @@
 package org.apache.carbondata.core.compression;
 
 import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.CompressionFinder;
 import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
 /**
  * Measure compressor
  */

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 7c59a59..627975a 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -647,12 +647,6 @@ public final class CarbonCommonConstants {
   public static final char BIG_INT_MEASURE = 'd';
 
   /**
-   * This determines the size of array to be processed in data load steps. one
-   * for dimensions , one of ignore dictionary dimensions , one for measures.
-   */
-  public static final int ARRAYSIZE = 3;
-
-  /**
    * CARBON_PREFETCH_BUFFERSIZE
    */
   public static final String CARBON_PREFETCH_BUFFERSIZE = "carbon.prefetch.buffersize";

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
index 2a3a2a7..12bfea9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
@@ -30,8 +30,8 @@ import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeDouble
 import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeIntMeasureChunkStore;
 import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeLongMeasureChunkStore;
 import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeShortMeasureChunkStore;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 /**
  * Factory class for getting the measure store type
@@ -67,33 +67,33 @@ public class MeasureChunkStoreFactory {
   public MeasureDataChunkStore getMeasureDataChunkStore(DataType dataType, int numberOfRows) {
     if (!isUnsafe) {
       switch (dataType) {
-        case DATA_BYTE:
+        case BYTE:
           return new SafeByteMeasureChunkStore(numberOfRows);
-        case DATA_SHORT:
+        case SHORT:
           return new SafeShortMeasureChunkStore(numberOfRows);
-        case DATA_INT:
+        case INT:
           return new SafeIntMeasureChunkStore(numberOfRows);
-        case DATA_LONG:
+        case LONG:
           return new SafeLongMeasureChunkStore(numberOfRows);
-        case DATA_BIGDECIMAL:
+        case DECIMAL:
           return new SafeBigDecimalMeasureChunkStore(numberOfRows);
-        case DATA_DOUBLE:
+        case DOUBLE:
         default:
           return new SafeDoubleMeasureChunkStore(numberOfRows);
       }
     } else {
       switch (dataType) {
-        case DATA_BYTE:
+        case BYTE:
           return new UnsafeByteMeasureChunkStore(numberOfRows);
-        case DATA_SHORT:
+        case SHORT:
           return new UnsafeShortMeasureChunkStore(numberOfRows);
-        case DATA_INT:
+        case INT:
           return new UnsafeIntMeasureChunkStore(numberOfRows);
-        case DATA_LONG:
+        case LONG:
           return new UnsafeLongMeasureChunkStore(numberOfRows);
-        case DATA_BIGDECIMAL:
+        case DECIMAL:
           return new UnsafeBigDecimalMeasureChunkStore(numberOfRows);
-        case DATA_DOUBLE:
+        case DOUBLE:
         default:
           return new UnsafeDoubleMeasureChunkStore(numberOfRows);
       }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
index 0ef2518..f36ee97 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
@@ -37,7 +37,7 @@ public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]
   private byte[] min;
   private byte[] max;
 
-  public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean isNoDictionary) {
+  public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput) {
     this.keyBlock = keyBlockInput;
     min = keyBlock[0];
     max = keyBlock[0];

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
index 731df96..901084f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
@@ -36,8 +36,7 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
   private byte[] min;
   private byte[] max;
 
-  public BlockIndexerStorageForNoInvertedIndexForShort(byte[][] keyBlockInput,
-      boolean isNoDictionary) {
+  public BlockIndexerStorageForNoInvertedIndexForShort(byte[][] keyBlockInput) {
     this.keyBlock = keyBlockInput;
     min = keyBlock[0];
     max = keyBlock[0];

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
index 5d35b0d..7a39f7c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
@@ -17,6 +17,8 @@
 
 package org.apache.carbondata.core.datastore.compression;
 
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
 public class MeasureMetaDataModel {
   /**
    * maxValue
@@ -46,7 +48,7 @@ public class MeasureMetaDataModel {
   /**
    * type
    */
-  private char[] type;
+  private DataType[] type;
 
   /**
    * dataTypeSelected
@@ -54,7 +56,7 @@ public class MeasureMetaDataModel {
   private byte[] dataTypeSelected;
 
   public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa,
-      int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) {
+      int measureCount, Object[] uniqueValue, DataType[] type, byte[] dataTypeSelected) {
     this.minValue = minValue;
     this.maxValue = maxValue;
     this.mantissa = mantissa;
@@ -112,7 +114,7 @@ public class MeasureMetaDataModel {
   /**
    * @return the type
    */
-  public char[] getType() {
+  public DataType[] getType() {
     return type;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
index 101a24b..60687d1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
@@ -18,13 +18,13 @@
 package org.apache.carbondata.core.datastore.compression;
 
 import org.apache.carbondata.core.metadata.ValueEncoderMeta;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 
 // Used in read path for decompression preparation
 public class ReaderCompressModel {
   private ValueEncoderMeta valueEncoderMeta;
 
-  private ValueCompressionUtil.DataType convertedDataType;
+  private DataType convertedDataType;
 
   private ValueCompressionHolder valueHolder;
 
@@ -32,11 +32,11 @@ public class ReaderCompressModel {
     this.valueEncoderMeta = valueEncoderMeta;
   }
 
-  public ValueCompressionUtil.DataType getConvertedDataType() {
+  public DataType getConvertedDataType() {
     return convertedDataType;
   }
 
-  public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) {
+  public void setConvertedDataType(DataType convertedDataType) {
     this.convertedDataType = convertedDataType;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
index 9b3a18a..614eb32 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.datastore.compression;
 
 import java.math.BigDecimal;
 
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 
 /**
  * ValueCompressionHolder is the base class for handling
@@ -40,24 +40,23 @@ public abstract class ValueCompressionHolder<T> {
   protected void unCompress(Compressor compressor, DataType dataType, byte[] data, int offset,
       int length, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     switch (dataType) {
-      case DATA_BYTE:
+      case BYTE:
         setValue((T) compressor.unCompressByte(data, offset, length), numberOfRows, maxValueObject,
             decimalPlaces);
         break;
-      case DATA_SHORT:
+      case SHORT:
         setValue((T) compressor.unCompressShort(data, offset, length), numberOfRows, maxValueObject,
             decimalPlaces);
         break;
-      case DATA_INT:
+      case INT:
         setValue((T) compressor.unCompressInt(data, offset, length), numberOfRows, maxValueObject,
             decimalPlaces);
         break;
-      case DATA_LONG:
-      case DATA_BIGINT:
+      case LONG:
         setValue((T) compressor.unCompressLong(data, offset, length), numberOfRows, maxValueObject,
             decimalPlaces);
         break;
-      case DATA_FLOAT:
+      case FLOAT:
         setValue((T) compressor.unCompressFloat(data, offset, length), numberOfRows, maxValueObject,
             decimalPlaces);
         break;
@@ -75,18 +74,17 @@ public abstract class ValueCompressionHolder<T> {
    */
   public byte[] compress(Compressor compressor, DataType dataType, Object data) {
     switch (dataType) {
-      case DATA_BYTE:
+      case BYTE:
         return compressor.compressByte((byte[]) data);
-      case DATA_SHORT:
+      case SHORT:
         return compressor.compressShort((short[]) data);
-      case DATA_INT:
+      case INT:
         return compressor.compressInt((int[]) data);
-      case DATA_LONG:
-      case DATA_BIGINT:
+      case LONG:
         return compressor.compressLong((long[]) data);
-      case DATA_FLOAT:
+      case FLOAT:
         return compressor.compressFloat((float[]) data);
-      case DATA_DOUBLE:
+      case DOUBLE:
       default:
         return compressor.compressDouble((double[]) data);
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java
index 0430c8c..a2bf47a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java
@@ -17,19 +17,24 @@
 
 package org.apache.carbondata.core.datastore.compression;
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.CompressionFinder;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
 
+import static org.apache.carbondata.core.metadata.datatype.DataType.INT;
+import static org.apache.carbondata.core.metadata.datatype.DataType.SHORT;
+
 public class WriterCompressModel {
 
   /**
    * DataType[]  variable.
    */
-  private ValueCompressionUtil.DataType[] convertedDataType;
+  private DataType[] convertedDataType;
   /**
    * DataType[]  variable.
    */
-  private ValueCompressionUtil.DataType[] actualDataType;
+  private DataType[] actualDataType;
 
   /**
    * maxValue
@@ -52,7 +57,7 @@ public class WriterCompressModel {
   /**
    * aggType
    */
-  private char[] type;
+  private DataType[] type;
 
   /**
    * dataTypeSelected
@@ -68,28 +73,28 @@ public class WriterCompressModel {
   /**
    * @return the convertedDataType
    */
-  public ValueCompressionUtil.DataType[] getConvertedDataType() {
+  public DataType[] getConvertedDataType() {
     return convertedDataType;
   }
 
   /**
    * @param convertedDataType the convertedDataType to set
    */
-  public void setConvertedDataType(ValueCompressionUtil.DataType[] convertedDataType) {
+  public void setConvertedDataType(DataType[] convertedDataType) {
     this.convertedDataType = convertedDataType;
   }
 
   /**
    * @return the actualDataType
    */
-  public ValueCompressionUtil.DataType[] getActualDataType() {
+  public DataType[] getActualDataType() {
     return actualDataType;
   }
 
   /**
    * @param actualDataType
    */
-  public void setActualDataType(ValueCompressionUtil.DataType[] actualDataType) {
+  public void setActualDataType(DataType[] actualDataType) {
     this.actualDataType = actualDataType;
   }
 
@@ -159,13 +164,33 @@ public class WriterCompressModel {
    * @return the aggType
    */
   public char[] getType() {
+    char[] ret = new char[type.length];
+    for (int i = 0; i < ret.length; i++) {
+      switch (type[i]) {
+        case SHORT:
+        case INT:
+        case LONG:
+          ret[i] = CarbonCommonConstants.BIG_INT_MEASURE;
+          break;
+        case DOUBLE:
+          ret[i] = CarbonCommonConstants.DOUBLE_MEASURE;
+          break;
+        case DECIMAL:
+          ret[i] = CarbonCommonConstants.BIG_DECIMAL_MEASURE;
+          break;
+      }
+    }
+    return ret;
+  }
+
+  public DataType[] getDataType() {
     return type;
   }
 
   /**
    * @param type the type to set
    */
-  public void setType(char[] type) {
+  public void setType(DataType[] type) {
     this.type = type;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java
index e517e41..7a36e66 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java
@@ -29,7 +29,7 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 
 public class CompressByteArray extends ValueCompressionHolder<byte[]> {
 
@@ -101,7 +101,7 @@ public class CompressByteArray extends ValueCompressionHolder<byte[]> {
   @Override
   public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_BIGDECIMAL, numberOfRows);
+        .getMeasureDataChunkStore(DataType.DECIMAL, numberOfRows);
     this.measureChunkStore.putData(data);
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
index ed50cd3..45e3b47 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 
 public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> {
 
@@ -68,7 +68,7 @@ public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+    compressedValue = super.compress(compressor, DataType.BYTE, value);
   }
 
   @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
@@ -103,7 +103,7 @@ public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> {
   @Override
   public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_BYTE, numberOfRows);
+        .getMeasureDataChunkStore(DataType.BYTE, numberOfRows);
     this.measureChunkStore.putData(data);
     if (maxValueObject instanceof Long) {
       this.maxValue = (long) maxValueObject;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
index 6550c46..6bd1947 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> {
 
@@ -70,7 +70,7 @@ public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+    compressedValue = super.compress(compressor, DataType.DOUBLE, value);
   }
 
   @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
@@ -106,7 +106,7 @@ public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> {
   @Override
   public void setValue(double[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_DOUBLE, numberOfRows);
+        .getMeasureDataChunkStore(DataType.DOUBLE, numberOfRows);
     this.measureChunkStore.putData(data);
     if (maxValueObject instanceof Long) {
       this.maxValue = (long) maxValueObject;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
index 512cf2c..60ddfea 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> {
   /**
@@ -72,7 +72,7 @@ public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+    compressedValue = super.compress(compressor, DataType.INT, value);
   }
 
   @Override public void setValueInBytes(byte[] value) {
@@ -102,7 +102,7 @@ public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> {
   @Override
   public void setValue(int[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore =
-        MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, numberOfRows);
+        MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.INT, numberOfRows);
     this.measureChunkStore.putData(data);
     if (maxValueObject instanceof Long) {
       this.maxValue = (long) maxValueObject;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
index ca91d44..159e741 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> {
   /**
@@ -57,7 +57,7 @@ public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+    compressedValue = super.compress(compressor, DataType.LONG, value);
   }
 
   @Override public void setValue(long[] value) {
@@ -102,7 +102,7 @@ public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> {
   @Override
   public void setValue(long[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_LONG, numberOfRows);
+        .getMeasureDataChunkStore(DataType.LONG, numberOfRows);
     this.measureChunkStore.putData(data);
     if (maxValueObject instanceof Long) {
       this.maxValue = (long) maxValueObject;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
index d44875a..1d36375 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> {
 
@@ -74,7 +74,7 @@ public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+    compressedValue = super.compress(compressor, DataType.SHORT, value);
   }
 
   @Override public void setValueInBytes(byte[] value) {
@@ -104,7 +104,7 @@ public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> {
   @Override
   public void setValue(short[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_SHORT, numberOfRows);
+        .getMeasureDataChunkStore(DataType.SHORT, numberOfRows);
     this.measureChunkStore.putData(data);
     if (maxValueObject instanceof Long) {
       this.maxValue = (long) maxValueObject;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
index 13f962f..1c46dea 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 
 public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
   /**
@@ -58,7 +58,7 @@ public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+    compressedValue = super.compress(compressor, DataType.BYTE, value);
   }
 
   @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
@@ -92,7 +92,7 @@ public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
   @Override
   public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_BYTE, numberOfRows);
+        .getMeasureDataChunkStore(DataType.BYTE, numberOfRows);
     this.measureChunkStore.putData(data);
     this.divisionFactory = Math.pow(10, decimalPlaces);
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java
index 15c7bb3..40312db 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNonDecimalDefault extends ValueCompressionHolder<double[]> {
   /**
@@ -56,7 +56,7 @@ public class CompressionNonDecimalDefault extends ValueCompressionHolder<double[
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+    compressedValue = super.compress(compressor, DataType.DOUBLE, value);
   }
 
   @Override public void setValue(double[] value) {
@@ -94,7 +94,7 @@ public class CompressionNonDecimalDefault extends ValueCompressionHolder<double[
   @Override
   public void setValue(double[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_DOUBLE, numberOfRows);
+        .getMeasureDataChunkStore(DataType.DOUBLE, numberOfRows);
     this.measureChunkStore.putData(data);
     this.divisionFactory = Math.pow(10, decimalPlaces);
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java
index 7e3606e..ada751e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNonDecimalInt extends ValueCompressionHolder<int[]> {
   /**
@@ -58,7 +58,7 @@ public class CompressionNonDecimalInt extends ValueCompressionHolder<int[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+    compressedValue = super.compress(compressor, DataType.INT, value);
   }
 
   @Override public void setValueInBytes(byte[] bytesArr) {
@@ -93,7 +93,7 @@ public class CompressionNonDecimalInt extends ValueCompressionHolder<int[]> {
   @Override
   public void setValue(int[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore =
-        MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, numberOfRows);
+        MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.INT, numberOfRows);
     this.measureChunkStore.putData(data);
     this.divisionFactory = Math.pow(10, decimalPlaces);
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java
index d810972..f0b2323 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNonDecimalLong extends ValueCompressionHolder<long[]> {
   /**
@@ -60,7 +60,7 @@ public class CompressionNonDecimalLong extends ValueCompressionHolder<long[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+    compressedValue = super.compress(compressor, DataType.LONG, value);
   }
 
   @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
@@ -95,7 +95,7 @@ public class CompressionNonDecimalLong extends ValueCompressionHolder<long[]> {
   @Override
   public void setValue(long[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_LONG, numberOfRows);
+        .getMeasureDataChunkStore(DataType.LONG, numberOfRows);
     this.measureChunkStore.putData(data);
     this.divisionFactory = Math.pow(10, decimalPlaces);
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
index 83e8a1a..c425706 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 
 public class CompressionNonDecimalMaxMinByte extends ValueCompressionHolder<byte[]> {
   /**
@@ -58,7 +58,7 @@ public class CompressionNonDecimalMaxMinByte extends ValueCompressionHolder<byte
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+    compressedValue = super.compress(compressor, DataType.BYTE, value);
   }
 
   @Override public void setValueInBytes(byte[] value) {
@@ -97,7 +97,7 @@ public class CompressionNonDecimalMaxMinByte extends ValueCompressionHolder<byte
   @Override
   public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_BYTE, numberOfRows);
+        .getMeasureDataChunkStore(DataType.BYTE, numberOfRows);
     this.measureChunkStore.putData(data);
     this.maxValue = BigDecimal.valueOf((double) maxValueObject);
     this.divisionFactor = Math.pow(10, decimalPlaces);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
index c408d9a..521328e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNonDecimalMaxMinDefault extends ValueCompressionHolder<double[]> {
   /**
@@ -62,7 +62,7 @@ public class CompressionNonDecimalMaxMinDefault extends ValueCompressionHolder<d
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+    compressedValue = super.compress(compressor, DataType.DOUBLE, value);
   }
 
   @Override public void setValueInBytes(byte[] value) {
@@ -99,7 +99,7 @@ public class CompressionNonDecimalMaxMinDefault extends ValueCompressionHolder<d
   @Override
   public void setValue(double[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_DOUBLE, numberOfRows);
+        .getMeasureDataChunkStore(DataType.DOUBLE, numberOfRows);
     this.measureChunkStore.putData(data);
     this.maxValue = BigDecimal.valueOf((double) maxValueObject);
     this.divisionFactor = Math.pow(10, decimalPlaces);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
index 40b1099..efd21d5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNonDecimalMaxMinInt extends ValueCompressionHolder<int[]> {
   /**
@@ -60,7 +60,7 @@ public class CompressionNonDecimalMaxMinInt extends ValueCompressionHolder<int[]
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+    compressedValue = super.compress(compressor, DataType.INT, value);
   }
 
   @Override public void setValueInBytes(byte[] value) {
@@ -97,7 +97,7 @@ public class CompressionNonDecimalMaxMinInt extends ValueCompressionHolder<int[]
   @Override
   public void setValue(int[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore =
-        MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, numberOfRows);
+        MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.INT, numberOfRows);
     this.measureChunkStore.putData(data);
     this.maxValue = BigDecimal.valueOf((double) maxValueObject);
     this.divisionFactor = Math.pow(10, decimalPlaces);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
index f8d8ed5..d2e2771 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNonDecimalMaxMinLong extends ValueCompressionHolder<long[]> {
   /**
@@ -68,7 +68,7 @@ public class CompressionNonDecimalMaxMinLong extends ValueCompressionHolder<long
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+    compressedValue = super.compress(compressor, DataType.LONG, value);
   }
 
   @Override public void setValueInBytes(byte[] value) {
@@ -98,7 +98,7 @@ public class CompressionNonDecimalMaxMinLong extends ValueCompressionHolder<long
   @Override
   public void setValue(long[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_LONG, numberOfRows);
+        .getMeasureDataChunkStore(DataType.LONG, numberOfRows);
     this.measureChunkStore.putData(data);
     this.maxValue = BigDecimal.valueOf((double) maxValueObject);
     this.divisionFactor = Math.pow(10, decimalPlaces);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
index 14648ba..241344a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNonDecimalMaxMinShort extends ValueCompressionHolder<short[]> {
   /**
@@ -66,7 +66,7 @@ public class CompressionNonDecimalMaxMinShort extends ValueCompressionHolder<sho
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+    compressedValue = super.compress(compressor, DataType.SHORT, value);
   }
 
   @Override public void setValueInBytes(byte[] value) {
@@ -96,7 +96,7 @@ public class CompressionNonDecimalMaxMinShort extends ValueCompressionHolder<sho
   @Override
   public void setValue(short[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_SHORT, numberOfRows);
+        .getMeasureDataChunkStore(DataType.SHORT, numberOfRows);
     this.measureChunkStore.putData(data);
     this.maxValue = BigDecimal.valueOf((double) maxValueObject);
     this.divisionFactor = Math.pow(10, decimalPlaces);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java
index 8536630..4737e10 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNonDecimalShort extends ValueCompressionHolder<short[]> {
   /**
@@ -58,7 +58,7 @@ public class CompressionNonDecimalShort extends ValueCompressionHolder<short[]>
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+    compressedValue = super.compress(compressor, DataType.SHORT, value);
   }
 
   @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
@@ -93,7 +93,7 @@ public class CompressionNonDecimalShort extends ValueCompressionHolder<short[]>
   @Override
   public void setValue(short[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_SHORT, numberOfRows);
+        .getMeasureDataChunkStore(DataType.SHORT, numberOfRows);
     this.measureChunkStore.putData(data);
     this.divisionFactory = Math.pow(10, decimalPlaces);
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java
index acd73d9..13af929 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 
 public class CompressionNoneByte extends ValueCompressionHolder<byte[]> {
   /**
@@ -72,7 +72,7 @@ public class CompressionNoneByte extends ValueCompressionHolder<byte[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+    compressedValue = super.compress(compressor, DataType.BYTE, value);
   }
 
   @Override public void setValueInBytes(byte[] value) {
@@ -98,7 +98,7 @@ public class CompressionNoneByte extends ValueCompressionHolder<byte[]> {
   @Override
   public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_BYTE, numberOfRows);
+        .getMeasureDataChunkStore(DataType.BYTE, numberOfRows);
     this.measureChunkStore.putData(data);
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java
index 8e02fd8..cca7927 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNoneDefault extends ValueCompressionHolder<double[]> {
   /**
@@ -69,7 +69,7 @@ public class CompressionNoneDefault extends ValueCompressionHolder<double[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+    compressedValue = super.compress(compressor, DataType.DOUBLE, value);
   }
 
   @Override public void setValueInBytes(byte[] value) {
@@ -97,7 +97,7 @@ public class CompressionNoneDefault extends ValueCompressionHolder<double[]> {
   @Override
   public void setValue(double[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_DOUBLE, numberOfRows);
+        .getMeasureDataChunkStore(DataType.DOUBLE, numberOfRows);
     this.measureChunkStore.putData(data);
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java
index f0c0311..f5bb813 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNoneInt extends ValueCompressionHolder<int[]> {
   /**
@@ -62,7 +62,7 @@ public class CompressionNoneInt extends ValueCompressionHolder<int[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+    compressedValue = super.compress(compressor, DataType.INT, value);
   }
 
   @Override
@@ -96,7 +96,7 @@ public class CompressionNoneInt extends ValueCompressionHolder<int[]> {
   @Override
   public void setValue(int[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore =
-        MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, numberOfRows);
+        MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.INT, numberOfRows);
     this.measureChunkStore.putData(data);
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
index ee72c8a..99f24ac 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNoneLong extends ValueCompressionHolder<long[]> {
   /**
@@ -62,7 +62,7 @@ public class CompressionNoneLong extends ValueCompressionHolder<long[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+    compressedValue = super.compress(compressor, DataType.LONG, value);
   }
 
   @Override
@@ -96,7 +96,7 @@ public class CompressionNoneLong extends ValueCompressionHolder<long[]> {
   @Override
   public void setValue(long[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_LONG, numberOfRows);
+        .getMeasureDataChunkStore(DataType.LONG, numberOfRows);
     this.measureChunkStore.putData(data);
 
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
index d4289ab..664f9e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
 import org.apache.carbondata.core.datastore.compression.Compressor;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 public class CompressionNoneShort extends ValueCompressionHolder<short[]> {
   /**
@@ -71,7 +71,7 @@ public class CompressionNoneShort extends ValueCompressionHolder<short[]> {
   }
 
   @Override public void compress() {
-    compressedValue = super.compress(compressor, DataType.DATA_SHORT, shortValue);
+    compressedValue = super.compress(compressor, DataType.SHORT, shortValue);
   }
 
   @Override public void setValueInBytes(byte[] value) {
@@ -98,7 +98,7 @@ public class CompressionNoneShort extends ValueCompressionHolder<short[]> {
   @Override
   public void setValue(short[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
     this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
-        .getMeasureDataChunkStore(DataType.DATA_SHORT, numberOfRows);
+        .getMeasureDataChunkStore(DataType.SHORT, numberOfRows);
     this.measureChunkStore.putData(data);
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java
index fb21d95..8d3cc0d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java
@@ -34,137 +34,57 @@ public class CarbonWriteDataHolder {
   private byte[][] byteValues;
 
   /**
-   * byteValues for no dictionary.
-   */
-  private byte[][][] byteValuesForNonDictionary;
-
-  /**
-   * byteValues
-   */
-  private byte[][][] columnByteValues;
-
-  /**
    * size
    */
   private int size;
 
   /**
-   * totalSize
+   * total size of the data in bytes added
    */
   private int totalSize;
 
-  /**
-   * Method to initialise double array
-   *
-   * @param size
-   */
-  public void initialiseDoubleValues(int size) {
-    if (size < 1) {
-      throw new IllegalArgumentException("Invalid array size");
-    }
-    doubleValues = new double[size];
-  }
-
   public void reset() {
     size = 0;
     totalSize = 0;
   }
 
   /**
-   * Method to initialise double array
-   *
-   * @param size
-   */
-  public void initialiseByteArrayValues(int size) {
-    if (size < 1) {
-      throw new IllegalArgumentException("Invalid array size");
-    }
-
-    byteValues = new byte[size][];
-    columnByteValues = new byte[size][][];
-  }
-
-  /**
-   * Method to initialise byte array
-   *
-   * @param size
+   * set long data type columnar data
+   * @param values
    */
-  public void initialiseByteArrayValuesForKey(int size) {
-    if (size < 1) {
-      throw new IllegalArgumentException("Invalid array size");
-    }
-
-    byteValues = new byte[size][];
-  }
-
-  public void initialiseByteArrayValuesForNonDictionary(int size) {
-    if (size < 1) {
-      throw new IllegalArgumentException("Invalid array size");
+  public void setWritableLongPage(long[] values) {
+    if (values != null) {
+      longValues = values;
+      size += values.length;
+      totalSize += values.length;
     }
-
-    byteValuesForNonDictionary = new byte[size][][];
   }
 
   /**
-   * Method to initialise long array
-   *
-   * @param size
+   * set double data type columnar data
+   * @param values
    */
-  public void initialiseLongValues(int size) {
-    if (size < 1) {
-      throw new IllegalArgumentException("Invalid array size");
+  public void setWritableDoublePage(double[] values) {
+    if (values != null) {
+      doubleValues = values;
+      size += values.length;
+      totalSize += values.length;
     }
-    longValues = new long[size];
-  }
-
-  /**
-   * set double value by index
-   *
-   * @param index
-   * @param value
-   */
-  public void setWritableDoubleValueByIndex(int index, Object value) {
-    doubleValues[index] = (Double) value;
-    size++;
   }
 
   /**
-   * set double value by index
-   *
-   * @param index
-   * @param value
-   */
-  public void setWritableLongValueByIndex(int index, Object value) {
-    longValues[index] = (Long) value;
-    size++;
-  }
-
-  /**
-   * set byte array value by index
-   *
-   * @param index
-   * @param value
+   * set decimal data type columnar data
+   * @param values
    */
-  public void setWritableByteArrayValueByIndex(int index, byte[] value) {
-    byteValues[index] = value;
-    size++;
-    if (null != value) totalSize += value.length;
-  }
-
-  public void setWritableNonDictByteArrayValueByIndex(int index, byte[][] value) {
-    byteValuesForNonDictionary[index] = value;
-    size++;
-    if (null != value) totalSize += value.length;
-  }
-
-  /**
-   * set byte array value by index
-   */
-  public void setWritableByteArrayValueByIndex(int index, int mdKeyIndex, Object[] columnData) {
-    int l = 0;
-    columnByteValues[index] = new byte[columnData.length - (mdKeyIndex + 1)][];
-    for (int i = mdKeyIndex + 1; i < columnData.length; i++) {
-      columnByteValues[index][l++] = (byte[]) columnData[i];
+  public void setWritableDecimalPage(byte[][] values) {
+    if (values != null) {
+      byteValues = values;
+      size += values.length;
+      for (int i = 0; i < values.length; i++) {
+        if (values[i] != null) {
+          totalSize += values[i].length;
+        }
+      }
     }
   }
 
@@ -187,30 +107,14 @@ public class CarbonWriteDataHolder {
     byte[] temp = new byte[totalSize];
     int startIndexToCopy = 0;
     for (int i = 0; i < size; i++) {
-      System.arraycopy(byteValues[i], 0, temp, startIndexToCopy, byteValues[i].length);
-      startIndexToCopy += byteValues[i].length;
+      if (byteValues[i] != null) {
+        System.arraycopy(byteValues[i], 0, temp, startIndexToCopy, byteValues[i].length);
+        startIndexToCopy += byteValues[i].length;
+      }
     }
     return temp;
   }
 
-  public byte[][] getByteArrayValues() {
-    if (size < byteValues.length) {
-      byte[][] temp = new byte[size][];
-      System.arraycopy(byteValues, 0, temp, 0, size);
-      byteValues = temp;
-    }
-    return byteValues;
-  }
-
-  public byte[][][] getNonDictByteArrayValues() {
-    if (size < byteValuesForNonDictionary.length) {
-      byte[][][] temp = new byte[size][][];
-      System.arraycopy(byteValuesForNonDictionary, 0, temp, 0, size);
-      byteValuesForNonDictionary = temp;
-    }
-    return byteValuesForNonDictionary;
-  }
-
   /**
    * Get Writable Double Values
    *

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataStore.java
deleted file mode 100644
index d3d67fd..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataStore.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.impl.data.compressed;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public class HeavyCompressedDoubleArrayDataStore {
-
-  // this method first invokes encoding routine to encode the data chunk,
-  // followed by invoking compression routine for preparing the data chunk for writing.
-  public static byte[][] encodeMeasureDataArray(
-      WriterCompressModel compressionModel,
-      CarbonWriteDataHolder[] dataHolder) {
-    char[] type = compressionModel.getType();
-    ValueCompressionHolder[] values =
-        new ValueCompressionHolder[compressionModel.getValueCompressionHolder().length];
-    byte[][] returnValue = new byte[values.length][];
-    for (int i = 0; i < compressionModel.getValueCompressionHolder().length; i++) {
-      values[i] = compressionModel.getValueCompressionHolder()[i];
-      if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE
-          && type[i] != CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
-        // first perform encoding of the data chunk
-        values[i].setValue(
-            ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i])
-                .getCompressedValues(compressionModel.getCompressionFinders()[i], dataHolder[i],
-                    compressionModel.getMaxValue()[i],
-                    compressionModel.getMantissa()[i]));
-      } else {
-        values[i].setValue(dataHolder[i].getWritableByteArrayValues());
-      }
-      values[i].compress();
-      returnValue[i] = values[i].getCompressedData();
-    }
-
-    return returnValue;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d9ecfb50/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
new file mode 100644
index 0000000..25a813c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page;
+
+import org.apache.carbondata.core.datastore.page.statistics.PageStatistics;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+public class ColumnPage {
+
+  protected final DataType dataType;
+  protected final int pageSize;
+  protected PageStatistics stats;
+
+  protected ColumnPage(DataType dataType, int pageSize) {
+    this.dataType = dataType;
+    this.pageSize = pageSize;
+    this.stats = new PageStatistics(dataType);
+  }
+
+  protected void updateStatistics(Object value) {
+    stats.update(value);
+  }
+
+  public PageStatistics getStatistics() {
+    return stats;
+  }
+}


Mime
View raw message