carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [5/7] carbondata git commit: [CARBONDATA-1098] Change page statistics use exact type and use column page in writer
Date Fri, 28 Jul 2017 06:10:05 GMT
[CARBONDATA-1098] Change page statistics use exact type and use column page in writer

This PR changes writer in data load:

make statistics collection use exact data type in schema instead of generic type
change consumer and writer to use EncodedTablePage instead of NodeHolder. EncodedTablePage is the output of TablePage.encode

This closes#1102


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bc3e6843
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bc3e6843
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bc3e6843

Branch: refs/heads/master
Commit: bc3e6843ee83370b6b20e5c9eef92f10667edbae
Parents: c504dd2
Author: jackylk <jacky.likun@huawei.com>
Authored: Tue Jul 4 08:12:13 2017 +0800
Committer: Raghunandan S <carbondatacontributions@gmail.com>
Committed: Fri Jul 28 01:05:59 2017 +0800

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |   2 -
 .../carbondata/core/datastore/TableSpec.java    |   1 +
 .../AbstractMeasureChunkReaderV2V3Format.java   |  25 --
 ...CompressedMeasureChunkFileBasedReaderV2.java |  14 +
 ...CompressedMeasureChunkFileBasedReaderV3.java |  15 +
 .../chunk/store/MeasureChunkStoreFactory.java   | 102 -----
 .../chunk/store/MeasureDataChunkStore.java      |  87 ----
 .../safe/SafeAbstractMeasureDataChunkStore.java | 113 -----
 .../safe/SafeBigDecimalMeasureChunkStore.java   |  98 -----
 .../impl/safe/SafeByteMeasureChunkStore.java    |  56 ---
 .../impl/safe/SafeDoubleMeasureChunkStore.java  |  55 ---
 .../impl/safe/SafeIntMeasureChunkStore.java     |  55 ---
 .../impl/safe/SafeLongMeasureChunkStore.java    |  56 ---
 .../impl/safe/SafeShortMeasureChunkStore.java   |  57 ---
 .../UnsafeAbstractMeasureDataChunkStore.java    | 129 ------
 .../UnsafeBigDecimalMeasureChunkStore.java      | 139 ------
 .../unsafe/UnsafeByteMeasureChunkStore.java     |  59 ---
 .../unsafe/UnsafeDoubleMeasureChunkStore.java   |  61 ---
 .../impl/unsafe/UnsafeIntMeasureChunkStore.java |  61 ---
 .../unsafe/UnsafeLongMeasureChunkStore.java     |  60 ---
 .../unsafe/UnsafeShortMeasureChunkStore.java    |  60 ---
 .../columnar/BlockIndexerStorageForInt.java     |  27 ++
 ...kIndexerStorageForNoInvertedIndexForInt.java |  15 +
 ...ndexerStorageForNoInvertedIndexForShort.java |  15 +
 .../columnar/BlockIndexerStorageForShort.java   |  27 ++
 .../core/datastore/columnar/IndexStorage.java   |   7 +
 .../core/datastore/page/ColumnPage.java         |  36 +-
 .../core/datastore/page/EncodedTablePage.java   | 154 +++++++
 .../core/datastore/page/LazyColumnPage.java     |   8 +-
 .../datastore/page/compression/Compression.java |  23 -
 .../page/encoding/AdaptiveCompressionCodec.java |   8 +-
 .../page/encoding/AdaptiveIntegerCodec.java     | 224 ----------
 .../page/encoding/AdaptiveIntegralCodec.java    | 219 ++++++++++
 .../page/encoding/ColumnPageCodec.java          |   7 +-
 .../page/encoding/DefaultEncodingStrategy.java  |  79 +++-
 .../page/encoding/DeltaIntegerCodec.java        | 255 -----------
 .../page/encoding/DeltaIntegralCodec.java       | 257 +++++++++++
 .../page/encoding/DirectCompressCodec.java      |  92 +++-
 .../page/encoding/EncodedColumnPage.java        |  78 ++++
 .../datastore/page/encoding/EncodedData.java    |  32 --
 .../page/encoding/EncodedDimensionPage.java     | 141 ++++++
 .../page/encoding/EncodedMeasurePage.java       |  87 ++++
 .../page/encoding/EncodingStrategy.java         |  58 ++-
 .../core/datastore/page/key/TablePageKey.java   | 212 +++++++++
 .../statistics/ColumnPageStatsCollector.java    |  33 ++
 .../page/statistics/ColumnPageStatsVO.java      | 186 --------
 .../page/statistics/MeasurePageStatsVO.java     |  82 ----
 .../statistics/PrimitivePageStatsCollector.java | 294 +++++++++++++
 .../page/statistics/SimpleStatsResult.java      |  35 ++
 .../page/statistics/TablePageStatistics.java    | 130 ++++++
 .../statistics/VarLengthPageStatsCollector.java | 107 +++++
 .../core/metadata/BlockletInfoColumnar.java     |  34 +-
 .../core/metadata/CodecMetaFactory.java         |  90 ++++
 .../core/metadata/ColumnPageCodecMeta.java      | 270 ++++++++++++
 .../core/metadata/ValueEncoderMeta.java         |   2 +-
 .../impl/AbstractScannedResultCollector.java    |   6 -
 .../core/util/CarbonMetadataUtil.java           | 433 ++++---------------
 .../apache/carbondata/core/util/CarbonUtil.java |  99 +++--
 .../carbondata/core/util/DataTypeUtil.java      |  19 -
 .../apache/carbondata/core/util/NodeHolder.java |  37 +-
 .../carbondata/core/util/NonDictionaryUtil.java | 167 +++++++
 .../core/util/CarbonMetadataUtilTest.java       | 106 +++--
 .../carbondata/core/util/DataTypeUtilTest.java  |   8 -
 .../core/writer/CarbonFooterWriterTest.java     | 115 +++--
 format/src/main/thrift/carbondata.thrift        |   2 +-
 .../apache/carbondata/hive/CarbonHiveSerDe.java |   4 +-
 .../apache/carbondata/hive/TestCarbonSerde.java |   2 +-
 .../src/test/resources/complexTypeDecimal.csv   |  16 +-
 .../resources/complexTypeDecimalNestedHive.csv  |   2 +-
 .../columnar/ColGroupBlockStorage.java          | 104 -----
 .../newflow/sort/SortStepRowUtil.java           |   6 +-
 .../steps/DataWriterProcessorStepImpl.java      |   2 +-
 .../sortdata/IntermediateFileMerger.java        |   2 +-
 .../sortandgroupby/sortdata/RowComparator.java  |   2 +-
 .../sortdata/RowComparatorForNormalDims.java    |   2 +-
 .../sortdata/SortTempFileChunkHolder.java       |   3 +-
 .../UnCompressedTempSortFileWriter.java         |   2 +-
 .../store/CarbonFactDataHandlerColumnar.java    |  73 ++--
 .../carbondata/processing/store/TablePage.java  | 226 ++++++++--
 .../processing/store/TablePageEncoder.java      | 203 ---------
 .../processing/store/TablePageKey.java          | 138 ------
 .../processing/store/TablePageStatistics.java   | 142 ------
 .../store/writer/AbstractFactDataWriter.java    |  36 +-
 .../store/writer/CarbonFactDataWriter.java      |  22 +-
 .../writer/v1/CarbonFactDataWriterImplV1.java   | 129 +++---
 .../writer/v2/CarbonFactDataWriterImplV2.java   |  58 +--
 .../writer/v3/CarbonFactDataWriterImplV3.java   | 390 ++++-------------
 .../store/writer/v3/DataWriterHolder.java       |  24 +-
 .../processing/util/NonDictionaryUtil.java      | 167 -------
 .../carbondata/processing/StoreCreator.java     |   4 +-
 90 files changed, 3382 insertions(+), 3998 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 3acdba9..f2e59ab 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -693,12 +693,10 @@ public final class CarbonCommonConstants {
    * BIG_DECIMAL_MEASURE
    */
   public static final char BIG_DECIMAL_MEASURE = 'b';
-
   /**
    * BIG_INT_MEASURE
    */
   public static final char BIG_INT_MEASURE = 'd';
-
   /**
    * CARBON_PREFETCH_BUFFERSIZE
    */

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
index 650c2a6..87c4934 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
@@ -178,6 +178,7 @@ public class TableSpec {
     public int getNumExpandedDimensions() {
       return numDimensionExpanded;
     }
+
   }
 
   public class MeasureSpec {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
index c35cefb..dd61826 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
@@ -24,14 +24,8 @@ import java.util.List;
 import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
 import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.datastore.page.encoding.ColumnPageCodec;
-import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.ValueEncoderMeta;
 import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
 import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.format.DataChunk2;
 
 /**
  * Abstract class for V2, V3 format measure column reader
@@ -129,23 +123,4 @@ public abstract class AbstractMeasureChunkReaderV2V3Format extends AbstractMeasu
   protected abstract MeasureRawColumnChunk[] readRawMeasureChunksInGroup(FileHolder fileReader,
       int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException;
 
-
-  protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk,
-      DataChunk2 measureColumnChunk, int copyPoint) throws MemoryException {
-    // for measure, it should have only one ValueEncoderMeta
-    List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta();
-    assert (encoder_meta.size() > 0);
-    byte[] encodedMeta = encoder_meta.get(0).array();
-    ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV3(encodedMeta);
-    int scale = -1;
-    int precision = -1;
-    if (encoder_meta.size() > 1) {
-      ByteBuffer decimalInfo = encoder_meta.get(1);
-      scale = decimalInfo.getInt();
-      precision = decimalInfo.getInt();
-    }
-    ColumnPageCodec codec = strategy.createCodec(meta, scale, precision);
-    byte[] rawData = measureRawColumnChunk.getRawData().array();
-    return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);
-  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index d90c7fe..09f367a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -24,7 +24,9 @@ import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
 import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format;
 import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.datastore.page.encoding.ColumnPageCodec;
 import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.ValueEncoderMeta;
 import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
 import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.format.DataChunk2;
@@ -125,4 +127,16 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
     datChunk.setNullValueIndexHolder(getPresenceMeta(measureColumnChunk.presence));
     return datChunk;
   }
+
+  protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk,
+      DataChunk2 measureColumnChunk, int copyPoint) throws MemoryException, IOException {
+    // for measure, it should have only one ValueEncoderMeta
+    assert (measureColumnChunk.getEncoder_meta().size() == 1);
+    byte[] encodedMeta = measureColumnChunk.getEncoder_meta().get(0).array();
+
+    ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV3(encodedMeta);
+    ColumnPageCodec codec = strategy.createCodec(meta);
+    byte[] rawData = measureRawColumnChunk.getRawData().array();
+    return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
index 2ca7193..492d46a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
@@ -24,7 +24,9 @@ import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
 import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format;
 import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.datastore.page.encoding.ColumnPageCodec;
 import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.metadata.ColumnPageCodecMeta;
 import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
 import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.format.DataChunk2;
@@ -222,4 +224,17 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
     return datChunk;
   }
 
+  protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk,
+      DataChunk2 measureColumnChunk, int copyPoint) throws MemoryException {
+    // for measure, it should have only one ValueEncoderMeta
+    assert (measureColumnChunk.getEncoder_meta().size() == 1);
+    byte[] encodedMeta = measureColumnChunk.getEncoder_meta().get(0).array();
+
+    ColumnPageCodecMeta meta = new ColumnPageCodecMeta();
+    meta.deserialize(encodedMeta);
+    ColumnPageCodec codec = strategy.createCodec(meta);
+    byte[] rawData = measureRawColumnChunk.getRawData().array();
+    return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
deleted file mode 100644
index 12bfea9..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeBigDecimalMeasureChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeByteMeasureChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeDoubleMeasureChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeIntMeasureChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeLongMeasureChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeShortMeasureChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeBigDecimalMeasureChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeByteMeasureChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeDoubleMeasureChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeIntMeasureChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeLongMeasureChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeShortMeasureChunkStore;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-/**
- * Factory class for getting the measure store type
- */
-public class MeasureChunkStoreFactory {
-
-  /**
-   * instance type
-   */
-  public static final MeasureChunkStoreFactory INSTANCE = new MeasureChunkStoreFactory();
-
-  /**
-   * is unsafe
-   */
-  private static final boolean isUnsafe;
-
-  static {
-    isUnsafe = Boolean.parseBoolean(CarbonProperties.getInstance()
-        .getProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION,
-            CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE));
-  }
-
-  private MeasureChunkStoreFactory() {
-  }
-
-  /**
-   * Below method will be used to get the measure data chunk store based on data type
-   *
-   * @param dataType     data type
-   * @param numberOfRows number of rows
-   * @return measure chunk store
-   */
-  public MeasureDataChunkStore getMeasureDataChunkStore(DataType dataType, int numberOfRows) {
-    if (!isUnsafe) {
-      switch (dataType) {
-        case BYTE:
-          return new SafeByteMeasureChunkStore(numberOfRows);
-        case SHORT:
-          return new SafeShortMeasureChunkStore(numberOfRows);
-        case INT:
-          return new SafeIntMeasureChunkStore(numberOfRows);
-        case LONG:
-          return new SafeLongMeasureChunkStore(numberOfRows);
-        case DECIMAL:
-          return new SafeBigDecimalMeasureChunkStore(numberOfRows);
-        case DOUBLE:
-        default:
-          return new SafeDoubleMeasureChunkStore(numberOfRows);
-      }
-    } else {
-      switch (dataType) {
-        case BYTE:
-          return new UnsafeByteMeasureChunkStore(numberOfRows);
-        case SHORT:
-          return new UnsafeShortMeasureChunkStore(numberOfRows);
-        case INT:
-          return new UnsafeIntMeasureChunkStore(numberOfRows);
-        case LONG:
-          return new UnsafeLongMeasureChunkStore(numberOfRows);
-        case DECIMAL:
-          return new UnsafeBigDecimalMeasureChunkStore(numberOfRows);
-        case DOUBLE:
-        default:
-          return new UnsafeDoubleMeasureChunkStore(numberOfRows);
-      }
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureDataChunkStore.java
deleted file mode 100644
index 05a050d..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureDataChunkStore.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store;
-
-import java.math.BigDecimal;
-
-/**
- * Responsibility is store the measure data in memory,
- * memory can be on heap or offheap based on the user configuration
- */
-public interface MeasureDataChunkStore<T> {
-
-  /**
-   * Below method will be used to put the data to memory
-   *
-   * @param data
-   */
-  void putData(T data);
-
-  /**
-   * to get byte value
-   *
-   * @param index
-   * @return byte value based on index
-   */
-  byte getByte(int index);
-
-  /**
-   * to get the short value
-   *
-   * @param index
-   * @return short value based on index
-   */
-  short getShort(int index);
-
-  /**
-   * to get the int value
-   *
-   * @param index
-   * @return int value based on index
-   */
-  int getInt(int index);
-
-  /**
-   * to get the long value
-   *
-   * @param index
-   * @return long value based on index
-   */
-  long getLong(int index);
-
-  /**
-   * to get the double value
-   *
-   * @param index
-   * @return double value based on index
-   */
-  double getDouble(int index);
-
-  /**
-   * To get the bigdecimal value
-   *
-   * @param index
-   * @return bigdecimal value based on index
-   */
-  BigDecimal getBigDecimal(int index);
-
-  /**
-   * To free the occupied memory
-   */
-  void freeMemory();
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java
deleted file mode 100644
index dc9fb89..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.safe;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-
-/**
- * Responsibility is store the measure data in memory,
- */
-public abstract class SafeAbstractMeasureDataChunkStore<T> implements
-    MeasureDataChunkStore<T> {
-
-  /**
-   * number of rows
-   */
-  protected int numberOfRows;
-
-  public SafeAbstractMeasureDataChunkStore(int numberOfRows) {
-    this.numberOfRows = numberOfRows;
-  }
-
-  /**
-   * to get the byte value
-   *
-   * @param index
-   * @return byte value based on index
-   */
-  @Override
-  public byte getByte(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * to get the short value
-   *
-   * @param index
-   * @return short value based on index
-   */
-  @Override
-  public short getShort(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * to get the int value
-   *
-   * @param index
-   * @return int value based on index
-   */
-  @Override
-  public int getInt(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * to get the long value
-   *
-   * @param index
-   * @return long value based on index
-   */
-  @Override
-  public long getLong(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * to get the double value
-   *
-   * @param index
-   * @return double value based on index
-   */
-  @Override
-  public double getDouble(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * To get the bigdecimal value
-   *
-   * @param index
-   * @return bigdecimal value based on index
-   */
-  @Override
-  public BigDecimal getBigDecimal(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * To free the occupied memory
-   */
-  @Override
-  public void freeMemory() {
-    // do nothing as GC will take care of freeing the memory
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeBigDecimalMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeBigDecimalMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeBigDecimalMeasureChunkStore.java
deleted file mode 100644
index 03f59fc..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeBigDecimalMeasureChunkStore.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk.store.impl.safe;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.DataTypeUtil;
-
-/**
- * Responsibility is store the big decimal measure data in memory,
- */
-public class SafeBigDecimalMeasureChunkStore extends SafeAbstractMeasureDataChunkStore<byte[]> {
-
-  /**
-   * data chunk
-   */
-  private byte[] dataChunk;
-
-  /**
-   * offset of actual data
-   */
-  private int[] dataOffsets;
-
-  public SafeBigDecimalMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-    this.dataOffsets = new int[numberOfRows];
-  }
-
-  @Override public void putData(byte[] data) {
-    this.dataChunk = data;
-    // As data is of variable length and data format is
-    // <length in int><data><length in int><data>
-    // we need to store offset of each data so data can be accessed directly
-    // for example:
-    //data = {0,0,0,0,5,1,2,3,4,5,0,0,0,0,6,0,1,2,3,4,5,0,0,0,0,2,8,9}
-    //so value stored in offset will be position of actual data
-    // [5,14,24]
-    // to store this value we need to get the actual data length + 4 bytes used for storing the
-    // length
-
-    // start position will be used to store the current data position
-    int startOffset = 0;
-    // as first position will be start from 4 byte as data is stored first in the memory block
-    // we need to skip first two bytes this is because first two bytes will be length of the data
-    // which we have to skip
-    dataOffsets[0] = CarbonCommonConstants.INT_SIZE_IN_BYTE;
-    // creating a byte buffer which will wrap the length of the row
-    ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
-    for (int i = 1; i < numberOfRows; i++) {
-      buffer.put(data, startOffset, CarbonCommonConstants.INT_SIZE_IN_BYTE);
-      buffer.flip();
-      // so current row position will be
-      // previous row length + 4 bytes used for storing previous row data
-      startOffset += buffer.getInt() + CarbonCommonConstants.INT_SIZE_IN_BYTE;
-      // as same byte buffer is used to avoid creating many byte buffer for each row
-      // we need to clear the byte buffer
-      buffer.clear();
-      dataOffsets[i] = startOffset + CarbonCommonConstants.INT_SIZE_IN_BYTE;
-    }
-  }
-
-  /**
-   * to get the byte value
-   *
-   * @param index
-   * @return byte value based on index
-   */
-  @Override public BigDecimal getBigDecimal(int index) {
-    int currentDataOffset = dataOffsets[index];
-    int length = 0;
-    // calculating the length of data
-    if (index < numberOfRows - 1) {
-      length = (int) (dataOffsets[index + 1] - (currentDataOffset
-          + CarbonCommonConstants.INT_SIZE_IN_BYTE));
-    } else {
-      // for last record
-      length = (int) (this.dataChunk.length - currentDataOffset);
-    }
-    return DataTypeUtil.byteToBigDecimal(dataChunk, currentDataOffset, length);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java
deleted file mode 100644
index 74565a8..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.safe;
-
-/**
- * Responsible for storing Byte array data to memory.
- */
-public class SafeByteMeasureChunkStore extends
-    SafeAbstractMeasureDataChunkStore<byte[]> {
-
-  /**
-   * data
-   */
-  private byte[] data;
-
-  public SafeByteMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-  }
-
-  /**
-   * Below method will be used to put byte array data to memory
-   *
-   * @param data
-   */
-  @Override
-  public void putData(byte[] data) {
-    this.data = data;
-  }
-
-  /**
-   * to get the byte value
-   *
-   * @param index
-   * @return byte value based on index
-   */
-  @Override
-  public byte getByte(int index) {
-    return this.data[index];
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java
deleted file mode 100644
index 39d7ee7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.safe;
-
-/**
- * Below class will be used to store the measure values of double data type
- */
-public class SafeDoubleMeasureChunkStore extends
-    SafeAbstractMeasureDataChunkStore<double[]> {
-
-  /**
-   * data
-   */
-  private double[] data;
-
-  public SafeDoubleMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-  }
-
-  /**
-   * Below method will be used to store double array data
-   *
-   * @param data
-   */
-  @Override
-  public void putData(double[] data) {
-    this.data = data;
-  }
-
-  /**
-   * to get the double value
-   *
-   * @param index
-   * @return double value based on index
-   */
-  @Override
-  public double getDouble(int index) {
-    return this.data[index];
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java
deleted file mode 100644
index be7c8fd..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.safe;
-
-/**
- * Responsible for storing int array data to memory.
- */
-public class SafeIntMeasureChunkStore extends
-    SafeAbstractMeasureDataChunkStore<int[]> {
-
-  /**
-   * data
-   */
-  private int[] data;
-
-  public SafeIntMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-  }
-
-  /**
-   * Below method will be used to put int array data to memory
-   *
-   * @param data
-   */
-  @Override
-  public void putData(int[] data) {
-    this.data = data;
-  }
-
-  /**
-   * to get the int value
-   *
-   * @param index
-   * @return int value based on index
-   */
-  @Override
-  public int getInt(int index) {
-    return this.data[index];
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java
deleted file mode 100644
index 31c12a0..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.safe;
-
-/**
- * Below class will be used to store the measure values of long data type
- *
- */
-public class SafeLongMeasureChunkStore extends
-    SafeAbstractMeasureDataChunkStore<long[]> {
-
-  /**
-   * data
-   */
-  private long[] data;
-
-  public SafeLongMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-  }
-
-  /**
-   * Below method will be used to store long array data
-   *
-   * @param data
-   */
-  @Override
-  public void putData(long[] data) {
-    this.data = data;
-  }
-
-  /**
-   * to get the long value
-   *
-   * @param index
-   * @return long value based on index
-   */
-  @Override
-  public long getLong(int index) {
-    return this.data[index];
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java
deleted file mode 100644
index a9a6a25..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.safe;
-
-/**
- * Below class will be used to store the measure values of short data type
- *
- */
-public class SafeShortMeasureChunkStore extends
-    SafeAbstractMeasureDataChunkStore<short[]> {
-
-  /**
-   * data
-   */
-  private short[] data;
-
-  public SafeShortMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-  }
-
-  /**
-   * Below method will be used to put short array data
-   *
-   * @param data
-   */
-  @Override
-  public void putData(short[] data) {
-    this.data = data;
-  }
-
-  /**
-   * to get the short value
-   *
-   * @param index
-   * @return shot value based on index
-   */
-  @Override
-  public short getShort(int index) {
-    return data[index];
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java
deleted file mode 100644
index 545b864..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
-import org.apache.carbondata.core.memory.MemoryBlock;
-
-/**
- * Responsibility is store the measure data in memory, memory can be on heap or
- * offheap based on the user configuration using unsafe interface
- */
-public abstract class UnsafeAbstractMeasureDataChunkStore<T> implements MeasureDataChunkStore<T> {
-
-  /**
-   * memory block
-   */
-  protected MemoryBlock dataPageMemoryBlock;
-
-  /**
-   * number of rows
-   */
-  protected int numberOfRows;
-
-  /**
-   * to check memory is released or not
-   */
-  protected boolean isMemoryReleased;
-
-  /**
-   * to check memory is occupied or not
-   */
-  protected boolean isMemoryOccupied;
-
-  public UnsafeAbstractMeasureDataChunkStore(int numberOfRows) {
-    this.numberOfRows = numberOfRows;
-  }
-
-  /**
-   * to get the byte value
-   *
-   * @param index
-   * @return byte value based on index
-   */
-  @Override public byte getByte(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * to get the short value
-   *
-   * @param index
-   * @return short value based on index
-   */
-  @Override public short getShort(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * to get the int value
-   *
-   * @param index
-   * @return int value based on index
-   */
-  @Override public int getInt(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * to get the long value
-   *
-   * @param index
-   * @return long value based on index
-   */
-  @Override public long getLong(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * to get the double value
-   *
-   * @param index
-   * @return double value based on index
-   */
-  @Override public double getDouble(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * To get the bigdecimal value
-   *
-   * @param index
-   * @return bigdecimal value based on index
-   */
-  @Override public BigDecimal getBigDecimal(int index) {
-    throw new UnsupportedOperationException("Operation not supported");
-  }
-
-  /**
-   * To free the occupied memory
-   */
-  @Override public void freeMemory() {
-    if (isMemoryReleased) {
-      return;
-    }
-    MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().free(dataPageMemoryBlock);
-    isMemoryReleased = true;
-    this.dataPageMemoryBlock = null;
-    this.isMemoryOccupied = false;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeBigDecimalMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeBigDecimalMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeBigDecimalMeasureChunkStore.java
deleted file mode 100644
index 4082689..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeBigDecimalMeasureChunkStore.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.memory.CarbonUnsafe;
-import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
-import org.apache.carbondata.core.util.DataTypeUtil;
-
-/**
- * Responsible for storing big decimal array data to memory. memory can be on heap or
- * offheap based on the user configuration using unsafe interface
- */
-public class UnsafeBigDecimalMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<byte[]> {
-
-  /**
-   * start position of data offsets
-   */
-  private long offsetStartPosition;
-
-  public UnsafeBigDecimalMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-  }
-
-  @Override public void putData(byte[] data) {
-    assert (!this.isMemoryOccupied);
-    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
-        .allocate(data.length + (numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE));
-    this.offsetStartPosition = data.length;
-    // copy the data to memory
-    CarbonUnsafe.unsafe
-        .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
-            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
-    // As data is of variable length and data format is
-    // <length in short><data><length in short><data>
-    // we need to store offset of each data so data can be accessed directly
-    // for example:
-    //data = {0,0,0,0,5,1,2,3,4,5,0,0,0,0,6,0,1,2,3,4,5,0,0,0,0,2,8,9}
-    //so value stored in offset will be position of actual data
-    // [5,14,24]
-    // to store this value we need to get the actual data length + 4 bytes used for storing the
-    // length
-    // start position will be used to store the current data position
-    int startOffset = 0;
-    // position from where offsets will start
-    long pointerOffsets = this.offsetStartPosition;
-    // as first position will be start from 4 byte as data is stored first in the memory block
-    // we need to skip first two bytes this is because first two bytes will be length of the data
-    // which we have to skip
-    CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
-        dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
-        CarbonCommonConstants.INT_SIZE_IN_BYTE);
-    // incrementing the pointers as first value is already filled and as we are storing as int
-    // we need to increment the 4 bytes to set the position of the next value to set
-    pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE;
-    // creating a byte buffer which will wrap the length of the row
-    // using byte buffer as unsafe will return bytes in little-endian encoding
-    ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
-    // store length of data
-    byte[] length = new byte[CarbonCommonConstants.INT_SIZE_IN_BYTE];
-    // as first offset is already stored, we need to start from the 2nd row in data array
-    for (int i = 1; i < numberOfRows; i++) {
-      // first copy the length of previous row
-      CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
-          dataPageMemoryBlock.getBaseOffset() + startOffset, length, CarbonUnsafe.BYTE_ARRAY_OFFSET,
-          CarbonCommonConstants.INT_SIZE_IN_BYTE);
-      buffer.put(length);
-      buffer.flip();
-      // so current row position will be
-      // previous row length + 4 bytes used for storing previous row data
-      startOffset += CarbonCommonConstants.INT_SIZE_IN_BYTE + buffer.getInt();
-      // as same byte buffer is used to avoid creating many byte buffer for each row
-      // we need to clear the byte buffer
-      buffer.clear();
-      // now put the offset of current row, here we need to add 4 more bytes as current will
-      // also have length part so we have to skip length
-      CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
-          dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
-          startOffset + CarbonCommonConstants.INT_SIZE_IN_BYTE);
-      // incrementing the pointers as first value is already filled and as we are storing as int
-      // we need to increment the 4 bytes to set the position of the next value to set
-      pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE;
-
-      this.isMemoryOccupied = true;
-    }
-  }
-
-  /**
-   * to get the byte value
-   *
-   * @param index
-   * @return byte value based on index
-   */
-  @Override public BigDecimal getBigDecimal(int index) {
-    // now to get the row from memory block we need to do following thing
-    // 1. first get the current offset
-    // 2. if it's not a last row- get the next row offset
-    // Subtract the current row offset + 4 bytes(to skip the data length) with next row offset
-    // else subtract the current row offset
-    // with complete data length get the offset of set of data
-    int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
-        dataPageMemoryBlock.getBaseOffset() + this.offsetStartPosition + (index
-            * CarbonCommonConstants.INT_SIZE_IN_BYTE));
-    int length = 0;
-    // calculating the length of data
-    if (index < numberOfRows - 1) {
-      int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
-          dataPageMemoryBlock.getBaseOffset() + this.offsetStartPosition + ((index + 1)
-              * CarbonCommonConstants.INT_SIZE_IN_BYTE));
-      length = OffsetOfNextdata - (currentDataOffset + CarbonCommonConstants.INT_SIZE_IN_BYTE);
-    } else {
-      // for last record we need to subtract with data length
-      length = (int) this.offsetStartPosition - currentDataOffset;
-    }
-    byte[] row = new byte[length];
-    CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
-        dataPageMemoryBlock.getBaseOffset() + currentDataOffset, row,
-        CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
-    return DataTypeUtil.byteToBigDecimal(row);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
deleted file mode 100644
index f44ddb5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
-
-import org.apache.carbondata.core.memory.CarbonUnsafe;
-import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
-
-/**
- * Responsible for storing Byte array data to memory. memory can be on heap or
- * offheap based on the user configuration using unsafe interface
- */
-public class UnsafeByteMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<byte[]> {
-
-  public UnsafeByteMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-  }
-
-  /**
-   * Below method will be used to put byte array data to memory
-   *
-   * @param data
-   */
-  @Override public void putData(byte[] data) {
-    assert (!this.isMemoryOccupied);
-    this.dataPageMemoryBlock =
-        MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(data.length);
-    // copy the data to memory
-    CarbonUnsafe.unsafe
-        .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
-            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
-    this.isMemoryOccupied = true;
-  }
-
-  /**
-   * to get the byte value
-   *
-   * @param index
-   * @return byte value based on index
-   */
-  @Override public byte getByte(int index) {
-    return CarbonUnsafe.unsafe
-        .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index);
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
deleted file mode 100644
index 40c7753..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.memory.CarbonUnsafe;
-import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
-
-/**
- * Responsible for storing double array data to memory. memory can be on heap or
- * offheap based on the user configuration using unsafe interface
- */
-public class UnsafeDoubleMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<double[]> {
-
-  public UnsafeDoubleMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-  }
-
-  /**
-   * Below method will be used to put double array data to memory
-   *
-   * @param data
-   */
-  @Override public void putData(double[] data) {
-    assert (!this.isMemoryOccupied);
-    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
-        .allocate(data.length * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE);
-    // copy the data to memory
-    CarbonUnsafe.unsafe
-        .copyMemory(data, CarbonUnsafe.DOUBLE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
-            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
-    this.isMemoryOccupied = true;
-  }
-
-  /**
-   * to get the double value
-   *
-   * @param index
-   * @return double value based on index
-   */
-  @Override public double getDouble(int index) {
-    return CarbonUnsafe.unsafe.getDouble(dataPageMemoryBlock.getBaseObject(),
-        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE));
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
deleted file mode 100644
index 1e1a5d2..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.memory.CarbonUnsafe;
-import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
-
-/**
- * Responsible for storing int array data to memory. memory can be on heap or
- * offheap based on the user configuration using unsafe interface
- */
-public class UnsafeIntMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<int[]> {
-
-  public UnsafeIntMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-  }
-
-  /**
-   * Below method will be used to put int array data to memory
-   *
-   * @param data
-   */
-  @Override public void putData(int[] data) {
-    assert (!this.isMemoryOccupied);
-    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
-        .allocate(data.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
-    // copy the data to memory
-    CarbonUnsafe.unsafe
-        .copyMemory(data, CarbonUnsafe.INT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
-            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
-    this.isMemoryOccupied = true;
-  }
-
-  /**
-   * to get the int value
-   *
-   * @param index
-   * @return int value based on index
-   */
-  @Override public int getInt(int index) {
-    return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
-        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.INT_SIZE_IN_BYTE));
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
deleted file mode 100644
index 82ebd13..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.memory.CarbonUnsafe;
-import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
-
-/**
- * Responsible for storing long array data to memory. memory can be on heap or
- * offheap based on the user configuration using unsafe interface
- */
-public class UnsafeLongMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<long[]> {
-
-  public UnsafeLongMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-  }
-
-  /**
-   * Below method will be used to put long array data to memory
-   *
-   * @param data
-   */
-  @Override public void putData(long[] data) {
-    assert (!this.isMemoryOccupied);
-    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
-        .allocate(data.length * CarbonCommonConstants.LONG_SIZE_IN_BYTE);
-    // copy the data to memory
-    CarbonUnsafe.unsafe
-        .copyMemory(data, CarbonUnsafe.LONG_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
-            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
-    this.isMemoryOccupied = true;
-  }
-
-  /**
-   * to get the long value
-   *
-   * @param index
-   * @return long value based on index
-   */
-  @Override public long getLong(int index) {
-    return CarbonUnsafe.unsafe.getLong(dataPageMemoryBlock.getBaseObject(),
-        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.LONG_SIZE_IN_BYTE));
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
deleted file mode 100644
index afe6ce7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.memory.CarbonUnsafe;
-import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
-
-/**
- * Responsible for storing short array data to memory. memory can be on heap or
- * offheap based on the user configuration using unsafe interface
- */
-public class UnsafeShortMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<short[]> {
-
-  public UnsafeShortMeasureChunkStore(int numberOfRows) {
-    super(numberOfRows);
-  }
-
-  /**
-   * Below method will be used to put short array data to memory
-   *
-   * @param data
-   */
-  @Override public void putData(short[] data) {
-    assert (!this.isMemoryOccupied);
-    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
-        .allocate(data.length * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
-    // copy the data to memory
-    CarbonUnsafe.unsafe
-        .copyMemory(data, CarbonUnsafe.SHORT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
-            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
-    this.isMemoryOccupied = true;
-  }
-
-  /**
-   * to get the short value
-   *
-   * @param index
-   * @return shot value based on index
-   */
-  @Override public short getShort(int index) {
-    return CarbonUnsafe.unsafe.getShort(dataPageMemoryBlock.getBaseObject(),
-        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
index 78e6d35..27194bb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
@@ -148,6 +148,15 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
     return rowIdPage;
   }
 
+  @Override
+  public int getRowIdPageLengthInBytes() {
+    if (rowIdPage != null) {
+      return rowIdPage.length * 4;
+    } else {
+      return 0;
+    }
+  }
+
   /**
    * @return the rowIdRlePage
    */
@@ -155,6 +164,15 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
     return rowIdRlePage;
   }
 
+  @Override
+  public int getRowIdRlePageLengthInBytes() {
+    if (rowIdRlePage != null) {
+      return rowIdRlePage.length * 4;
+    } else {
+      return 0;
+    }
+  }
+
   /**
    * @return the dataPage
    */
@@ -205,6 +223,15 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
     return dataRlePage;
   }
 
+  @Override
+  public int getDataRlePageLengthInBytes() {
+    if (dataRlePage != null) {
+      return dataRlePage.length * 4;
+    } else {
+      return 0;
+    }
+  }
+
   @Override public int getTotalSize() {
     return totalSize;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java
index c168614..218694f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java
@@ -60,6 +60,11 @@ public class BlockIndexerStorageForNoInvertedIndexForInt implements IndexStorage
     return new int[0];
   }
 
+  @Override
+  public int getDataRlePageLengthInBytes() {
+    return 0;
+  }
+
   @Override public int getTotalSize() {
     return totalSize;
   }
@@ -77,6 +82,11 @@ public class BlockIndexerStorageForNoInvertedIndexForInt implements IndexStorage
     return new int[0];
   }
 
+  @Override
+  public int getRowIdPageLengthInBytes() {
+    return 0;
+  }
+
   /**
    * no use
    *
@@ -86,6 +96,11 @@ public class BlockIndexerStorageForNoInvertedIndexForInt implements IndexStorage
     return new int[0];
   }
 
+  @Override
+  public int getRowIdRlePageLengthInBytes() {
+    return 0;
+  }
+
   /**
    * @return the dataPage
    */

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
index 5712655..911a260 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
@@ -77,6 +77,11 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
     return new short[0];
   }
 
+  @Override
+  public int getDataRlePageLengthInBytes() {
+    return 0;
+  }
+
   @Override public int getTotalSize() {
     return totalSize;
   }
@@ -94,6 +99,11 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
     return new short[0];
   }
 
+  @Override
+  public int getRowIdPageLengthInBytes() {
+    return 0;
+  }
+
   /**
    * no use
    *
@@ -103,6 +113,11 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
     return new short[0];
   }
 
+  @Override
+  public int getRowIdRlePageLengthInBytes() {
+    return 0;
+  }
+
   /**
    * @return the dataPage
    */

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
index 2ffb610..7303b66 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
@@ -155,6 +155,15 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
     return rowIdPage;
   }
 
+  @Override
+  public int getRowIdPageLengthInBytes() {
+    if (rowIdPage != null) {
+      return rowIdPage.length * 2;
+    } else {
+      return 0;
+    }
+  }
+
   /**
    * @return the rowIdRlePage
    */
@@ -162,6 +171,15 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
     return rowIdRlePage;
   }
 
+  @Override
+  public int getRowIdRlePageLengthInBytes() {
+    if (rowIdRlePage != null) {
+      return rowIdRlePage.length * 2;
+    } else {
+      return 0;
+    }
+  }
+
   /**
    * @return the dataPage
    */
@@ -224,6 +242,15 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
     return dataRlePage;
   }
 
+  @Override
+  public int getDataRlePageLengthInBytes() {
+    if (dataRlePage != null) {
+      return dataRlePage.length * 2;
+    } else {
+      return 0;
+    }
+  }
+
   @Override public int getTotalSize() {
     return totalSize;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
index 0570317..0ef8cad 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
@@ -18,16 +18,23 @@
 package org.apache.carbondata.core.datastore.columnar;
 
 public interface IndexStorage<T> {
+
   boolean isAlreadySorted();
 
   T getRowIdPage();
 
+  int getRowIdPageLengthInBytes();
+
   T getRowIdRlePage();
 
+  int getRowIdRlePageLengthInBytes();
+
   byte[][] getDataPage();
 
   T getDataRlePage();
 
+  int getDataRlePageLengthInBytes();
+
   int getTotalSize();
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index ef2c9ec..180c092 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -19,11 +19,10 @@ package org.apache.carbondata.core.datastore.page;
 
 import java.io.IOException;
 import java.math.BigDecimal;
-import java.util.BitSet;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.compression.Compressor;
-import org.apache.carbondata.core.datastore.page.statistics.ColumnPageStatsVO;
+import org.apache.carbondata.core.datastore.page.statistics.ColumnPageStatsCollector;
 import org.apache.carbondata.core.memory.MemoryException;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.datatype.DecimalConverterFactory;
@@ -46,11 +45,8 @@ public abstract class ColumnPage {
   protected int scale;
   protected int precision;
 
-  // statistics of this column page
-  private ColumnPageStatsVO stats;
-
-  // The index of the rowId whose value is null, will be set to 1
-  private BitSet nullBitSet;
+  // statistics collector for this column page
+  private ColumnPageStatsCollector statsCollector;
 
   protected DecimalConverterFactory.DecimalConverter decimalConverter;
 
@@ -74,14 +70,18 @@ public abstract class ColumnPage {
     return dataType;
   }
 
-  public ColumnPageStatsVO getStatistics() {
-    return stats;
+  public Object getStatistics() {
+    return statsCollector.getPageStats();
   }
 
   public int getPageSize() {
     return pageSize;
   }
 
+  public void setStatsCollector(ColumnPageStatsCollector statsCollector) {
+    this.statsCollector = statsCollector;
+  }
+
   private static ColumnPage createVarLengthPage(DataType dataType, int pageSize, int scale,
       int precision) {
     if (unsafe) {
@@ -299,36 +299,40 @@ public abstract class ColumnPage {
   public void putData(int rowId, Object value) {
     if (value == null) {
       putNull(rowId);
-      stats.updateNull();
+      statsCollector.updateNull(rowId);
       return;
     }
     switch (dataType) {
       case BYTE:
-        // TODO: change sort step to store as exact data type
         putByte(rowId, (byte) value);
+        statsCollector.update((byte) value);
         break;
       case SHORT:
         putShort(rowId, (short) value);
+        statsCollector.update((short) value);
         break;
       case INT:
         putInt(rowId, (int) value);
+        statsCollector.update((int) value);
         break;
       case LONG:
         putLong(rowId, (long) value);
+        statsCollector.update((long) value);
         break;
       case DOUBLE:
         putDouble(rowId, (double) value);
+        statsCollector.update((double) value);
         break;
       case DECIMAL:
         putDecimal(rowId, (BigDecimal) value);
         break;
       case BYTE_ARRAY:
         putBytes(rowId, (byte[]) value);
+        statsCollector.update((byte[]) value);
         break;
       default:
         throw new RuntimeException("unsupported data type: " + dataType);
     }
-    stats.update(value);
   }
 
   /**
@@ -381,7 +385,6 @@ public abstract class ColumnPage {
    * Set null at rowId
    */
   private void putNull(int rowId) {
-    nullBitSet.set(rowId);
     switch (dataType) {
       case BYTE:
         putByte(rowId, (byte) 0);
@@ -405,13 +408,6 @@ public abstract class ColumnPage {
   }
 
   /**
-   * Get null bitset page
-   */
-  public BitSet getNullBitSet() {
-    return nullBitSet;
-  }
-
-  /**
    * Get byte value at rowId
    */
   public abstract byte getByte(int rowId);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/page/EncodedTablePage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/EncodedTablePage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/EncodedTablePage.java
new file mode 100644
index 0000000..ea9c373
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/EncodedTablePage.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.columnar.IndexStorage;
+import org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage;
+import org.apache.carbondata.core.datastore.page.encoding.EncodedDimensionPage;
+import org.apache.carbondata.core.datastore.page.encoding.EncodedMeasurePage;
+import org.apache.carbondata.core.datastore.page.key.TablePageKey;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * Table page that after encoding and compression.
+ */
+public class EncodedTablePage {
+
+  // encoded data and metadata for each dimension column
+  private EncodedDimensionPage[] dimensions;
+
+  // encoded data and metadata for each measure column
+  private EncodedMeasurePage[] measures;
+
+  // key of this page
+  private TablePageKey pageKey;
+
+  // number of row in this page
+  private int pageSize;
+
+  // true if it is last page of all input rows
+  private boolean isLastPage;
+
+  // size in bytes of all encoded columns (including data and metadate)
+  private int encodedSize;
+
+  public static EncodedTablePage newEmptyInstance() {
+    EncodedTablePage page = new EncodedTablePage();
+    page.pageSize = 0;
+    page.encodedSize = 0;
+    page.measures = new EncodedMeasurePage[0];
+    page.dimensions = new EncodedDimensionPage[0];
+    return page;
+  }
+
+  public static EncodedTablePage newInstance(int pageSize,
+      EncodedDimensionPage[] dimensions, EncodedMeasurePage[] measures,
+      TablePageKey tablePageKey) {
+    return new EncodedTablePage(pageSize, dimensions, measures, tablePageKey);
+  }
+
+  private EncodedTablePage() {
+  }
+
+  private EncodedTablePage(int pageSize, EncodedDimensionPage[] encodedDimensions,
+      EncodedMeasurePage[] encodedMeasures, TablePageKey tablePageKey) {
+    this.dimensions = encodedDimensions;
+    this.measures = encodedMeasures;
+    this.pageSize = pageSize;
+    this.pageKey = tablePageKey;
+    this.encodedSize = calculatePageSize(encodedDimensions, encodedMeasures);
+  }
+
+  // return size in bytes of this encoded page
+  private int calculatePageSize(EncodedDimensionPage[] encodedDimensions,
+      EncodedMeasurePage[] encodedMeasures) {
+    int size = 0;
+    int totalEncodedDimensionDataLength = 0;
+    int totalEncodedMeasuredDataLength = 0;
+    // add row id index length
+    for (EncodedDimensionPage dimension : dimensions) {
+      IndexStorage indexStorage = dimension.getIndexStorage();
+      if (!indexStorage.isAlreadySorted()) {
+        size += indexStorage.getRowIdPageLengthInBytes() +
+            indexStorage.getRowIdRlePageLengthInBytes() +
+            CarbonCommonConstants.INT_SIZE_IN_BYTE;
+      }
+      if (indexStorage.getDataRlePageLengthInBytes() > 0) {
+        size += indexStorage.getDataRlePageLengthInBytes();
+      }
+      totalEncodedDimensionDataLength += dimension.getEncodedData().length;
+    }
+    for (EncodedColumnPage measure : measures) {
+      size += measure.getEncodedData().length;
+    }
+
+    for (EncodedDimensionPage encodedDimension : encodedDimensions) {
+      size += CarbonUtil.getByteArray(encodedDimension.getDataChunk2()).length;
+    }
+    for (EncodedMeasurePage encodedMeasure : encodedMeasures) {
+      size += CarbonUtil.getByteArray(encodedMeasure.getDataChunk2()).length;
+    }
+    size += totalEncodedDimensionDataLength + totalEncodedMeasuredDataLength;
+    return size;
+  }
+
+  public int getEncodedSize() {
+    return encodedSize;
+  }
+
+  public int getPageSize() {
+    return pageSize;
+  }
+
+  public int getNumDimensions() {
+    return dimensions.length;
+  }
+
+  public int getNumMeasures() {
+    return measures.length;
+  }
+
+  public TablePageKey getPageKey() {
+    return pageKey;
+  }
+
+  public boolean isLastPage() {
+    return isLastPage;
+  }
+
+  public void setIsLastPage(boolean isWriteAll) {
+    this.isLastPage = isWriteAll;
+  }
+
+  public EncodedMeasurePage getMeasure(int measureIndex) {
+    return measures[measureIndex];
+  }
+
+  public EncodedMeasurePage[] getMeasures() {
+    return measures;
+  }
+
+  public EncodedDimensionPage getDimension(int dimensionIndex) {
+    return dimensions[dimensionIndex];
+  }
+
+  public EncodedDimensionPage[] getDimensions() {
+    return dimensions;
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bc3e6843/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
index b0978d3..2864e80 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.datastore.page;
 import java.math.BigDecimal;
 
 /**
- * This is a decorator of column page, it performs transformation lazily (when caller calls getXXX
+ * This is a decorator of column page, it performs decoding lazily (when caller calls getXXX
  * method to get the value from the page)
  */
 public class LazyColumnPage extends ColumnPage {
@@ -38,8 +38,8 @@ public class LazyColumnPage extends ColumnPage {
     this.codec = codec;
   }
 
-  public static ColumnPage newPage(ColumnPage columnPage, PrimitiveCodec transform) {
-    return new LazyColumnPage(columnPage, transform);
+  public static ColumnPage newPage(ColumnPage columnPage, PrimitiveCodec codec) {
+    return new LazyColumnPage(columnPage, codec);
   }
 
   @Override
@@ -94,7 +94,7 @@ public class LazyColumnPage extends ColumnPage {
 
   @Override
   public BigDecimal getDecimal(int rowId) {
-    throw new UnsupportedOperationException("internal error");
+    return columnPage.getDecimal(rowId);
   }
 
   @Override


Mime
View raw message