carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [5/7] incubator-carbondata git commit: Added support for offheap storage in query
Date Sun, 08 Jan 2017 14:30:23 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java
new file mode 100644
index 0000000..e7068b0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeByteMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeDoubleMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeIntMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeLongMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeShortMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeByteMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeDoubleMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeIntMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeLongMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeShortMeasureChunkStore;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+/**
+ * Factory class for getting the measure store type
+ */
+public class MeasureChunkStoreFactory {
+
+  /**
+   * instance type
+   */
+  public static final MeasureChunkStoreFactory INSTANCE = new MeasureChunkStoreFactory();
+
+  /**
+   * is unsafe
+   */
+  private static final boolean isUnsafe;
+
+  static {
+    isUnsafe = Boolean.parseBoolean(CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION,
+            CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE));
+  }
+
+  private MeasureChunkStoreFactory() {
+  }
+
+  /**
+   * Below method will be used to get the measure data chunk store based on data type
+   *
+   * @param dataType     data type
+   * @param numberOfRows number of rows
+   * @return measure chunk store
+   */
+  public MeasureDataChunkStore getMeasureDataChunkStore(DataType dataType, int numberOfRows) {
+    if (isUnsafe) {
+      switch (dataType) {
+        case DATA_BYTE:
+          return new SafeByteMeasureChunkStore(numberOfRows);
+        case DATA_SHORT:
+          return new SafeShortMeasureChunkStore(numberOfRows);
+        case DATA_INT:
+          return new SafeIntMeasureChunkStore(numberOfRows);
+        case DATA_LONG:
+          return new SafeLongMeasureChunkStore(numberOfRows);
+        case DATA_DOUBLE:
+          return new SafeDoubleMeasureChunkStore(numberOfRows);
+        default:
+          return new SafeDoubleMeasureChunkStore(numberOfRows);
+      }
+    } else {
+      switch (dataType) {
+        case DATA_BYTE:
+          return new UnsafeByteMeasureChunkStore(numberOfRows);
+        case DATA_SHORT:
+          return new UnsafeShortMeasureChunkStore(numberOfRows);
+        case DATA_INT:
+          return new UnsafeIntMeasureChunkStore(numberOfRows);
+        case DATA_LONG:
+          return new UnsafeLongMeasureChunkStore(numberOfRows);
+        case DATA_DOUBLE:
+          return new UnsafeDoubleMeasureChunkStore(numberOfRows);
+        default:
+          return new UnsafeDoubleMeasureChunkStore(numberOfRows);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java
new file mode 100644
index 0000000..fed4ab4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store;
+
+import java.math.BigDecimal;
+
+/**
+ * Responsibility is store the measure data in memory,
+ * memory can be on heap or offheap based on the user configuration
+ */
+public interface MeasureDataChunkStore<T> {
+
+  /**
+   * Below method will be used to put the data to memory
+   *
+   * @param data
+   */
+  void putData(T data);
+
+  /**
+   * to get byte value
+   *
+   * @param index
+   * @return byte value based on index
+   */
+  byte getByte(int index);
+
+  /**
+   * to get the short value
+   *
+   * @param index
+   * @return short value based on index
+   */
+  short getShort(int index);
+
+  /**
+   * to get the int value
+   *
+   * @param index
+   * @return int value based on index
+   */
+  int getInt(int index);
+
+  /**
+   * to get the long value
+   *
+   * @param index
+   * @return long value based on index
+   */
+  long getLong(int index);
+
+  /**
+   * to get the double value
+   *
+   * @param index
+   * @return double value based on index
+   */
+  double getDouble(int index);
+
+  /**
+   * To get the bigdecimal value
+   *
+   * @param index
+   * @return bigdecimal value based on index
+   */
+  BigDecimal getBigDecimal(int index);
+
+  /**
+   * To free the occupied memory
+   */
+  void freeMemory();
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
new file mode 100644
index 0000000..0ed2218
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.store.DimensionDataChunkStore;
+
+/**
+ * Responsibility is to store dimension data
+ */
+public abstract class SafeAbsractDimensionDataChunkStore implements DimensionDataChunkStore {
+
+  /**
+   * data chunk for dimension column
+   */
+  protected byte[] data;
+
+  /**
+   * inverted index
+   */
+  protected int[] invertedIndex;
+
+  /**
+   * inverted index reverser
+   */
+  protected int[] invertedIndexReverse;
+
+  /**
+   * to check whether dimension column was explicitly sorted or not
+   */
+  protected boolean isExplictSorted;
+
+  /**
+   * Constructor
+   *
+   * @param totalSize      total size of the data to be kept
+   * @param isInvertedIdex is inverted index present
+   * @param numberOfRows   total number of rows
+   */
+  public SafeAbsractDimensionDataChunkStore(boolean isInvertedIdex) {
+    this.isExplictSorted = isInvertedIdex;
+  }
+
+  /**
+   * Below method will be used to put the rows and its metadata in offheap
+   *
+   * @param invertedIndex        inverted index to be stored
+   * @param invertedIndexReverse inverted index reverse to be stored
+   * @param data                 data to be stored
+   */
+  @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
+      final byte[] data) {
+    this.data = data;
+    this.invertedIndex = invertedIndex;
+    this.invertedIndexReverse = invertedIndexReverse;
+  }
+
+  /**
+   * Below method will be used to free the memory occupied by the column chunk
+   */
+  @Override public void freeMemory() {
+    // do nothing as GC will take care of freeing memory
+  }
+
+  /**
+   * Below method will be used to get the inverted index
+   *
+   * @param rowId row id
+   * @return inverted index based on row id passed
+   */
+  @Override public int getInvertedIndex(int rowId) {
+    return invertedIndex[rowId];
+  }
+
+  /**
+   * Below method will be used to get the surrogate key of the based on the row
+   * id passed
+   *
+   * @param rowId row id
+   * @return surrogate key
+   */
+  @Override public int getSurrogate(int rowId) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * @return size of each column value
+   */
+  @Override public int getColumnValueSize() {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * @return whether column was explicitly sorted or not
+   */
+  @Override public boolean isExplicitSorted() {
+    return isExplictSorted;
+  }
+
+  /**
+   * Below method will be used to fill the row values to data array
+   *
+   * @param rowId  row id of the data to be filled
+   * @param data   buffer in which data will be filled
+   * @param offset off the of the buffer
+   */
+  @Override public void fillRow(int rowId, byte[] data, int offset) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java
new file mode 100644
index 0000000..0daeaa4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
+
+/**
+ * Responsibility is store the measure data in memory,
+ */
+public abstract class SafeAbstractMeasureDataChunkStore<T> implements
+    MeasureDataChunkStore<T> {
+
+  /**
+   * number of rows
+   */
+  protected int numberOfRows;
+
+  public SafeAbstractMeasureDataChunkStore(int numberOfRows) {
+    this.numberOfRows = numberOfRows;
+  }
+
+  /**
+   * to get the byte value
+   *
+   * @param index
+   * @return byte value based on index
+   */
+  @Override
+  public byte getByte(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * to get the short value
+   *
+   * @param index
+   * @return short value based on index
+   */
+  @Override
+  public short getShort(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * to get the int value
+   *
+   * @param index
+   * @return int value based on index
+   */
+  @Override
+  public int getInt(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * to get the long value
+   *
+   * @param index
+   * @return long value based on index
+   */
+  @Override
+  public long getLong(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * to get the double value
+   *
+   * @param index
+   * @return double value based on index
+   */
+  @Override
+  public double getDouble(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * To get the bigdecimal value
+   *
+   * @param index
+   * @return bigdecimal value based on index
+   */
+  @Override
+  public BigDecimal getBigDecimal(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * To free the occupied memory
+   */
+  @Override
+  public void freeMemory() {
+    // do nothing as GC will take care of freeing the memory
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java
new file mode 100644
index 0000000..1f69cd2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+/**
+ * Responsible for storing Byte array data to memory.
+ */
+public class SafeByteMeasureChunkStore extends
+    SafeAbstractMeasureDataChunkStore<byte[]> {
+
+  /**
+   * data
+   */
+  private byte[] data;
+
+  public SafeByteMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put byte array data to memory
+   *
+   * @param data
+   */
+  @Override
+  public void putData(byte[] data) {
+    this.data = data;
+  }
+
+  /**
+   * to get the byte value
+   *
+   * @param index
+   * @return byte value based on index
+   */
+  @Override
+  public byte getByte(int index) {
+    return this.data[index];
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java
new file mode 100644
index 0000000..dc485c6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+/**
+ * Below class will be used to store the measure values of double data type
+ */
+public class SafeDoubleMeasureChunkStore extends
+    SafeAbstractMeasureDataChunkStore<double[]> {
+
+  /**
+   * data
+   */
+  private double[] data;
+
+  public SafeDoubleMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to store double array data
+   *
+   * @param data
+   */
+  @Override
+  public void putData(double[] data) {
+    this.data = data;
+  }
+
+  /**
+   * to get the double value
+   *
+   * @param index
+   * @return double value based on index
+   */
+  @Override
+  public double getDouble(int index) {
+    return this.data[index];
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
new file mode 100644
index 0000000..3f5afc5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Below class will be used to store fixed length dimension data
+ */
+public class SafeFixedLengthDimensionDataChunkStore extends SafeAbsractDimensionDataChunkStore {
+
+  /**
+   * Size of each value
+   */
+  private int columnValueSize;
+
+  public SafeFixedLengthDimensionDataChunkStore(boolean isInvertedIndex, int columnValueSize) {
+    super(isInvertedIndex);
+    this.columnValueSize = columnValueSize;
+  }
+
+  /**
+   * Below method will be used to get the row based inverted index
+   *
+   * @param rowId Inverted index
+   */
+  @Override public byte[] getRow(int rowId) {
+    // if column was explicitly sorted we need to get the rowid based inverted index reverse
+    if (isExplictSorted) {
+      rowId = invertedIndexReverse[rowId];
+    }
+    // creating a row
+    byte[] row = new byte[columnValueSize];
+    //copy the row from data chunk based on offset
+    // offset position will be index * each column value length
+    System.arraycopy(this.data, rowId * columnValueSize, row, 0, columnValueSize);
+    return row;
+  }
+
+  /**
+   * Below method will be used to get the surrogate key of the based on the row
+   * id passed
+   *
+   * @param rowId row id
+   * @return surrogate key
+   */
+  @Override public int getSurrogate(int index) {
+    // if column was explicitly sorted we need to get the rowid based inverted index reverse
+    if (isExplictSorted) {
+      index = invertedIndexReverse[index];
+    }
+    // below part is to convert the byte array to surrogate value
+    int startOffsetOfData = index * columnValueSize;
+    int surrogate = 0;
+    for (int i = 0; i < columnValueSize; i++) {
+      surrogate <<= 8;
+      surrogate ^= data[startOffsetOfData] & 0xFF;
+      startOffsetOfData++;
+    }
+    return surrogate;
+  }
+
+  /**
+   * Below method will be used to fill the row values to buffer array
+   *
+   * @param rowId  row id of the data to be filled
+   * @param data   buffer in which data will be filled
+   * @param offset off the of the buffer
+   */
+  @Override public void fillRow(int rowId, byte[] buffer, int offset) {
+    // if column was explicitly sorted we need to get the rowid based inverted index reverse
+    if (isExplictSorted) {
+      rowId = invertedIndexReverse[rowId];
+    }
+    //copy the row from memory block based on offset
+    // offset position will be index * each column value length
+    System.arraycopy(data, rowId * columnValueSize, buffer, offset, columnValueSize);
+  }
+
+  /**
+   * @return size of each column value
+   */
+  @Override public int getColumnValueSize() {
+    return columnValueSize;
+  }
+
+  /**
+   * to compare the two byte array
+   *
+   * @param index        index of first byte array
+   * @param compareValue value of to be compared
+   * @return compare result
+   */
+  @Override public int compareTo(int index, byte[] compareValue) {
+    return ByteUtil.UnsafeComparer.INSTANCE
+        .compareTo(data, index * columnValueSize, columnValueSize, compareValue, 0,
+            columnValueSize);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java
new file mode 100644
index 0000000..29b77cb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+/**
+ * Responsible for storing int array data to memory.
+ */
+public class SafeIntMeasureChunkStore extends
+    SafeAbstractMeasureDataChunkStore<int[]> {
+
+  /**
+   * data
+   */
+  private int[] data;
+
+  public SafeIntMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put int array data to memory
+   *
+   * @param data
+   */
+  @Override
+  public void putData(int[] data) {
+    this.data = data;
+  }
+
+  /**
+   * to get the int value
+   *
+   * @param index
+   * @return int value based on index
+   */
+  @Override
+  public int getInt(int index) {
+    return this.data[index];
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java
new file mode 100644
index 0000000..3fd0370
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+/**
+ * Below class will be used to store the measure values of long data type
+ *
+ */
+public class SafeLongMeasureChunkStore extends
+    SafeAbstractMeasureDataChunkStore<long[]> {
+
+  /**
+   * data
+   */
+  private long[] data;
+
+  public SafeLongMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to store long array data
+   *
+   * @param data
+   */
+  @Override
+  public void putData(long[] data) {
+    this.data = data;
+  }
+
+  /**
+   * to get the long value
+   *
+   * @param index
+   * @return long value based on index
+   */
+  @Override
+  public long getLong(int index) {
+    return this.data[index];
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java
new file mode 100644
index 0000000..269ad79
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+/**
+ * Below class will be used to store the measure values of short data type
+ *
+ */
+public class SafeShortMeasureChunkStore extends
+    SafeAbstractMeasureDataChunkStore<short[]> {
+
+  /**
+   * data
+   */
+  private short[] data;
+
+  public SafeShortMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put short array data
+   *
+   * @param data
+   */
+  @Override
+  public void putData(short[] data) {
+    this.data = data;
+  }
+
+  /**
+   * to get the short value
+   *
+   * @param index
+   * @return shot value based on index
+   */
+  @Override
+  public short getShort(int index) {
+    return data[index];
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
new file mode 100644
index 0000000..7d5ccd2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Below class is responsible to store variable length dimension data chunk in
+ * memory Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimensionDataChunkStore {
+
+  /**
+   * total number of rows
+   */
+  private int numberOfRows;
+
+  /**
+   * offset of the data this will be used during search, as we can directly jump
+   * to particular location
+   */
+  private int[] dataOffsets;
+
+  public SafeVariableLengthDimensionDataChunkStore(boolean isInvertedIndex, int numberOfRows) {
+    super(isInvertedIndex);
+    this.numberOfRows = numberOfRows;
+    this.dataOffsets = new int[numberOfRows];
+  }
+
+  /**
+   * Below method will be used to put the rows and its metadata in offheap
+   *
+   * @param invertedIndex        inverted index to be stored
+   * @param invertedIndexReverse inverted index reverse to be stored
+   * @param data                 data to be stored
+   */
+  @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
+      byte[] data) {
+    // first put the data, inverted index and reverse inverted index to memory
+    super.putArray(invertedIndex, invertedIndexReverse, data);
+    // As data is of variable length and data format is
+    // <length in short><data><length in short><data>
+    // we need to store offset of each data so data can be accessed directly
+    // for example:
+    //data = {0,5,1,2,3,4,5,0,6,0,1,2,3,4,5,0,2,8,9}
+    //so value stored in offset will be position of actual data
+    // [2,9,17]
+    // to store this value we need to get the actual data length + 2 bytes used for storing the
+    // length
+
+    // start position will be used to store the current data position
+    int startOffset = 0;
+    // as first position will be start from 2 byte as data is stored first in the memory block
+    // we need to skip first two bytes this is because first two bytes will be length of the data
+    // which we have to skip
+    dataOffsets[0] = CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+    // creating a byte buffer which will wrap the length of the row
+    ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+    for (int i = 1; i < numberOfRows; i++) {
+      buffer.put(data, startOffset, CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+      buffer.flip();
+      // so current row position will be
+      // previous row length + 2 bytes used for storing previous row data
+      startOffset += buffer.getShort() + CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+      // as same byte buffer is used to avoid creating many byte buffer for each row
+      // we need to clear the byte buffer
+      buffer.clear();
+      dataOffsets[i] = startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+    }
+  }
+
+  @Override public byte[] getRow(int rowId) {
+    // if column was explicitly sorted we need to get the rowid based inverted index reverse
+    if (isExplictSorted) {
+      rowId = invertedIndexReverse[rowId];
+    }
+    // now to get the row from memory block we need to do following thing
+    // 1. first get the current offset
+    // 2. if it's not a last row- get the next row offset
+    // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
+    // else subtract the current row offset with complete data
+    // length get the offset of set of data
+    int currentDataOffset = dataOffsets[rowId];
+    short length = 0;
+    // calculating the length of data
+    if (rowId < numberOfRows - 1) {
+      length = (short) (dataOffsets[rowId + 1] - (currentDataOffset
+          + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+    } else {
+      // for last record
+      length = (short) (this.data.length - currentDataOffset);
+    }
+    byte[] currentRowData = new byte[length];
+    System.arraycopy(data, currentDataOffset, currentRowData, 0, length);
+    return currentRowData;
+  }
+
+  @Override public int compareTo(int index, byte[] compareValue) {
+    // now to get the row from memory block we need to do following thing
+    // 1. first get the current offset
+    // 2. if it's not a last row- get the next row offset
+    // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
+    // else subtract the current row offset with complete data
+    // length
+
+    // get the offset of set of data
+    int currentDataOffset = dataOffsets[index];
+    short length = 0;
+    // calculating the length of data
+    if (index < numberOfRows - 1) {
+      length = (short) (dataOffsets[index + 1] - (currentDataOffset
+          + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+    } else {
+      // for last record
+      length = (short) (this.data.length - currentDataOffset);
+    }
+    return ByteUtil.UnsafeComparer.INSTANCE
+        .compareTo(data, currentDataOffset, length, compareValue, 0, compareValue.length);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
new file mode 100644
index 0000000..044e5f4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.store.DimensionDataChunkStore;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.memory.MemoryBlock;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsibility is to store dimension data in memory. storage can be on heap
+ * or offheap.
+ */
+public abstract class UnsafeAbstractDimensionDataChunkStore implements DimensionDataChunkStore {
+
+  /**
+   * memory block for data page
+   */
+  protected MemoryBlock dataPageMemoryBlock;
+
+  /**
+   * to check whether dimension column was explicitly sorted or not
+   */
+  protected boolean isExplicitSorted;
+
+  /**
+   * is memory released
+   */
+  protected boolean isMemoryReleased;
+
+  /**
+   * length of the actual data
+   */
+  protected int dataLength;
+
+  /**
+   * offset of the inverted index reverse
+   */
+  protected long invertedIndexReverseOffset;
+
+  /**
+   * to validate whether data is already kept in memory or not
+   */
+  protected boolean isMemoryOccupied;
+
+  /**
+   * Constructor
+   *
+   * @param totalSize      total size of the data to be kept
+   * @param isInvertedIdex is inverted index present
+   * @param numberOfRows   total number of rows
+   */
+  public UnsafeAbstractDimensionDataChunkStore(long totalSize, boolean isInvertedIdex,
+      int numberOfRows) {
+    // allocating the data page
+    this.dataPageMemoryBlock =
+        MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(totalSize);
+    this.isExplicitSorted = isInvertedIdex;
+  }
+
+  /**
+   * Below method will be used to put the rows and its metadata in offheap
+   *
+   * @param invertedIndex        inverted index to be stored
+   * @param invertedIndexReverse inverted index reverse to be stored
+   * @param data                 data to be stored
+   */
+  @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
+      final byte[] data) {
+    assert (!isMemoryOccupied);
+    this.dataLength = data.length;
+    this.invertedIndexReverseOffset = dataLength;
+    if (isExplicitSorted) {
+      this.invertedIndexReverseOffset +=
+          invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE;
+    }
+    // copy the data to memory
+    CarbonUnsafe.unsafe
+        .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset(), this.dataLength);
+    // if inverted index is present then copy the inverted index
+    // and reverse inverted index to memory
+    if (isExplicitSorted) {
+      CarbonUnsafe.unsafe.copyMemory(invertedIndex, CarbonUnsafe.INT_ARRAY_OFFSET,
+          dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + dataLength,
+          invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
+      CarbonUnsafe.unsafe.copyMemory(invertedIndexReverse, CarbonUnsafe.INT_ARRAY_OFFSET,
+          dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset,
+          invertedIndexReverse.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
+    }
+  }
+
+  /**
+   * Below method will be used to free the memory occupied by the column chunk
+   */
+  @Override public void freeMemory() {
+    if (isMemoryReleased) {
+      return;
+    }
+    // free data page memory
+    MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().free(dataPageMemoryBlock);
+    isMemoryReleased = true;
+    this.dataPageMemoryBlock = null;
+    this.isMemoryOccupied = false;
+  }
+
+  /**
+   * Below method will be used to get the inverted index
+   *
+   * @param rowId row id
+   * @return inverted index based on row id passed
+   */
+  @Override public int getInvertedIndex(int rowId) {
+    return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + dataLength + (rowId
+            * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+  }
+
+  /**
+   * Below method will be used to get the surrogate key of the based on the row
+   * id passed
+   *
+   * @param rowId row id
+   * @return surrogate key
+   */
+  @Override public int getSurrogate(int rowId) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * @return size of each column value
+   */
+  @Override public int getColumnValueSize() {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * @return whether column was explicitly sorted or not
+   */
+  @Override public boolean isExplicitSorted() {
+    return isExplicitSorted;
+  }
+
+  /**
+   * Below method will be used to fill the row values to data array
+   *
+   * @param rowId  row id of the data to be filled
+   * @param data   buffer in which data will be filled
+   * @param offset off the of the buffer
+   */
+  @Override public void fillRow(int rowId, byte[] data, int offset) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java
new file mode 100644
index 0000000..98cb2f6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.memory.MemoryBlock;
+
+/**
+ * Responsibility is store the measure data in memory, memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public abstract class UnsafeAbstractMeasureDataChunkStore<T> implements MeasureDataChunkStore<T> {
+
+  /**
+   * memory block
+   */
+  protected MemoryBlock dataPageMemoryBlock;
+
+  /**
+   * number of rows
+   */
+  protected int numberOfRows;
+
+  /**
+   * to check memory is released or not
+   */
+  protected boolean isMemoryReleased;
+
+  /**
+   * to check memory is occupied or not
+   */
+  protected boolean isMemoryOccupied;
+
+  public UnsafeAbstractMeasureDataChunkStore(int numberOfRows) {
+    this.numberOfRows = numberOfRows;
+  }
+
+  /**
+   * to get the byte value
+   *
+   * @param index
+   * @return byte value based on index
+   */
+  @Override public byte getByte(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * to get the short value
+   *
+   * @param index
+   * @return short value based on index
+   */
+  @Override public short getShort(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * to get the int value
+   *
+   * @param index
+   * @return int value based on index
+   */
+  @Override public int getInt(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * to get the long value
+   *
+   * @param index
+   * @return long value based on index
+   */
+  @Override public long getLong(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * to get the double value
+   *
+   * @param index
+   * @return double value based on index
+   */
+  @Override public double getDouble(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * To get the bigdecimal value
+   *
+   * @param index
+   * @return bigdecimal value based on index
+   */
+  @Override public BigDecimal getBigDecimal(int index) {
+    throw new UnsupportedOperationException("Operation not supported");
+  }
+
+  /**
+   * To free the occupied memory
+   */
+  @Override public void freeMemory() {
+    if (isMemoryReleased) {
+      return;
+    }
+    MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().free(dataPageMemoryBlock);
+    isMemoryReleased = true;
+    this.dataPageMemoryBlock = null;
+    this.isMemoryOccupied = false;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
new file mode 100644
index 0000000..5d0a802
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsible for storing Byte array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeByteMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<byte[]> {
+
+  public UnsafeByteMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put byte array data to memory
+   *
+   * @param data
+   */
+  @Override public void putData(byte[] data) {
+    assert (!this.isMemoryOccupied);
+    this.dataPageMemoryBlock =
+        MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(data.length);
+    // copy the data to memory
+    CarbonUnsafe.unsafe
+        .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+    this.isMemoryOccupied = true;
+  }
+
+  /**
+   * to get the byte value
+   *
+   * @param index
+   * @return byte value based on index
+   */
+  @Override public byte getByte(int index) {
+    return CarbonUnsafe.unsafe
+        .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
new file mode 100644
index 0000000..9c61ff2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsible for storing double array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeDoubleMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<double[]> {
+
+  public UnsafeDoubleMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put double array data to memory
+   *
+   * @param data
+   */
+  @Override public void putData(double[] data) {
+    assert (!this.isMemoryOccupied);
+    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+        .allocate(data.length * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE);
+    // copy the data to memory
+    CarbonUnsafe.unsafe
+        .copyMemory(data, CarbonUnsafe.DOUBLE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+    this.isMemoryOccupied = true;
+  }
+
+  /**
+   * to get the double value
+   *
+   * @param index
+   * @return double value based on index
+   */
+  @Override public double getDouble(int index) {
+    return CarbonUnsafe.unsafe.getDouble(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
new file mode 100644
index 0000000..ff28559
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Below class is responsible to store fixed length dimension data chunk in
+ * memory Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class UnsafeFixedLengthDimensionDataChunkStore
+    extends UnsafeAbstractDimensionDataChunkStore {
+
+  /**
+   * Size of each value
+   */
+  private int columnValueSize;
+
+  /**
+   * Constructor
+   *
+   * @param columnValueSize value of each column
+   * @param isInvertedIdex  is inverted index present
+   * @param numberOfRows    total number of rows
+   */
+  public UnsafeFixedLengthDimensionDataChunkStore(long totalDataSize, int columnValueSize,
+      boolean isInvertedIdex, int numberOfRows) {
+    super(totalDataSize, isInvertedIdex, numberOfRows);
+    this.columnValueSize = columnValueSize;
+  }
+
+  /**
+   * Below method will be used to get the row based inverted index
+   *
+   * @param rowId Inverted index
+   */
+  @Override public byte[] getRow(int rowId) {
+    // if column was explicitly sorted we need to get the rowid based inverted index reverse
+    if (isExplicitSorted) {
+      rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+              * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+    }
+    // creating a row
+    byte[] data = new byte[columnValueSize];
+    //copy the row from memory block based on offset
+    // offset position will be index * each column value length
+    CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), data,
+        CarbonUnsafe.BYTE_ARRAY_OFFSET, columnValueSize);
+    return data;
+  }
+
+  /**
+   * Below method will be used to get the surrogate key of the based on the row
+   * id passed
+   *
+   * @param rowId row id
+   * @return surrogate key
+   */
+  @Override public int getSurrogate(int index) {
+    // if column was explicitly sorted we need to get the rowid based inverted index reverse
+    if (isExplicitSorted) {
+      index = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (index
+              * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+    }
+    // below part is to convert the byte array to surrogate value
+    int startOffsetOfData = index * columnValueSize;
+    int surrogate = 0;
+    for (int i = 0; i < columnValueSize; i++) {
+      surrogate <<= 8;
+      surrogate ^= CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + startOffsetOfData) & 0xFF;
+      startOffsetOfData++;
+    }
+    return surrogate;
+  }
+
+  /**
+   * Below method will be used to fill the row values to buffer array
+   *
+   * @param rowId  row id of the data to be filled
+   * @param data   buffer in which data will be filled
+   * @param offset off the of the buffer
+   */
+  @Override public void fillRow(int rowId, byte[] buffer, int offset) {
+    // if column was explicitly sorted we need to get the rowid based inverted index reverse
+    if (isExplicitSorted) {
+      rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+              * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+    }
+    //copy the row from memory block based on offset
+    // offset position will be index * each column value length
+    CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), buffer,
+        CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, columnValueSize);
+  }
+
+  /**
+   * @return size of each column value
+   */
+  @Override public int getColumnValueSize() {
+    return columnValueSize;
+  }
+
+  /**
+   * to compare the two byte array
+   *
+   * @param index        index of first byte array
+   * @param compareValue value of to be compared
+   * @return compare result
+   */
+  @Override public int compareTo(int index, byte[] compareValue) {
+    // based on index we need to calculate the actual position in memory block
+    index = index * columnValueSize;
+    int compareResult = 0;
+    for (int i = 0; i < compareValue.length; i++) {
+      compareResult = (CarbonUnsafe.unsafe
+          .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index)
+          & 0xff) - (compareValue[i] & 0xff);
+      if (compareResult != 0) {
+        break;
+      }
+      index++;
+    }
+    return compareResult;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
new file mode 100644
index 0000000..0111648
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsible for storing int array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeIntMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<int[]> {
+
+  public UnsafeIntMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put int array data to memory
+   *
+   * @param data
+   */
+  @Override public void putData(int[] data) {
+    assert (!this.isMemoryOccupied);
+    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+        .allocate(data.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
+    // copy the data to memory
+    CarbonUnsafe.unsafe
+        .copyMemory(data, CarbonUnsafe.INT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+    this.isMemoryOccupied = true;
+  }
+
+  /**
+   * to get the int value
+   *
+   * @param index
+   * @return int value based on index
+   */
+  @Override public int getInt(int index) {
+    return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
new file mode 100644
index 0000000..d3bed25
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsible for storing long array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeLongMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<long[]> {
+
+  public UnsafeLongMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put long array data to memory
+   *
+   * @param data
+   */
+  @Override public void putData(long[] data) {
+    assert (!this.isMemoryOccupied);
+    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+        .allocate(data.length * CarbonCommonConstants.LONG_SIZE_IN_BYTE);
+    // copy the data to memory
+    CarbonUnsafe.unsafe
+        .copyMemory(data, CarbonUnsafe.LONG_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+    this.isMemoryOccupied = true;
+  }
+
+  /**
+   * to get the long value
+   *
+   * @param index
+   * @return long value based on index
+   */
+  @Override public long getLong(int index) {
+    return CarbonUnsafe.unsafe.getLong(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.LONG_SIZE_IN_BYTE));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
new file mode 100644
index 0000000..ba865cd
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsible for storing short array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeShortMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<short[]> {
+
+  public UnsafeShortMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put short array data to memory
+   *
+   * @param data
+   */
+  @Override public void putData(short[] data) {
+    assert (!this.isMemoryOccupied);
+    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+        .allocate(data.length * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+    // copy the data to memory
+    CarbonUnsafe.unsafe
+        .copyMemory(data, CarbonUnsafe.SHORT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+    this.isMemoryOccupied = true;
+  }
+
+  /**
+   * to get the short value
+   *
+   * @param index
+   * @return shot value based on index
+   */
+  @Override public short getShort(int index) {
+    return CarbonUnsafe.unsafe.getShort(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
new file mode 100644
index 0000000..c6155e7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Below class is responsible to store variable length dimension data chunk in
+ * memory Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class UnsafeVariableLengthDimesionDataChunkStore
+    extends UnsafeAbstractDimensionDataChunkStore {
+
+  /**
+   * total number of rows
+   */
+  private int numberOfRows;
+
+  /**
+   * pointers offsets
+   */
+  private long dataPointersOffsets;
+
+  public UnsafeVariableLengthDimesionDataChunkStore(long totalSize, boolean isInvertedIdex,
+      int numberOfRows) {
+    super(totalSize, isInvertedIdex, numberOfRows);
+    this.numberOfRows = numberOfRows;
+  }
+
+  /**
+   * Below method will be used to put the rows and its metadata in offheap
+   *
+   * @param invertedIndex        inverted index to be stored
+   * @param invertedIndexReverse inverted index reverse to be stored
+   * @param data                 data to be stored
+   */
+  @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
+      byte[] data) {
+    // first put the data, inverted index and reverse inverted index to memory
+    super.putArray(invertedIndex, invertedIndexReverse, data);
+    // position from where offsets will start
+    this.dataPointersOffsets = this.invertedIndexReverseOffset;
+    if (isExplicitSorted) {
+      this.dataPointersOffsets += numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE;
+    }
+    // As data is of variable length and data format is
+    // <length in short><data><length in short><data>
+    // we need to store offset of each data so data can be accessed directly
+    // for example:
+    //data = {0,5,1,2,3,4,5,0,6,0,1,2,3,4,5,0,2,8,9}
+    //so value stored in offset will be position of actual data
+    // [2,9,17]
+    // to store this value we need to get the actual data length + 2 bytes used for storing the
+    // length
+
+    // start position will be used to store the current data position
+    int startOffset = 0;
+    // position from where offsets will start
+    long pointerOffsets = this.dataPointersOffsets;
+    // as first position will be start from 2 byte as data is stored first in the memory block
+    // we need to skip first two bytes this is because first two bytes will be length of the data
+    // which we have to skip
+    CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
+        CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+    // incrementing the pointers as first value is already filled and as we are storing as int
+    // we need to increment the 4 bytes to set the position of the next value to set
+    pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE;
+    // creating a byte buffer which will wrap the length of the row
+    // using byte buffer as unsafe will return bytes in little-endian encoding
+    ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+    // store length of data
+    byte[] length = new byte[CarbonCommonConstants.SHORT_SIZE_IN_BYTE];
+    // as first offset is already stored, we need to start from the 2nd row in data array
+    for (int i = 1; i < numberOfRows; i++) {
+      // first copy the length of previous row
+      CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + startOffset, length, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+          CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+      buffer.put(length);
+      buffer.flip();
+      // so current row position will be
+      // previous row length + 2 bytes used for storing previous row data
+      startOffset += CarbonCommonConstants.SHORT_SIZE_IN_BYTE + buffer.getShort();
+      // as same byte buffer is used to avoid creating many byte buffer for each row
+      // we need to clear the byte buffer
+      buffer.clear();
+      // now put the offset of current row, here we need to add 2 more bytes as current will
+      // also have length part so we have to skip length
+      CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
+          startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+      // incrementing the pointers as first value is already filled and as we are storing as int
+      // we need to increment the 4 bytes to set the position of the next value to set
+      pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE;
+    }
+
+  }
+
+  /**
+   * Below method will be used to get the row based on row id passed
+   *
+   * @param index
+   * @return row
+   */
+  @Override public byte[] getRow(int rowId) {
+    byte[] data = null;
+    try {
+      // if column was explicitly sorted we need to get the rowid based inverted index reverse
+      if (isExplicitSorted) {
+        rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+                * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+      }
+      // now to get the row from memory block we need to do following thing
+      // 1. first get the current offset
+      // 2. if it's not a last row- get the next row offset
+      // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
+      // else subtract the current row offset + 2 bytes(to skip the data length)
+      // with complete data length
+      int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (rowId
+              * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+      short length = 0;
+      // calculating the length of data
+      if (rowId < numberOfRows - 1) {
+        int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((rowId + 1)
+                * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+        length = (short) (OffsetOfNextdata - (currentDataOffset
+            + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+      } else {
+        // for last record we need to subtract with data length
+        length = (short) (this.dataLength - currentDataOffset);
+      }
+      data = new byte[length];
+      CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + currentDataOffset, data,
+          CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
+    } catch (Throwable t) {
+      System.out.println();
+    }
+    return data;
+  }
+
+  /**
+   * to compare the two byte array
+   *
+   * @param index        index of first byte array
+   * @param compareValue value of to be compared
+   * @return compare result
+   */
+  @Override public int compareTo(int index, byte[] compareValue) {
+    // now to get the row from memory block we need to do following thing
+    // 1. first get the current offset
+    // 2. if it's not a last row- get the next row offset
+    // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
+    // else subtract the current row offset
+    // with complete data length get the offset of set of data
+    int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (index
+            * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+    short length = 0;
+    // calculating the length of data
+    if (index < numberOfRows - 1) {
+      int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((index + 1)
+              * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+      length = (short) (OffsetOfNextdata - (currentDataOffset
+          + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+    } else {
+      // for last record we need to subtract with data length
+      length = (short) (this.dataLength - currentDataOffset);
+    }
+    // as this class handles this variable length data, so filter value can be
+    // smaller or bigger than than actual data, so we need to take the smaller length
+    int compareResult = 0;
+    int compareLength = length < compareValue.length ? length : compareValue.length;
+    for (int i = 0; i < compareLength; i++) {
+      compareResult = (CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + currentDataOffset) & 0xff) - (compareValue[i]
+          & 0xff);
+      // if compare result is not equal we can break
+      if (compareResult != 0) {
+        break;
+      }
+      // increment the offset by one as comparison is done byte by byte
+      currentDataOffset++;
+    }
+    return compareResult;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index f91a75c..041f5ed 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1081,6 +1081,7 @@ public final class CarbonCommonConstants {
 
   /**
    * Default carbon dictionary server port
+
    */
   public static final String DICTIONARY_SERVER_PORT_DEFAULT = "2030";
 
@@ -1094,6 +1095,26 @@ public final class CarbonCommonConstants {
    */
   public static final int DEFAULT_COLUMN_LENGTH = 100000;
 
+  /**
+   * property for enabling unsafe based query processing
+   */
+  public static final String ENABLE_UNSAFE_IN_QUERY_EXECUTION = "enable.unsafe.in.query.processing";
+
+  /**
+   * default property of unsafe processing
+   */
+  public static final String ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE = "false";
+
+  /**
+   * property for offheap based processing
+   */
+  public static final String USE_OFFHEAP_IN_QUERY_PROCSSING = "use.offheap.in.query.processing";
+
+  /**
+   * default value of offheap based processing
+   */
+  public static final String USE_OFFHEAP_IN_QUERY_PROCSSING_DEFAULT = "true";
+
   private CarbonCommonConstants() {
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
index a174a18..e40dadd 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
@@ -31,5 +31,4 @@ public interface NodeMeasureDataStore {
    * @return writable array (compressed or normal)
    */
   byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
-
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java
index a3d922d..1b90bc6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java
@@ -25,24 +25,36 @@ public interface Compressor {
 
   byte[] unCompressByte(byte[] compInput);
 
+  byte[] unCompressByte(byte[] compInput, int offset, int length);
+
   byte[] compressShort(short[] unCompInput);
 
   short[] unCompressShort(byte[] compInput);
 
+  short[] unCompressShort(byte[] compInput, int offset, int lenght);
+
   byte[] compressInt(int[] unCompInput);
 
   int[] unCompressInt(byte[] compInput);
 
+  int[] unCompressInt(byte[] compInput, int offset, int length);
+
   byte[] compressLong(long[] unCompInput);
 
   long[] unCompressLong(byte[] compInput);
 
+  long[] unCompressLong(byte[] compInput, int offset, int length);
+
   byte[] compressFloat(float[] unCompInput);
 
   float[] unCompressFloat(byte[] compInput);
 
+  float[] unCompressFloat(byte[] compInput, int offset, int length);
+
   byte[] compressDouble(double[] unCompInput);
 
   double[] unCompressDouble(byte[] compInput);
 
+  double[] unCompressDouble(byte[] compInput, int offset, int length);
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java
index 110010f..5160879 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java
@@ -24,19 +24,29 @@ import org.apache.carbondata.core.util.CarbonProperties;
 
 public class CompressorFactory {
 
-  private static Compressor snappy;
+  private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory();
 
-  public static Compressor getInstance() {
-    String compressor = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.COMPRESSOR,
-        CarbonCommonConstants.DEFAULT_COMPRESSOR);
-    switch (compressor) {
+  private final Compressor compressor;
+
+  private CompressorFactory() {
+    String compressorType = CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR);
+    switch (compressorType) {
       case "snappy":
-        if (snappy == null) {
-          snappy = new SnappyCompressor();
-        }
-        return snappy;
+        compressor = new SnappyCompressor();
+        break;
       default:
-        throw new RuntimeException("Unsupported compressor: " + compressor);
+        throw new RuntimeException(
+            "Invalid compressor type provided! Please provide valid compressor type");
     }
   }
+
+  public static CompressorFactory getInstance() {
+    return COMPRESSOR_FACTORY;
+  }
+
+  public Compressor getCompressor() {
+    return compressor;
+  }
+
 }



Mime
View raw message