carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qiang...@apache.org
Subject [1/2] carbondata git commit: rename RLE
Date Sat, 17 Jun 2017 17:30:12 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 7bb822031 -> b7518aa19


rename RLE


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6a09ee84
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6a09ee84
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6a09ee84

Branch: refs/heads/master
Commit: 6a09ee84c6efde7675f1e49aaa1adf1a7ff1ba34
Parents: 7bb8220
Author: jackylk <jacky.likun@huawei.com>
Authored: Tue Jun 13 21:56:06 2017 +0800
Committer: QiangCai <qiangcai@qq.com>
Committed: Sun Jun 18 01:27:02 2017 +0800

----------------------------------------------------------------------
 .../columnar/BlockIndexerStorageForInt.java     | 141 +++++++++----------
 .../BlockIndexerStorageForNoInvertedIndex.java  | 104 --------------
 ...kIndexerStorageForNoInvertedIndexForInt.java | 103 ++++++++++++++
 ...ndexerStorageForNoInvertedIndexForShort.java |  48 +++----
 .../columnar/BlockIndexerStorageForShort.java   | 135 +++++++++---------
 .../datastore/columnar/ColumnWithIntIndex.java  |  80 -----------
 .../columnar/ColumnWithIntIndexForHighCard.java |  47 -------
 .../datastore/columnar/ColumnWithRowId.java     |  77 ++++++++++
 .../columnar/ColumnWithRowIdForHighCard.java    |  47 +++++++
 .../columnar/ColumnWithShortIndex.java          |  76 ----------
 .../ColumnWithShortIndexForNoDictionay.java     |  46 ------
 .../core/datastore/columnar/IndexStorage.java   |   8 +-
 .../columnar/ColGroupBlockStorage.java          |  25 ++--
 .../processing/store/TablePageEncoder.java      |  16 +--
 .../writer/v1/CarbonFactDataWriterImplV1.java   |  10 +-
 .../writer/v3/CarbonFactDataWriterImplV3.java   |  10 +-
 16 files changed, 421 insertions(+), 552 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
index 826f6d2..78e6d35 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
@@ -27,61 +27,56 @@ import org.apache.carbondata.core.util.ByteUtil;
 public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
   private boolean alreadySorted;
 
-  private int[] dataAfterComp;
+  private int[] rowIdPage;
 
-  private int[] indexMap;
+  private int[] rowIdRlePage;
 
-  private byte[][] keyBlock;
+  private byte[][] dataPage;
 
-  private int[] dataIndexMap;
+  private int[] dataRlePage;
 
   private int totalSize;
 
-  public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary,
+  public BlockIndexerStorageForInt(byte[][] dataPage, boolean rleOnData, boolean isNoDictionary,
       boolean isSortRequired) {
-    ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
+    ColumnWithRowId<Integer>[] dataWithRowId = createColumnWithRowId(dataPage, isNoDictionary);
     if (isSortRequired) {
-      Arrays.sort(columnWithIndexs);
+      Arrays.sort(dataWithRowId);
     }
-    compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
-    if (compressData) {
-      compressDataMyOwnWay(columnWithIndexs);
+    int[] rowIds = extractDataAndReturnRowIds(dataWithRowId, dataPage);
+    rleEncodeOnRowId(rowIds);
+    if (rleOnData) {
+      rleEncodeOnData(dataWithRowId);
     }
   }
 
   /**
-   * Create an object with each column array and respective index
-   *
-   * @return
+   * Create an object with each column array and respective rowId
    */
-  private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
+  private ColumnWithRowId<Integer>[] createColumnWithRowId(byte[][] dataPage,
       boolean isNoDictionary) {
-    ColumnWithIntIndex[] columnWithIndexs;
+    ColumnWithRowId<Integer>[] columnWithRowId = new ColumnWithRowId[dataPage.length];
     if (isNoDictionary) {
-      columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
-      for (int i = 0; i < columnWithIndexs.length; i++) {
-        columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i);
+      for (int i = 0; i < columnWithRowId.length; i++) {
+        columnWithRowId[i] = new ColumnWithRowIdForHighCard<>(dataPage[i], i);
       }
-
     } else {
-      columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
-      for (int i = 0; i < columnWithIndexs.length; i++) {
-        columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
+      for (int i = 0; i < columnWithRowId.length; i++) {
+        columnWithRowId[i] = new ColumnWithRowId<>(dataPage[i], i);
       }
     }
-
-    return columnWithIndexs;
+    return columnWithRowId;
   }
 
-  private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs,
+  private int[] extractDataAndReturnRowIds(ColumnWithRowId<Integer>[] dataWithRowId,
       byte[][] keyBlock) {
-    int[] indexes = new int[columnWithIndexs.length];
-    for (int i = 0; i < indexes.length; i++) {
-      indexes[i] = columnWithIndexs[i].getIndex();
-      keyBlock[i] = columnWithIndexs[i].getColumn();
+    int[] rowId = new int[dataWithRowId.length];
+    for (int i = 0; i < rowId.length; i++) {
+      rowId[i] = dataWithRowId[i].getIndex();
+      keyBlock[i] = dataWithRowId[i].getColumn();
     }
-    this.keyBlock = keyBlock;
-    return indexes;
+    this.dataPage = keyBlock;
+    return rowId;
   }
 
   /**
@@ -92,41 +87,41 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
    * sequential numbers then the same array it returns with empty second
    * array.
    *
-   * @param indexes
+   * @param rowIds
    */
-  public void compressMyOwnWay(int[] indexes) {
+  public void rleEncodeOnRowId(int[] rowIds) {
     List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
     List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
     int k = 0;
     int i = 1;
-    for (; i < indexes.length; i++) {
-      if (indexes[i] - indexes[i - 1] == 1) {
+    for (; i < rowIds.length; i++) {
+      if (rowIds[i] - rowIds[i - 1] == 1) {
         k++;
       } else {
         if (k > 0) {
           map.add((list.size()));
-          list.add(indexes[i - k - 1]);
-          list.add(indexes[i - 1]);
+          list.add(rowIds[i - k - 1]);
+          list.add(rowIds[i - 1]);
         } else {
-          list.add(indexes[i - 1]);
+          list.add(rowIds[i - 1]);
         }
         k = 0;
       }
     }
     if (k > 0) {
       map.add((list.size()));
-      list.add(indexes[i - k - 1]);
-      list.add(indexes[i - 1]);
+      list.add(rowIds[i - k - 1]);
+      list.add(rowIds[i - 1]);
     } else {
-      list.add(indexes[i - 1]);
+      list.add(rowIds[i - 1]);
     }
-    dataAfterComp = convertToArray(list);
-    if (indexes.length == dataAfterComp.length) {
-      indexMap = new int[0];
+    rowIdPage = convertToArray(list);
+    if (rowIds.length == rowIdPage.length) {
+      rowIdRlePage = new int[0];
     } else {
-      indexMap = convertToArray(map);
+      rowIdRlePage = convertToArray(map);
     }
-    if (dataAfterComp.length == 2 && indexMap.length == 1) {
+    if (rowIdPage.length == 2 && rowIdRlePage.length == 1) {
       alreadySorted = true;
     }
   }
@@ -147,38 +142,38 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
   }
 
   /**
-   * @return the dataAfterComp
+   * @return the rowIdPage
    */
-  public int[] getDataAfterComp() {
-    return dataAfterComp;
+  public int[] getRowIdPage() {
+    return rowIdPage;
   }
 
   /**
-   * @return the indexMap
+   * @return the rowIdRlePage
    */
-  public int[] getIndexMap() {
-    return indexMap;
+  public int[] getRowIdRlePage() {
+    return rowIdRlePage;
   }
 
   /**
-   * @return the keyBlock
+   * @return the dataPage
    */
-  public byte[][] getKeyBlock() {
-    return keyBlock;
+  public byte[][] getDataPage() {
+    return dataPage;
   }
 
-  private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) {
-    byte[] prvKey = indexes[0].getColumn();
-    List<ColumnWithIntIndex> list =
-        new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-    list.add(indexes[0]);
+  private void rleEncodeOnData(ColumnWithRowId[] dataWithRowId) {
+    byte[] prvKey = dataWithRowId[0].getColumn();
+    List<ColumnWithRowId> list =
+        new ArrayList<ColumnWithRowId>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    list.add(dataWithRowId[0]);
     int counter = 1;
     int start = 0;
     List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-    for (int i = 1; i < indexes.length; i++) {
-      if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
-        prvKey = indexes[i].getColumn();
-        list.add(indexes[i]);
+    for (int i = 1; i < dataWithRowId.length; i++) {
+      if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, dataWithRowId[i].getColumn()) != 0) {
+        prvKey = dataWithRowId[i].getColumn();
+        list.add(dataWithRowId[i]);
         map.add(start);
         map.add(counter);
         start += counter;
@@ -189,15 +184,15 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
     }
     map.add(start);
     map.add(counter);
-    this.keyBlock = convertToKeyArray(list);
-    if (indexes.length == keyBlock.length) {
-      dataIndexMap = new int[0];
+    this.dataPage = convertToDataPage(list);
+    if (dataWithRowId.length == dataPage.length) {
+      dataRlePage = new int[0];
     } else {
-      dataIndexMap = convertToArray(map);
+      dataRlePage = convertToArray(map);
     }
   }
 
-  private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) {
+  private byte[][] convertToDataPage(List<ColumnWithRowId> list) {
     byte[][] shortArray = new byte[list.size()][];
     for (int i = 0; i < shortArray.length; i++) {
       shortArray[i] = list.get(i).getColumn();
@@ -206,8 +201,8 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
     return shortArray;
   }
 
-  @Override public int[] getDataIndexMap() {
-    return dataIndexMap;
+  public int[] getDataRlePage() {
+    return dataRlePage;
   }
 
   @Override public int getTotalSize() {
@@ -215,10 +210,10 @@ public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
   }
 
   @Override public byte[] getMin() {
-    return keyBlock[0];
+    return dataPage[0];
   }
 
   @Override public byte[] getMax() {
-    return keyBlock[keyBlock.length - 1];
+    return dataPage[dataPage.length - 1];
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
deleted file mode 100644
index f36ee97..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.columnar;
-
-import org.apache.carbondata.core.datastore.columnar.IndexStorage;
-import org.apache.carbondata.core.util.ByteUtil;
-
-/**
- * Below class will be used to for no inverted index
- */
-public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
-
-  /**
-   * column data
-   */
-  private byte[][] keyBlock;
-
-  /**
-   * total number of rows
-   */
-  private int totalSize;
-
-  private byte[] min;
-  private byte[] max;
-
-  public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput) {
-    this.keyBlock = keyBlockInput;
-    min = keyBlock[0];
-    max = keyBlock[0];
-    totalSize += keyBlock[0].length;
-    int minCompare = 0;
-    int maxCompare = 0;
-    for (int i = 1; i < keyBlock.length; i++) {
-      totalSize += keyBlock[i].length;
-      minCompare = ByteUtil.compare(min, keyBlock[i]);
-      maxCompare = ByteUtil.compare(max, keyBlock[i]);
-      if (minCompare > 0) {
-        min = keyBlock[i];
-      }
-      if (maxCompare < 0) {
-        max = keyBlock[i];
-      }
-    }
-  }
-
-  @Override public int[] getDataIndexMap() {
-    return new int[0];
-  }
-
-  @Override public int getTotalSize() {
-    return totalSize;
-  }
-
-  @Override public boolean isAlreadySorted() {
-    return true;
-  }
-
-  /**
-   * no use
-   *
-   * @return
-   */
-  @Override public int[] getDataAfterComp() {
-    return new int[0];
-  }
-
-  /**
-   * no use
-   *
-   * @return
-   */
-  @Override public int[] getIndexMap() {
-    return new int[0];
-  }
-
-  /**
-   * @return the keyBlock
-   */
-  public byte[][] getKeyBlock() {
-    return keyBlock;
-  }
-
-  @Override public byte[] getMin() {
-    return min;
-  }
-
-  @Override public byte[] getMax() {
-    return max;
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java
new file mode 100644
index 0000000..c168614
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForInt.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.columnar;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Below class will be used to for no inverted index
+ */
+public class BlockIndexerStorageForNoInvertedIndexForInt implements IndexStorage<int[]> {
+
+  /**
+   * column data
+   */
+  private byte[][] dataPage;
+
+  /**
+   * total number of rows
+   */
+  private int totalSize;
+
+  private byte[] min;
+  private byte[] max;
+
+  public BlockIndexerStorageForNoInvertedIndexForInt(byte[][] dataPage) {
+    this.dataPage = dataPage;
+    min = this.dataPage[0];
+    max = this.dataPage[0];
+    totalSize += this.dataPage[0].length;
+    int minCompare = 0;
+    int maxCompare = 0;
+    for (int i = 1; i < this.dataPage.length; i++) {
+      totalSize += this.dataPage[i].length;
+      minCompare = ByteUtil.compare(min, this.dataPage[i]);
+      maxCompare = ByteUtil.compare(max, this.dataPage[i]);
+      if (minCompare > 0) {
+        min = this.dataPage[i];
+      }
+      if (maxCompare < 0) {
+        max = this.dataPage[i];
+      }
+    }
+  }
+
+  public int[] getDataRlePage() {
+    return new int[0];
+  }
+
+  @Override public int getTotalSize() {
+    return totalSize;
+  }
+
+  @Override public boolean isAlreadySorted() {
+    return true;
+  }
+
+  /**
+   * no use
+   *
+   * @return
+   */
+  public int[] getRowIdPage() {
+    return new int[0];
+  }
+
+  /**
+   * no use
+   *
+   * @return
+   */
+  public int[] getRowIdRlePage() {
+    return new int[0];
+  }
+
+  /**
+   * @return the dataPage
+   */
+  public byte[][] getDataPage() {
+    return dataPage;
+  }
+
+  @Override public byte[] getMin() {
+    return min;
+  }
+
+  @Override public byte[] getMax() {
+    return max;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
index f14338b..5712655 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndexForShort.java
@@ -26,7 +26,7 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
   /**
    * column data
    */
-  private byte[][] keyBlock;
+  private byte[][] dataPage;
 
   /**
    * total number of rows
@@ -36,44 +36,44 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
   private byte[] min;
   private byte[] max;
 
-  public BlockIndexerStorageForNoInvertedIndexForShort(byte[][] keyBlockInput,
+  public BlockIndexerStorageForNoInvertedIndexForShort(byte[][] dataPage,
       boolean isNoDictonary) {
-    this.keyBlock = keyBlockInput;
-    min = keyBlock[0];
-    max = keyBlock[0];
-    totalSize += keyBlock[0].length;
+    this.dataPage = dataPage;
+    min = this.dataPage[0];
+    max = this.dataPage[0];
+    totalSize += this.dataPage[0].length;
     int minCompare = 0;
     int maxCompare = 0;
     if (!isNoDictonary) {
-      for (int i = 1; i < keyBlock.length; i++) {
-        totalSize += keyBlock[i].length;
-        minCompare = ByteUtil.compare(min, keyBlock[i]);
-        maxCompare = ByteUtil.compare(max, keyBlock[i]);
+      for (int i = 1; i < this.dataPage.length; i++) {
+        totalSize += this.dataPage[i].length;
+        minCompare = ByteUtil.compare(min, this.dataPage[i]);
+        maxCompare = ByteUtil.compare(max, this.dataPage[i]);
         if (minCompare > 0) {
-          min = keyBlock[i];
+          min = this.dataPage[i];
         }
         if (maxCompare < 0) {
-          max = keyBlock[i];
+          max = this.dataPage[i];
         }
       }
     } else {
-      for (int i = 1; i < keyBlock.length; i++) {
-        totalSize += keyBlock[i].length;
+      for (int i = 1; i < this.dataPage.length; i++) {
+        totalSize += this.dataPage[i].length;
         minCompare = ByteUtil.UnsafeComparer.INSTANCE
-            .compareTo(min, 2, min.length - 2, keyBlock[i], 2, keyBlock[i].length - 2);
+            .compareTo(min, 2, min.length - 2, this.dataPage[i], 2, this.dataPage[i].length - 2);
         maxCompare = ByteUtil.UnsafeComparer.INSTANCE
-            .compareTo(max, 2, max.length - 2, keyBlock[i], 2, keyBlock[i].length - 2);
+            .compareTo(max, 2, max.length - 2, this.dataPage[i], 2, this.dataPage[i].length - 2);
         if (minCompare > 0) {
-          min = keyBlock[i];
+          min = this.dataPage[i];
         }
         if (maxCompare < 0) {
-          max = keyBlock[i];
+          max = this.dataPage[i];
         }
       }
     }
   }
 
-  @Override public short[] getDataIndexMap() {
+  public short[] getDataRlePage() {
     return new short[0];
   }
 
@@ -90,7 +90,7 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
    *
    * @return
    */
-  @Override public short[] getDataAfterComp() {
+  public short[] getRowIdPage() {
     return new short[0];
   }
 
@@ -99,15 +99,15 @@ public class BlockIndexerStorageForNoInvertedIndexForShort implements IndexStora
    *
    * @return
    */
-  @Override public short[] getIndexMap() {
+  public short[] getRowIdRlePage() {
     return new short[0];
   }
 
   /**
-   * @return the keyBlock
+   * @return the dataPage
    */
-  public byte[][] getKeyBlock() {
-    return keyBlock;
+  public byte[][] getDataPage() {
+    return dataPage;
   }
 
   @Override public byte[] getMin() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
index aaea62b..2ffb610 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
@@ -27,58 +27,57 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
 
   private boolean alreadySorted;
 
-  private short[] dataAfterComp;
+  private short[] rowIdPage;
 
-  private short[] indexMap;
+  private short[] rowIdRlePage;
 
-  private byte[][] keyBlock;
+  private byte[][] dataPage;
 
-  private short[] dataIndexMap;
+  private short[] dataRlePage;
 
   private int totalSize;
 
-  public BlockIndexerStorageForShort(byte[][] keyBlock, boolean compressData,
+  public BlockIndexerStorageForShort(byte[][] dataPage, boolean rleOnData,
       boolean isNoDictionary, boolean isSortRequired) {
-    ColumnWithShortIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
+    ColumnWithRowId<Short>[] dataWithRowId = createColumnWithRowId(dataPage, isNoDictionary);
     if (isSortRequired) {
-      Arrays.sort(columnWithIndexs);
+      Arrays.sort(dataWithRowId);
     }
-    compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
-    if (compressData) {
-      compressDataMyOwnWay(columnWithIndexs);
+    short[] rowIds = extractDataAndReturnRowId(dataWithRowId, dataPage);
+    rleEncodeOnRowId(rowIds);
+    if (rleOnData) {
+      rleEncodeOnData(dataWithRowId);
     }
   }
 
   /**
-   * Create an object with each column array and respective index
+   * Create an object with each column array and respective rowId
    *
    * @return
    */
-  private ColumnWithShortIndex[] createColumnWithIndexArray(byte[][] keyBlock,
+  private ColumnWithRowId<Short>[] createColumnWithRowId(byte[][] dataPage,
       boolean isNoDictionary) {
-    ColumnWithShortIndex[] columnWithIndexs;
+    ColumnWithRowId<Short>[] columnWithIndexs = new ColumnWithRowId[dataPage.length];
     if (isNoDictionary) {
-      columnWithIndexs = new ColumnWithShortIndex[keyBlock.length];
       for (short i = 0; i < columnWithIndexs.length; i++) {
-        columnWithIndexs[i] = new ColumnWithShortIndexForNoDictionay(keyBlock[i], i);
+        columnWithIndexs[i] = new ColumnWithRowIdForHighCard<>(dataPage[i], i);
       }
     } else {
-      columnWithIndexs = new ColumnWithShortIndex[keyBlock.length];
       for (short i = 0; i < columnWithIndexs.length; i++) {
-        columnWithIndexs[i] = new ColumnWithShortIndex(keyBlock[i], i);
+        columnWithIndexs[i] = new ColumnWithRowId<>(dataPage[i], i);
       }
     }
     return columnWithIndexs;
   }
 
-  private short[] extractDataAndReturnIndexes(ColumnWithShortIndex[] columnWithIndexs,
-      byte[][] keyBlock) {
-    short[] indexes = new short[columnWithIndexs.length];
+  private short[] extractDataAndReturnRowId(ColumnWithRowId<Short>[] dataWithRowId,
+      byte[][] dataPage) {
+    short[] indexes = new short[dataWithRowId.length];
     for (int i = 0; i < indexes.length; i++) {
-      indexes[i] = columnWithIndexs[i].getIndex();
-      keyBlock[i] = columnWithIndexs[i].getColumn();
+      indexes[i] = dataWithRowId[i].getIndex();
+      dataPage[i] = dataWithRowId[i].getColumn();
     }
-    this.keyBlock = keyBlock;
+    this.dataPage = dataPage;
     return indexes;
   }
 
@@ -90,46 +89,46 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
    * sequential numbers then the same array it returns with empty second
    * array.
    *
-   * @param indexes
+   * @param rowIds
    */
-  private void compressMyOwnWay(short[] indexes) {
+  private void rleEncodeOnRowId(short[] rowIds) {
     List<Short> list = new ArrayList<Short>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
     List<Short> map = new ArrayList<Short>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
     int k = 0;
     int i = 1;
-    for (; i < indexes.length; i++) {
-      if (indexes[i] - indexes[i - 1] == 1) {
+    for (; i < rowIds.length; i++) {
+      if (rowIds[i] - rowIds[i - 1] == 1) {
         k++;
       } else {
         if (k > 0) {
           map.add(((short) list.size()));
-          list.add(indexes[i - k - 1]);
-          list.add(indexes[i - 1]);
+          list.add(rowIds[i - k - 1]);
+          list.add(rowIds[i - 1]);
         } else {
-          list.add(indexes[i - 1]);
+          list.add(rowIds[i - 1]);
         }
         k = 0;
       }
     }
     if (k > 0) {
       map.add(((short) list.size()));
-      list.add(indexes[i - k - 1]);
-      list.add(indexes[i - 1]);
+      list.add(rowIds[i - k - 1]);
+      list.add(rowIds[i - 1]);
     } else {
-      list.add(indexes[i - 1]);
+      list.add(rowIds[i - 1]);
     }
-    double compressionPercentage = (((list.size() + map.size()) * 100) / indexes.length);
+    double compressionPercentage = (((list.size() + map.size()) * 100) / rowIds.length);
     if (compressionPercentage > 70) {
-      dataAfterComp = indexes;
+      rowIdPage = rowIds;
     } else {
-      dataAfterComp = convertToArray(list);
+      rowIdPage = convertToArray(list);
     }
-    if (indexes.length == dataAfterComp.length) {
-      indexMap = new short[0];
+    if (rowIds.length == rowIdPage.length) {
+      rowIdRlePage = new short[0];
     } else {
-      indexMap = convertToArray(map);
+      rowIdRlePage = convertToArray(map);
     }
-    if (dataAfterComp.length == 2 && indexMap.length == 1) {
+    if (rowIdPage.length == 2 && rowIdRlePage.length == 1) {
       alreadySorted = true;
     }
   }
@@ -150,37 +149,37 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
   }
 
   /**
-   * @return the dataAfterComp
+   * @return the rowIdPage
    */
-  public short[] getDataAfterComp() {
-    return dataAfterComp;
+  public short[] getRowIdPage() {
+    return rowIdPage;
   }
 
   /**
-   * @return the indexMap
+   * @return the rowIdRlePage
    */
-  public short[] getIndexMap() {
-    return indexMap;
+  public short[] getRowIdRlePage() {
+    return rowIdRlePage;
   }
 
   /**
-   * @return the keyBlock
+   * @return the dataPage
    */
-  public byte[][] getKeyBlock() {
-    return keyBlock;
+  public byte[][] getDataPage() {
+    return dataPage;
   }
 
-  private void compressDataMyOwnWay(ColumnWithShortIndex[] indexes) {
-    byte[] prvKey = indexes[0].getColumn();
-    List<ColumnWithShortIndex> list = new ArrayList<ColumnWithShortIndex>(indexes.length / 2);
-    list.add(indexes[0]);
+  private void rleEncodeOnData(ColumnWithRowId<Short>[] dataWithRowId) {
+    byte[] prvKey = dataWithRowId[0].getColumn();
+    List<ColumnWithRowId> list = new ArrayList<>(dataWithRowId.length / 2);
+    list.add(dataWithRowId[0]);
     short counter = 1;
     short start = 0;
     List<Short> map = new ArrayList<Short>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-    for (int i = 1; i < indexes.length; i++) {
-      if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
-        prvKey = indexes[i].getColumn();
-        list.add(indexes[i]);
+    for (int i = 1; i < dataWithRowId.length; i++) {
+      if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, dataWithRowId[i].getColumn()) != 0) {
+        prvKey = dataWithRowId[i].getColumn();
+        list.add(dataWithRowId[i]);
         map.add(start);
         map.add(counter);
         start += counter;
@@ -193,17 +192,17 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
     map.add(counter);
     // if rle is index size is more than 70% then rle wont give any benefit
     // so better to avoid rle index and write data as it is
-    boolean useRle = (((list.size() + map.size()) * 100) / indexes.length) < 70;
+    boolean useRle = (((list.size() + map.size()) * 100) / dataWithRowId.length) < 70;
     if (useRle) {
-      this.keyBlock = convertToKeyArray(list);
-      dataIndexMap = convertToArray(map);
+      this.dataPage = convertToDataPage(list);
+      dataRlePage = convertToArray(map);
     } else {
-      this.keyBlock = convertToKeyArray(indexes);
-      dataIndexMap = new short[0];
+      this.dataPage = convertToDataPage(dataWithRowId);
+      dataRlePage = new short[0];
     }
   }
 
-  private byte[][] convertToKeyArray(ColumnWithShortIndex[] indexes) {
+  private byte[][] convertToDataPage(ColumnWithRowId[] indexes) {
     byte[][] shortArray = new byte[indexes.length][];
     for (int i = 0; i < shortArray.length; i++) {
       shortArray[i] = indexes[i].getColumn();
@@ -212,7 +211,7 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
     return shortArray;
   }
 
-  private byte[][] convertToKeyArray(List<ColumnWithShortIndex> list) {
+  private byte[][] convertToDataPage(List<ColumnWithRowId> list) {
     byte[][] shortArray = new byte[list.size()][];
     for (int i = 0; i < shortArray.length; i++) {
       shortArray[i] = list.get(i).getColumn();
@@ -221,8 +220,8 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
     return shortArray;
   }
 
-  @Override public short[] getDataIndexMap() {
-    return dataIndexMap;
+  public short[] getDataRlePage() {
+    return dataRlePage;
   }
 
   @Override public int getTotalSize() {
@@ -230,11 +229,11 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
   }
 
   @Override public byte[] getMin() {
-    return keyBlock[0];
+    return dataPage[0];
   }
 
   @Override public byte[] getMax() {
-    return keyBlock[keyBlock.length - 1];
+    return dataPage[dataPage.length - 1];
   }
 
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
deleted file mode 100644
index fa52e09..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> {
-  protected byte[] column;
-
-  private int index;
-
-  public ColumnWithIntIndex(byte[] column, int index) {
-    this.column = column;
-    this.index = index;
-  }
-
-  public ColumnWithIntIndex() {
-  }
-
-  /**
-   * @return the column
-   */
-  public byte[] getColumn() {
-    return column;
-  }
-
-  /**
-   * @param column the column to set
-   */
-  public void setColumn(byte[] column) {
-    this.column = column;
-  }
-
-  /**
-   * @return the index
-   */
-  public int getIndex() {
-    return index;
-  }
-
-  /**
-   * @param index the index to set
-   */
-  public void setIndex(int index) {
-    this.index = index;
-  }
-
-  @Override public int compareTo(ColumnWithIntIndex o) {
-    return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
-  }
-
-  @Override public boolean equals(Object obj) {
-    if (obj == null || getClass() != obj.getClass()) {
-      return false;
-    }
-    ColumnWithIntIndex o = (ColumnWithIntIndex)obj;
-    return Arrays.equals(column, o.column) && index == o.index;
-  }
-
-  @Override public int hashCode() {
-    return Arrays.hashCode(column) + index;
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
deleted file mode 100644
index 73f59d7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
-
-public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex
-    implements Comparable<ColumnWithIntIndex> {
-
-  public ColumnWithIntIndexForHighCard(byte[] column, int index) {
-    super(column, index);
-  }
-
-  @Override public int compareTo(ColumnWithIntIndex o) {
-    return UnsafeComparer.INSTANCE
-        .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
-  }
-
-  @Override public boolean equals(Object obj) {
-    if (obj == null || getClass() != obj.getClass()) {
-      return false;
-    }
-    ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj;
-    return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
-  }
-
-  @Override public int hashCode() {
-    return Arrays.hashCode(column) + getIndex();
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java
new file mode 100644
index 0000000..dd5d08d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowId.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class ColumnWithRowId<T> implements Comparable<ColumnWithRowId<T>> {
+  protected byte[] column;
+
+  private T index;
+
+  ColumnWithRowId(byte[] column, T index) {
+    this.column = column;
+    this.index = index;
+  }
+
+  /**
+   * @return the column
+   */
+  public byte[] getColumn() {
+    return column;
+  }
+
+  /**
+   * @param column the column to set
+   */
+  public void setColumn(byte[] column) {
+    this.column = column;
+  }
+
+  /**
+   * @return the index
+   */
+  public T getIndex() {
+    return index;
+  }
+
+  /**
+   * @param index the index to set
+   */
+  public void setIndex(T index) {
+    this.index = index;
+  }
+
+  @Override public int compareTo(ColumnWithRowId o) {
+    return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
+  }
+
+  @Override public boolean equals(Object obj) {
+    if (obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    ColumnWithRowId o = (ColumnWithRowId)obj;
+    return Arrays.equals(column, o.column) && index == o.index;
+  }
+
+  @Override public int hashCode() {
+    return Arrays.hashCode(column) + index.hashCode();
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowIdForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowIdForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowIdForHighCard.java
new file mode 100644
index 0000000..6aef7fd
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithRowIdForHighCard.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+
+public class ColumnWithRowIdForHighCard<T> extends ColumnWithRowId<T>
+    implements Comparable<ColumnWithRowId<T>> {
+
+  ColumnWithRowIdForHighCard(byte[] column, T index) {
+    super(column, index);
+  }
+
+  @Override public int compareTo(ColumnWithRowId o) {
+    return UnsafeComparer.INSTANCE
+        .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
+  }
+
+  @Override public boolean equals(Object obj) {
+    if (obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    ColumnWithRowIdForHighCard o = (ColumnWithRowIdForHighCard)obj;
+    return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
+  }
+
+  @Override public int hashCode() {
+    return Arrays.hashCode(column) + getIndex().hashCode();
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndex.java
deleted file mode 100644
index 57447a3..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndex.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class ColumnWithShortIndex implements Comparable<ColumnWithShortIndex> {
-  protected byte[] column;
-
-  private short index;
-
-  public ColumnWithShortIndex(byte[] column, short index) {
-    this.column = column;
-    this.index = index;
-  }
-
-  /**
-   * @return the column
-   */
-  public byte[] getColumn() {
-    return column;
-  }
-
-  /**
-   * @param column the column to set
-   */
-  public void setColumn(byte[] column) {
-    this.column = column;
-  }
-
-  /**
-   * @return the index
-   */
-  public short getIndex() {
-    return index;
-  }
-
-  /**
-   * @param index the index to set
-   */
-  public void setIndex(short index) {
-    this.index = index;
-  }
-
-  @Override public int compareTo(ColumnWithShortIndex o) {
-    return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
-  }
-
-  @Override public boolean equals(Object obj) {
-    if (obj == null || getClass() != obj.getClass()) {
-      return false;
-    }
-    ColumnWithShortIndex o = (ColumnWithShortIndex)obj;
-    return Arrays.equals(column, o.column) && index == o.index;
-  }
-
-  @Override public int hashCode() {
-    return Arrays.hashCode(column) + index;
-  }
-}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndexForNoDictionay.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndexForNoDictionay.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndexForNoDictionay.java
deleted file mode 100644
index 34cce63..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithShortIndexForNoDictionay.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.datastore.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
-
-public class ColumnWithShortIndexForNoDictionay extends ColumnWithShortIndex
-    implements Comparable<ColumnWithShortIndex> {
-
-  public ColumnWithShortIndexForNoDictionay(byte[] column, short index) {
-    super(column, index);
-  }
-
-  @Override public int compareTo(ColumnWithShortIndex o) {
-    return UnsafeComparer.INSTANCE
-        .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
-  }
-
-  @Override public boolean equals(Object obj) {
-    if (obj == null || getClass() != obj.getClass()) {
-      return false;
-    }
-    ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard) obj;
-    return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
-  }
-
-  @Override public int hashCode() {
-    return Arrays.hashCode(column) + getIndex();
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
index ff478f3..0570317 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
@@ -20,13 +20,13 @@ package org.apache.carbondata.core.datastore.columnar;
 public interface IndexStorage<T> {
   boolean isAlreadySorted();
 
-  T getDataAfterComp();
+  T getRowIdPage();
 
-  T getIndexMap();
+  T getRowIdRlePage();
 
-  byte[][] getKeyBlock();
+  byte[][] getDataPage();
 
-  T getDataIndexMap();
+  T getDataRlePage();
 
   int getTotalSize();
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/processing/src/main/java/org/apache/carbondata/core/datastore/columnar/ColGroupBlockStorage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/core/datastore/columnar/ColGroupBlockStorage.java b/processing/src/main/java/org/apache/carbondata/core/datastore/columnar/ColGroupBlockStorage.java
index fb8fe3d..70ac3d9 100644
--- a/processing/src/main/java/org/apache/carbondata/core/datastore/columnar/ColGroupBlockStorage.java
+++ b/processing/src/main/java/org/apache/carbondata/core/datastore/columnar/ColGroupBlockStorage.java
@@ -25,19 +25,20 @@ import org.apache.carbondata.processing.store.colgroup.ColGroupDataHolder;
 import org.apache.carbondata.processing.store.colgroup.ColGroupMinMax;
 
 /**
- * it is holder of column group data and also min max for colgroup block data
+ * it is holder of column group dataPage and also min max for colgroup block dataPage
  */
 public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage> {
 
-  private byte[][] data;
+  private byte[][] dataPage;
 
   private ColGroupMinMax colGrpMinMax;
 
-  public ColGroupBlockStorage(SegmentProperties segmentProperties, int colGrpIndex, byte[][] data) {
+  public ColGroupBlockStorage(SegmentProperties segmentProperties, int colGrpIndex,
+      byte[][] dataPage) {
     colGrpMinMax = new ColGroupMinMax(segmentProperties, colGrpIndex);
-    this.data = data;
-    for (int i = 0; i < data.length; i++) {
-      colGrpMinMax.add(data[i]);
+    this.dataPage = dataPage;
+    for (int i = 0; i < dataPage.length; i++) {
+      colGrpMinMax.add(dataPage[i]);
     }
   }
 
@@ -51,7 +52,7 @@ public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage
   /**
    * for column group storage its not required
    */
-  @Override public ColGroupDataHolder getDataAfterComp() {
+  public ColGroupDataHolder getRowIdPage() {
     //not required for column group storage
     return null;
   }
@@ -59,7 +60,7 @@ public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage
   /**
    * for column group storage its not required
    */
-  @Override public ColGroupDataHolder getIndexMap() {
+  public ColGroupDataHolder getRowIdRlePage() {
     // not required for column group storage
     return null;
   }
@@ -67,14 +68,14 @@ public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage
   /**
    * for column group storage its not required
    */
-  @Override public byte[][] getKeyBlock() {
-    return data;
+  public byte[][] getDataPage() {
+    return dataPage;
   }
 
   /**
    * for column group storage its not required
    */
-  @Override public ColGroupDataHolder getDataIndexMap() {
+  public ColGroupDataHolder getDataRlePage() {
     //not required for column group
     return null;
   }
@@ -83,7 +84,7 @@ public class ColGroupBlockStorage implements IndexStorage, Callable<IndexStorage
    * for column group storage its not required
    */
   @Override public int getTotalSize() {
-    return data.length;
+    return dataPage.length;
   }
 
   @Override public byte[] getMin() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/processing/src/main/java/org/apache/carbondata/processing/store/TablePageEncoder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePageEncoder.java b/processing/src/main/java/org/apache/carbondata/processing/store/TablePageEncoder.java
index 5d460a9..18c2297 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePageEncoder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePageEncoder.java
@@ -21,7 +21,7 @@ import java.util.Iterator;
 
 import org.apache.carbondata.core.datastore.TableSpec;
 import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForInt;
-import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndex;
+import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndexForInt;
 import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndexForShort;
 import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForShort;
 import org.apache.carbondata.core.datastore.columnar.IndexStorage;
@@ -84,7 +84,7 @@ public class TablePageEncoder {
       if (version == ColumnarFormatVersion.V3) {
         return new BlockIndexerStorageForNoInvertedIndexForShort(data, false);
       } else {
-        return new BlockIndexerStorageForNoInvertedIndex(data);
+        return new BlockIndexerStorageForNoInvertedIndexForInt(data);
       }
     }
   }
@@ -101,7 +101,7 @@ public class TablePageEncoder {
       if (version == ColumnarFormatVersion.V3) {
         return new BlockIndexerStorageForNoInvertedIndexForShort(data, false);
       } else {
-        return new BlockIndexerStorageForNoInvertedIndex(data);
+        return new BlockIndexerStorageForNoInvertedIndexForInt(data);
       }
     }
   }
@@ -126,7 +126,7 @@ public class TablePageEncoder {
       if (version == ColumnarFormatVersion.V3) {
         return new BlockIndexerStorageForNoInvertedIndexForShort(data, true);
       } else {
-        return new BlockIndexerStorageForNoInvertedIndex(data);
+        return new BlockIndexerStorageForNoInvertedIndexForInt(data);
       }
     }
   }
@@ -152,7 +152,7 @@ public class TablePageEncoder {
                   tablePage.getDictDimensionPage()[++dictionaryColumnCount].getByteArrayPage(),
                   isSortColumn,
                   isUseInvertedIndex[i] & isSortColumn);
-          flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getKeyBlock());
+          flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getDataPage());
           break;
         case DIRECT_DICTIONARY:
           // timestamp and date column
@@ -161,7 +161,7 @@ public class TablePageEncoder {
                   tablePage.getDictDimensionPage()[++dictionaryColumnCount].getByteArrayPage(),
                   isSortColumn,
                   isUseInvertedIndex[i] & isSortColumn);
-          flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getKeyBlock());
+          flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getDataPage());
           break;
         case PLAIN_VALUE:
           // high cardinality dimension, encoded as plain string
@@ -170,7 +170,7 @@ public class TablePageEncoder {
                   tablePage.getNoDictDimensionPage()[++noDictionaryColumnCount].getByteArrayPage(),
                   isSortColumn,
                   isUseInvertedIndex[i] & isSortColumn);
-          flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getKeyBlock());
+          flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getDataPage());
           break;
         case COMPLEX:
           // we need to add complex column at last, so skipping it here
@@ -188,7 +188,7 @@ public class TablePageEncoder {
       while (iterator.hasNext()) {
         byte[][] data = iterator.next();
         indexStorages[indexStorageOffset] = encodeAndCompressComplexDimension(data);
-        byte[] flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getKeyBlock());
+        byte[] flattened = ByteUtil.flatten(indexStorages[indexStorageOffset].getDataPage());
         compressedColumns[indexStorageOffset] = compressor.compressByte(flattened);
         indexStorageOffset++;
       }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
index bf35cd9..fab1a39 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
@@ -106,10 +106,10 @@ public class CarbonFactDataWriterImplV1 extends AbstractFactDataWriter<int[]> {
     for (int i = 0; i < isSortedData.length; i++) {
       if (!isSortedData[i]) {
         dataAfterCompression[idx] =
-            numberCompressor.compress((int[])keyStorageArray[i].getDataAfterComp());
-        if (null != keyStorageArray[i].getIndexMap()
-            && ((int[])keyStorageArray[i].getIndexMap()).length > 0) {
-          indexMap[idx] = numberCompressor.compress((int[])keyStorageArray[i].getIndexMap());
+            numberCompressor.compress((int[])keyStorageArray[i].getRowIdPage());
+        if (null != keyStorageArray[i].getRowIdRlePage()
+            && ((int[])keyStorageArray[i].getRowIdRlePage()).length > 0) {
+          indexMap[idx] = numberCompressor.compress((int[])keyStorageArray[i].getRowIdRlePage());
         } else {
           indexMap[idx] = new byte[0];
         }
@@ -131,7 +131,7 @@ public class CarbonFactDataWriterImplV1 extends AbstractFactDataWriter<int[]> {
       if (dataWriterVo.getRleEncodingForDictDim()[i]) {
         try {
           compressedDataIndex[idx] =
-              numberCompressor.compress((int[])keyStorageArray[i].getDataIndexMap());
+              numberCompressor.compress((int[])keyStorageArray[i].getDataRlePage());
           dataIndexMapLength[idx] = compressedDataIndex[idx].length;
           idx++;
         } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6a09ee84/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index b4bc040..a63e902 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -114,10 +114,10 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
       totalKeySize += keyLengths[i];
       if (!isSortedData[i]) {
         dataAfterCompression[i] =
-            getByteArray((short[])encoded.indexStorages[i].getDataAfterComp());
-        if (null != encoded.indexStorages[i].getIndexMap() &&
-            ((short[])encoded.indexStorages[i].getIndexMap()).length > 0) {
-          indexMap[i] = getByteArray((short[])encoded.indexStorages[i].getIndexMap());
+            getByteArray((short[])encoded.indexStorages[i].getRowIdPage());
+        if (null != encoded.indexStorages[i].getRowIdRlePage() &&
+            ((short[])encoded.indexStorages[i].getRowIdRlePage()).length > 0) {
+          indexMap[i] = getByteArray((short[])encoded.indexStorages[i].getRowIdRlePage());
         } else {
           indexMap[i] = new byte[0];
         }
@@ -135,7 +135,7 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter<short[]>
       if (dataWriterVo.getRleEncodingForDictDim()[i]) {
         try {
           compressedDataIndex[i] =
-              getByteArray((short[])encoded.indexStorages[i].getDataIndexMap());
+              getByteArray((short[])encoded.indexStorages[i].getDataRlePage());
           dataIndexMapLength[i] = compressedDataIndex[i].length;
         } catch (Exception e) {
           throw new CarbonDataWriterException(e.getMessage(), e);


Mime
View raw message