carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: Fixed RLE Encoding Issue
Date Mon, 13 Mar 2017 11:47:22 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 63bf4e0de -> a0ae747d2


Fixed RLE Encoding Issue


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7a0fcb25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7a0fcb25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7a0fcb25

Branch: refs/heads/master
Commit: 7a0fcb25010ab60bd9f382089e89f06985cdcc16
Parents: 63bf4e0
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Fri Mar 10 19:13:24 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Mon Mar 13 19:42:38 2017 +0800

----------------------------------------------------------------------
 .../columnar/BlockIndexerStorageForShort.java   | 24 +++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7a0fcb25/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
index 346d8d8..7b47fee 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
@@ -172,8 +172,7 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]>
{
 
   private void compressDataMyOwnWay(ColumnWithShortIndex[] indexes) {
     byte[] prvKey = indexes[0].getColumn();
-    List<ColumnWithShortIndex> list =
-        new ArrayList<ColumnWithShortIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    List<ColumnWithShortIndex> list = new ArrayList<ColumnWithShortIndex>(indexes.length
/ 2);
     list.add(indexes[0]);
     short counter = 1;
     short start = 0;
@@ -192,12 +191,25 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]>
{
     }
     map.add(start);
     map.add(counter);
-    this.keyBlock = convertToKeyArray(list);
-    if (indexes.length == keyBlock.length) {
-      dataIndexMap = new short[0];
-    } else {
+    // if rle is index size is more than 70% then rle wont give any benefit
+    // so better to avoid rle index and write data as it is
+    boolean useRle = (((list.size() + map.size()) * 100) / indexes.length) < 70;
+    if (useRle) {
+      this.keyBlock = convertToKeyArray(list);
       dataIndexMap = convertToArray(map);
+    } else {
+      this.keyBlock = convertToKeyArray(indexes);
+      dataIndexMap = new short[0];
+    }
+  }
+
+  private byte[][] convertToKeyArray(ColumnWithShortIndex[] indexes) {
+    byte[][] shortArray = new byte[indexes.length][];
+    for (int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = indexes[i].getColumn();
+      totalSize += shortArray[i].length;
     }
+    return shortArray;
   }
 
   private byte[][] convertToKeyArray(List<ColumnWithShortIndex> list) {


Mime
View raw message