carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [CARBONDATA-2770][BloomDataMap] Optimize code to get blocklet id when rebuilding datamap
Date Tue, 24 Jul 2018 02:29:36 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 83562ae75 -> ce53b48a1


[CARBONDATA-2770][BloomDataMap] Optimize code to get blocklet id when rebuilding datamap

we should get exactly number of blocklet id from blocklet scanned result instead of building
it ourselves.

This closes #2539


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ce53b48a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ce53b48a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ce53b48a

Branch: refs/heads/master
Commit: ce53b48a15b9a9a61921b50ee721142eede280cd
Parents: 83562ae
Author: Manhua <kevinjmh@qq.com>
Authored: Mon Jul 23 17:03:58 2018 +0800
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Tue Jul 24 10:29:21 2018 +0800

----------------------------------------------------------------------
 .../scan/collector/impl/RowIdRawBasedResultCollector.java |  2 +-
 .../impl/RowIdRestructureBasedRawResultCollector.java     |  2 +-
 .../carbondata/datamap/IndexDataMapRebuildRDD.scala       | 10 +---------
 3 files changed, 3 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/ce53b48a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RowIdRawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RowIdRawBasedResultCollector.java
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RowIdRawBasedResultCollector.java
index a68265d..f0291fa 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RowIdRawBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RowIdRawBasedResultCollector.java
@@ -134,7 +134,7 @@ public class RowIdRawBasedResultCollector extends AbstractScannedResultCollector
       // 3 for blockletId, pageId, rowId
       Object[] row = new Object[1 + queryMeasures.length + 3];
       scannedResult.incrementCounter();
-      row[1 + queryMeasures.length] = scannedResult.getBlockletNumber();
+      row[1 + queryMeasures.length] = Integer.parseInt(scannedResult.getBlockletNumber());
       row[1 + queryMeasures.length + 1] = scannedResult.getCurrentPageCounter();
       ByteArrayWrapper wrapper = new ByteArrayWrapper();
       wrapper.setDictionaryKey(dictionaryKeyArrayBatch.get(i));

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ce53b48a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RowIdRestructureBasedRawResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RowIdRestructureBasedRawResultCollector.java
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RowIdRestructureBasedRawResultCollector.java
index 28e778f..e343dea 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RowIdRestructureBasedRawResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RowIdRestructureBasedRawResultCollector.java
@@ -106,7 +106,7 @@ public class RowIdRestructureBasedRawResultCollector extends RestructureBasedRaw
       // 3 for blockletId, pageId, rowId
       Object[] row = new Object[1 + queryMeasures.length + 3];
       scannedResult.incrementCounter();
-      row[1 + queryMeasures.length] = scannedResult.getBlockletNumber();
+      row[1 + queryMeasures.length] = Integer.parseInt(scannedResult.getBlockletNumber());
       row[1 + queryMeasures.length + 1] = scannedResult.getCurrentPageCounter();
       ByteArrayWrapper wrapper = new ByteArrayWrapper();
       wrapper.setDictionaryKey(dictionaryKeyArrayBatch.get(i));

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ce53b48a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
index 29df1ca..e4d5b26 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
@@ -357,21 +357,13 @@ class IndexDataMapRebuildRDD[K, V](
         // skip clear datamap and we will do this adter rebuild
         reader.setSkipClearDataMapAtClose(true)
 
-        var blockletId = 0
-        var firstRow = true
         while (reader.nextKeyValue()) {
           val rowWithPosition = reader.getCurrentValue
           val size = rowWithPosition.length
+          val blockletId = rowWithPosition(size - 3).asInstanceOf[Int]
           val pageId = rowWithPosition(size - 2).asInstanceOf[Int]
           val rowId = rowWithPosition(size - 1).asInstanceOf[Int]
 
-          if (!firstRow && pageId == 0 && rowId == 0) {
-            // new blocklet started, increase blockletId
-            blockletId = blockletId + 1
-          } else {
-            firstRow = false
-          }
-
           refresher.addRow(blockletId, pageId, rowId, rowWithPosition)
         }
 


Mime
View raw message