carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [05/50] carbondata git commit: Keep the pointers in unsafe to reduce the GC over head
Date Thu, 11 May 2017 13:52:55 GMT
Keep the pointers in unsafe to reduce the GC over head


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b051184f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b051184f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b051184f

Branch: refs/heads/branch-1.1
Commit: b051184f460f60a74ec896e73bf421360f5edf88
Parents: 8410081
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Mon May 1 08:34:26 2017 +0530
Committer: jackylk <jacky.likun@huawei.com>
Committed: Thu May 4 12:17:52 2017 +0800

----------------------------------------------------------------------
 .../newflow/sort/unsafe/IntPointerBuffer.java   | 23 +++++++++++++++++++-
 .../newflow/sort/unsafe/UnsafeSortDataRows.java |  1 +
 2 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b051184f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/IntPointerBuffer.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/IntPointerBuffer.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/IntPointerBuffer.java
index f6eac97..074e865 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/IntPointerBuffer.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/IntPointerBuffer.java
@@ -17,7 +17,9 @@
 
 package org.apache.carbondata.processing.newflow.sort.unsafe;
 
+import org.apache.carbondata.core.memory.CarbonUnsafe;
 import org.apache.carbondata.core.memory.MemoryBlock;
+import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
 
 /**
  * Holds the pointers for rows.
@@ -33,6 +35,8 @@ public class IntPointerBuffer {
 
   private MemoryBlock baseBlock;
 
+  private MemoryBlock pointerMemoryBlock;
+
   public IntPointerBuffer(MemoryBlock baseBlock) {
     // TODO can be configurable, it is initial size and it can grow automatically.
     this.length = 100000;
@@ -61,9 +65,23 @@ public class IntPointerBuffer {
   public int get(int index) {
     assert index >= 0 : "index (" + index + ") should >= 0";
     assert index < length : "index (" + index + ") should < length (" + length + ")";
+    if (pointerBlock == null) {
+      return CarbonUnsafe.unsafe.getInt(pointerMemoryBlock.getBaseObject(),
+          pointerMemoryBlock.getBaseOffset() + (index * 4));
+    }
     return pointerBlock[index];
   }
 
+  public void loadToUnsafe() throws CarbonSortKeyAndGroupByException {
+    pointerMemoryBlock = UnsafeSortDataRows.getMemoryBlock(pointerBlock.length * 4);
+    for (int i = 0; i < pointerBlock.length; i++) {
+      CarbonUnsafe.unsafe
+          .putInt(pointerMemoryBlock.getBaseObject(), pointerMemoryBlock.getBaseOffset()
+ i * 4,
+              pointerBlock[i]);
+    }
+    pointerBlock = null;
+  }
+
   public int getActualSize() {
     return actualSize;
   }
@@ -89,8 +107,11 @@ public class IntPointerBuffer {
 
   public void freeMemory() {
     pointerBlock = null;
+    if (pointerMemoryBlock != null) {
+      UnsafeMemoryManager.INSTANCE.freeMemory(pointerMemoryBlock);
+    }
     if (baseBlock != null) {
       UnsafeMemoryManager.INSTANCE.freeMemory(baseBlock);
     }
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b051184f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
index cb24968..3afd3b0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
@@ -317,6 +317,7 @@ public class UnsafeSortDataRows {
         } else {
           // add sort temp filename to and arrayList. When the list size reaches 20 then
           // intermediate merging of sort temp files will be triggered
+          page.getBuffer().loadToUnsafe();
           unsafeInMemoryIntermediateFileMerger.addDataChunkToMerge(page);
           LOGGER.info(
               "Time taken to sort row page with size" + page.getBuffer().getActualSize()
+ "is: "


Mime
View raw message