carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] carbondata git commit: Problem: Memory leak in case of compaction when unsafe is true
Date Thu, 22 Jun 2017 01:28:30 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 3a6f8ca2c -> 56e56c037


Problem: Memory leak in case of compaction when unsafe is true

Analysis: In case of compaction, queryExecutor object is formed for multiple blocks but the
objects are not retained and finish method is called only on the last instance created for
query executor. Due to this the memory allocated to precious objects is not released which
can lead to out of memory issue.

Fix: Add all the object to a list and call finish method on all the objects so that memory
gets released


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c0facf28
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c0facf28
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c0facf28

Branch: refs/heads/master
Commit: c0facf287ebf88277f0fd96212c76d96984dea73
Parents: 3a6f8ca
Author: manishgupta88 <tomanishgupta18@gmail.com>
Authored: Wed Jun 21 20:09:12 2017 +0530
Committer: jackylk <jacky.likun@huawei.com>
Committed: Thu Jun 22 09:27:50 2017 +0800

----------------------------------------------------------------------
 .../processing/merger/CarbonCompactionExecutor.java       | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0facf28/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
index c00fe2e..94bfbe4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
@@ -58,7 +58,7 @@ public class CarbonCompactionExecutor {
   private final Map<String, List<DataFileFooter>> dataFileMetadataSegMapping;
   private final SegmentProperties destinationSegProperties;
   private final Map<String, TaskBlockInfo> segmentMapping;
-  private QueryExecutor queryExecutor;
+  private List<QueryExecutor> queryExecutorList;
   private CarbonTable carbonTable;
   private QueryModel queryModel;
 
@@ -86,6 +86,7 @@ public class CarbonCompactionExecutor {
     this.carbonTable = carbonTable;
     this.dataFileMetadataSegMapping = dataFileMetadataSegMapping;
     this.restructuredBlockExists = restructuredBlockExists;
+    queryExecutorList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
   }
 
   /**
@@ -156,7 +157,8 @@ public class CarbonCompactionExecutor {
   private CarbonIterator<BatchResult> executeBlockList(List<TableBlockInfo> blockList)
       throws QueryExecutionException, IOException {
     queryModel.setTableBlockInfos(blockList);
-    this.queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
+    QueryExecutor queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
+    queryExecutorList.add(queryExecutor);
     return queryExecutor.execute(queryModel);
   }
 
@@ -166,7 +168,9 @@ public class CarbonCompactionExecutor {
    */
   public void finish() {
     try {
-      queryExecutor.finish();
+      for (QueryExecutor queryExecutor : queryExecutorList) {
+        queryExecutor.finish();
+      }
     } catch (QueryExecutionException e) {
       LOGGER.error(e, "Problem while finish: ");
     }


Mime
View raw message