kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject kylin git commit: KYLIN-1226 Calculate reserve MB based on mapreduce.task.io.sort.mb
Date Tue, 15 Dec 2015 07:48:57 GMT
Repository: kylin
Updated Branches:
  refs/heads/2.0-rc 49269d2b3 -> cd90eaa09


KYLIN-1226 Calculate reserve MB based on mapreduce.task.io.sort.mb


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cd90eaa0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cd90eaa0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cd90eaa0

Branch: refs/heads/2.0-rc
Commit: cd90eaa09a7944241827e99e4f40a8b6179bcbd1
Parents: 49269d2
Author: Li, Yang <yangli9@ebay.com>
Authored: Tue Dec 15 15:47:31 2015 +0800
Committer: Li, Yang <yangli9@ebay.com>
Committed: Tue Dec 15 15:48:04 2015 +0800

----------------------------------------------------------------------
 .../common/util/MemoryBudgetController.java     | 14 +++++++++----
 .../cube/inmemcubing/DoggedCubeBuilder.java     | 15 +-------------
 .../cube/inmemcubing/InMemCubeBuilder.java      |  6 +-----
 .../engine/mr/steps/InMemCuboidMapper.java      | 21 ++++++++++++--------
 4 files changed, 25 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/cd90eaa0/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index 37b0bd4..3fa1574 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -279,14 +279,20 @@ public class MemoryBudgetController {
 
         public void markHigh() {
             // get avail mem without gc
-            lowAvail = Math.min(lowAvail, MemoryBudgetController.getSystemAvailMB());
-            logger.info("Lower system avail " + lowAvail + " MB in markHigh()");
+            int mb = MemoryBudgetController.getSystemAvailMB();
+            if (mb < lowAvail) {
+                lowAvail = mb;
+                logger.info("Lower system avail " + lowAvail + " MB in markHigh()");
+            }
         }
 
         public void markLow() {
             // get avail mem after gc
-            highAvail = Math.max(highAvail, MemoryBudgetController.gcAndGetSystemAvailMB());
-            logger.info("Higher system avail " + highAvail + " MB in markLow()");
+            int mb = MemoryBudgetController.gcAndGetSystemAvailMB();
+            if (mb > highAvail) {
+                highAvail = mb;
+                logger.info("Higher system avail " + highAvail + " MB in markLow()");
+            }
         }
 
         public int getEstimateMB() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd90eaa0/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index e87d8ff..b616b4c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -60,19 +60,6 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
         // check memory more often if a single row is big
         if (cubeDesc.hasMemoryHungryCountDistinctMeasures())
             unitRows /= 10;
-
-        // a meaningful default reserved memory
-        int sysAvailMB = MemoryBudgetController.getSystemAvailMB();
-        int reserve = Math.max(sysAvailMB / 10, 100); // 10% of system available, or 100
MB at least
-        int halfSysAvail = sysAvailMB / 2;
-        if (reserve > halfSysAvail) { // but no more than half of system available
-            logger.info("Reserve " + reserve + " MB is more than half of system avail " +
sysAvailMB + " MB, override to " + halfSysAvail);
-            reserve = halfSysAvail;
-        }
-        
-        setReserveMemoryMB(reserve);
-        logger.info("Reserve memory is set to " + reserve + " MB");
-
     }
 
     public void setSplitRowThreshold(int rowThreshold) {
@@ -258,7 +245,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
             int nSplit = splits.size();
             long splitRowCount = nSplit == 0 ? 0 : splits.get(nSplit - 1).inputRowCount;
 
-            logger.debug(splitRowCount + " records went into split #" + nSplit + "; " + systemAvailMB
+ " MB left, " + reserveMemoryMB + " MB threshold");
+            logger.info(splitRowCount + " records went into split #" + nSplit + "; " + systemAvailMB
+ " MB left, " + reserveMemoryMB + " MB threshold");
 
             if (splitRowCount >= splitRowThreshold) {
                 logger.info("Split cut due to hitting splitRowThreshold " + splitRowThreshold);

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd90eaa0/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 2cc298b..36de8a9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -304,14 +304,10 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
         }
     }
 
-    private int getSystemAvailMB() {
-        return MemoryBudgetController.gcAndGetSystemAvailMB();
-    }
-
     private void makeMemoryBudget() {
         baseResult.aggrCacheMB = Math.max(baseCuboidMemTracker.getEstimateMB(), 10); // 10
MB at minimal
         logger.info("Base cuboid aggr cache is " + baseResult.aggrCacheMB + " MB");
-        int systemAvailMB = getSystemAvailMB();
+        int systemAvailMB = MemoryBudgetController.gcAndGetSystemAvailMB();
         logger.info("System avail " + systemAvailMB + " MB");
         int reserve = reserveMemoryMB;
         logger.info("Reserve " + reserve + " MB for system basics");

http://git-wip-us.apache.org/repos/asf/kylin/blob/cd90eaa0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
index 1e29066..9e627c9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
@@ -16,12 +16,12 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.MemoryBudgetController;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
 import org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder;
 import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
 import org.apache.kylin.dict.Dictionary;
 import org.apache.kylin.engine.mr.ByteArrayWritable;
 import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
@@ -29,7 +29,6 @@ import org.apache.kylin.engine.mr.KylinMapper;
 import org.apache.kylin.engine.mr.MRUtil;
 import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
 import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.TblColRef;
 
@@ -74,18 +73,24 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN,
Object, ByteArr
 
             dictionaryMap.put(col, cubeSegment.getDictionary(col));
         }
-        
+
         DoggedCubeBuilder cubeBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap);
-        // Some may want to left out memory for "mapreduce.task.io.sort.mb", but that is
not
-        // necessary, because the output phase is after all in-mem cubing is done, and at
that
-        // time all memory has been released, cuboid data is read from ConcurrentDiskStore.
-        //cubeBuilder.setReserveMemoryMB(mapreduce.task.io.sort.mb);
-        
+        cubeBuilder.setReserveMemoryMB(calculateReserveMB(context.getConfiguration()));
+
         ExecutorService executorService = Executors.newSingleThreadExecutor();
         future = executorService.submit(cubeBuilder.buildAsRunnable(queue, new MapContextGTRecordWriter(context,
cubeDesc, cubeSegment)));
 
     }
 
+    private int calculateReserveMB(Configuration configuration) {
+        int sysAvailMB = MemoryBudgetController.getSystemAvailMB();
+        int mrReserve = configuration.getInt("mapreduce.task.io.sort.mb", 100);
+        int sysReserve = Math.max(sysAvailMB / 10, 100);
+        int reserveMB = mrReserve + sysReserve;
+        logger.info("Reserve " + reserveMB + " MB = " + mrReserve + " (MR reserve) + " +
sysReserve + " (SYS reserve)");
+        return reserveMB;
+    }
+
     @Override
     public void map(KEYIN key, Object record, Context context) throws IOException, InterruptedException
{
         // put each row to the queue


Mime
View raw message