tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prak...@apache.org
Subject tez git commit: EZ-1482. Fix memory issues for Local Mode running concurrent tasks (pramachandran)
Date Wed, 15 Apr 2015 16:18:43 GMT
Repository: tez
Updated Branches:
  refs/heads/master 11b584318 -> 19378d5c2


EZ-1482. Fix memory issues for Local Mode running concurrent tasks (pramachandran)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/19378d5c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/19378d5c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/19378d5c

Branch: refs/heads/master
Commit: 19378d5c2d79006b19140f2c5b9740d04942a3b8
Parents: 11b5843
Author: Prakash Ramachandran <pramachandran@hortonworks.com>
Authored: Wed Apr 15 21:48:07 2015 +0530
Committer: Prakash Ramachandran <pramachandran@hortonworks.com>
Committed: Wed Apr 15 21:48:07 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                               |  1 +
 .../tez/dag/app/launcher/LocalContainerLauncher.java      | 10 +++++++---
 2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/19378d5c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 605577e..56f6ea9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
   TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
 
 ALL CHANGES:
+  TEZ-1482. Fix memory issues for Local Mode running concurrent tasks
   TEZ-2033. Update TestOrderedWordCount to add processor configs as history text
   and use MR configs correctly
   TEZ-2318. Tez UI: source and sink page is broken as they are not populated.

http://git-wip-us.apache.org/repos/asf/tez/blob/19378d5c/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
index 7f5a7b1..5c8aab6 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java
@@ -90,6 +90,7 @@ public class LocalContainerLauncher extends AbstractService implements
   private final String workingDirectory;
   private final Map<String, String> localEnv = new HashMap<String, String>();
   private final ExecutionContext executionContext;
+  private int numExecutors;
 
   private final ConcurrentHashMap<ContainerId, ListenableFuture<TezChild.ContainerExecutionResult>>
       runningContainers =
@@ -122,7 +123,7 @@ public class LocalContainerLauncher extends AbstractService implements
 
   @Override
   public synchronized void serviceInit(Configuration conf) {
-    int numExecutors = conf.getInt(TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS,
+    numExecutors = conf.getInt(TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS,
         TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS_DEFAULT);
     Preconditions.checkState(numExecutors >=1, "Must have at least 1 executor");
     ExecutorService rawExecutor = Executors.newFixedThreadPool(numExecutors,
@@ -328,11 +329,14 @@ public class LocalContainerLauncher extends AbstractService implements
     containerEnv.putAll(localEnv);
     containerEnv.put(Environment.USER.name(), context.getUser());
 
-    // TODO TEZ-1482. Control the memory available based on number of executors
+    long memAvailable;
+    synchronized (this) { // needed to fix findbugs Inconsistent synchronization warning
+      memAvailable = Runtime.getRuntime().maxMemory() / numExecutors;
+    }
     TezChild tezChild =
         TezChild.newTezChild(defaultConf, null, 0, containerId.toString(), tokenIdentifier,
             attemptNumber, localDirs, workingDirectory, containerEnv, "", executionContext,
credentials,
-            Runtime.getRuntime().maxMemory(), context.getUser());
+            memAvailable, context.getUser());
     tezChild.setUmbilical(tezTaskUmbilicalProtocol);
     return tezChild;
   }


Mime
View raw message