tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject git commit: TEZ-1618. LocalTaskSchedulerService.getTotalResources() and getAvailableResources() can get negative if JVM memory is larger than 2GB (Chen He via jeagles)
Date Thu, 25 Sep 2014 19:13:42 GMT
Repository: tez
Updated Branches:
  refs/heads/master df375e82f -> d1588c2e2


TEZ-1618. LocalTaskSchedulerService.getTotalResources() and getAvailableResources() can get
negative if JVM memory is larger than 2GB (Chen He via jeagles)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/d1588c2e
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/d1588c2e
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/d1588c2e

Branch: refs/heads/master
Commit: d1588c2e26aff66fc78740e71fc7fa091c913f8c
Parents: df375e8
Author: Jonathan Eagles <jeagles@gmail.com>
Authored: Thu Sep 25 14:13:29 2014 -0500
Committer: Jonathan Eagles <jeagles@gmail.com>
Committed: Thu Sep 25 14:13:29 2014 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../dag/app/rm/LocalTaskSchedulerService.java   | 23 +++++++++++++-------
 2 files changed, 16 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/d1588c2e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index eda417a..44a450a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -26,6 +26,7 @@ ALL CHANGES:
   TEZ-1499. Add SortMergeJoinExample to tez-examples
   TEZ-1613. Decrease running time for TestAMRecovery
   TEZ-1240. Add system test for propagation of diagnostics for errors
+  TEZ-1618. LocalTaskSchedulerService.getTotalResources() and getAvailableResources() can
get negative if JVM memory is larger than 2GB
 
 Release 0.5.1: Unreleased
 

http://git-wip-us.apache.org/repos/asf/tez/blob/d1588c2e/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
index 490ffeb..026ed7d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java
@@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 
+import com.google.common.primitives.Ints;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -91,10 +92,17 @@ public class LocalTaskSchedulerService extends TaskSchedulerService {
 
   @Override
   public Resource getAvailableResources() {
-    Resource freeResources = Resource.newInstance(
-        (int)Runtime.getRuntime().freeMemory()/(1024*1024),
-        Runtime.getRuntime().availableProcessors());
-    return freeResources;
+    long memory = Runtime.getRuntime().freeMemory();
+    int cores = Runtime.getRuntime().availableProcessors();
+    return createResource(memory, cores);
+  }
+
+  static Resource createResource(long runtimeMemory, int core) {
+    if (runtimeMemory < 0 || core < 0) {
+      throw new IllegalArgumentException("Negative Memory or Core provided!"
+          + "mem: "+runtimeMemory+" core:"+core);
+    }
+    return Resource.newInstance(Ints.checkedCast(runtimeMemory/(1024*1024)), core);
   }
 
   @Override
@@ -108,10 +116,9 @@ public class LocalTaskSchedulerService extends TaskSchedulerService {
 
   @Override
   public Resource getTotalResources() {
-    Resource totalResources = Resource.newInstance(
-        (int)Runtime.getRuntime().maxMemory()/(1024*1024),
-        Runtime.getRuntime().availableProcessors());
-    return totalResources;
+    long memory = Runtime.getRuntime().maxMemory();
+    int cores = Runtime.getRuntime().availableProcessors();
+    return createResource(memory, cores);
   }
 
   @Override


Mime
View raw message