hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ka...@apache.org
Subject hadoop git commit: YARN-6215. FairScheduler preemption and update should not run concurrently. (Tao Jie via kasha)
Date Mon, 27 Feb 2017 04:31:11 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 f3cdf29af -> a95d3e1fe


YARN-6215. FairScheduler preemption and update should not run concurrently. (Tao Jie via kasha)

(cherry picked from commit 815d53506fb0c5ca029c993d6b094db2ac0ca6eb)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a95d3e1f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a95d3e1f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a95d3e1f

Branch: refs/heads/branch-2
Commit: a95d3e1fe7d533990be305e6cf1d10c8c512908b
Parents: f3cdf29
Author: Karthik Kambatla <kasha@apache.org>
Authored: Sun Feb 26 20:16:36 2017 -0800
Committer: Karthik Kambatla <kasha@apache.org>
Committed: Sun Feb 26 20:31:03 2017 -0800

----------------------------------------------------------------------
 .../scheduler/fair/FSPreemptionThread.java              | 12 +++++++++++-
 .../resourcemanager/scheduler/fair/FairScheduler.java   |  5 +++++
 2 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a95d3e1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
index af73c10..65df0c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -32,6 +32,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Timer;
 import java.util.TimerTask;
+import java.util.concurrent.locks.Lock;
 
 /**
  * Thread that handles FairScheduler preemption.
@@ -43,6 +44,7 @@ class FSPreemptionThread extends Thread {
   private final long warnTimeBeforeKill;
   private final long delayBeforeNextStarvationCheck;
   private final Timer preemptionTimer;
+  private final Lock schedulerReadLock;
 
   FSPreemptionThread(FairScheduler scheduler) {
     setDaemon(true);
@@ -61,6 +63,7 @@ class FSPreemptionThread extends Thread {
         : 4 * scheduler.getNMHeartbeatInterval()); // 4 heartbeats
     delayBeforeNextStarvationCheck = warnTimeBeforeKill + allocDelay +
         fsConf.getWaitTimeBeforeNextStarvationCheck();
+    schedulerReadLock = scheduler.getSchedulerReadLock();
   }
 
   public void run() {
@@ -68,7 +71,14 @@ class FSPreemptionThread extends Thread {
       FSAppAttempt starvedApp;
       try{
         starvedApp = context.getStarvedApps().take();
-        preemptContainers(identifyContainersToPreempt(starvedApp));
+        // Hold the scheduler readlock so this is not concurrent with the
+        // update thread.
+        schedulerReadLock.lock();
+        try {
+          preemptContainers(identifyContainersToPreempt(starvedApp));
+        } finally {
+          schedulerReadLock.unlock();
+        }
         starvedApp.preemptionTriggered(delayBeforeNextStarvationCheck);
       } catch (InterruptedException e) {
         LOG.info("Preemption thread interrupted! Exiting.");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a95d3e1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 4f5ee36..eac7a77 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -33,6 +33,7 @@ import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -1786,4 +1787,8 @@ public class FairScheduler extends
   long getNMHeartbeatInterval() {
     return nmHeartbeatInterval;
   }
+
+  ReadLock getSchedulerReadLock() {
+    return this.readLock;
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message