hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ka...@apache.org
Subject [4/5] hadoop git commit: Comments to document the changes to FSPreemptionThread
Date Thu, 16 Jun 2016 02:07:13 GMT
Comments to document the changes to FSPreemptionThread


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/450f956a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/450f956a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/450f956a

Branch: refs/heads/fs-preemption
Commit: 450f956a3f1b948b544d0cfd837f3af0fc9837e5
Parents: e4eec25
Author: Karthik Kambatla <kasha@apache.org>
Authored: Wed Jun 15 09:14:32 2016 -0700
Committer: Karthik Kambatla <kasha@apache.org>
Committed: Wed Jun 15 09:14:32 2016 -0700

----------------------------------------------------------------------
 .../scheduler/fair/FSPreemptionThread.java      | 40 +++++++++++++++-----
 1 file changed, 31 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/450f956a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
index 0e99b64..766fd5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -74,39 +74,61 @@ public class FSPreemptionThread extends Thread {
   }
 
   /**
-   * Returns a non-null PremptionContext if it finds a node that can
-   * accommodate a request from this app. Also, reserves the node for this app.
+   * Given an app, identify containers to preempt to satisfy the app's next
+   * resource request.
+   *
+   * @param starvedApp
+   * @return
    */
-  private List<RMContainer> identifyContainersToPreempt(FSAppAttempt starvedApp) {
-    List<RMContainer> containers = new ArrayList<>();
+  private List<RMContainer> identifyContainersToPreempt(FSAppAttempt
+      starvedApp) {
+    List<RMContainer> containers = new ArrayList<>(); // return value
+
+    // Find the nodes that match the next resource request
     ResourceRequest request = starvedApp.getNextResourceRequest();
+    // TODO (KK): Should we check other resource requests if we can't match
+    // the first one?
+
     Resource requestCapability = request.getCapability();
-    List<FSSchedulerNode> nodes =
+    List<FSSchedulerNode> potentialNodes =
         scheduler.getNodeTracker().getNodes(request.getResourceName());
+
+    // From the potential nodes, pick a node that has enough containers
+    // from apps over their fairshare
     FSSchedulerNode targetNode = null;
-    Resource potential = Resources.clone(Resources.none());
-    for (FSSchedulerNode node : nodes) {
+    for (FSSchedulerNode node : potentialNodes) {
       containers.clear();
-      potential = Resources.clone(Resources.none());
+      Resource potential = Resources.clone(Resources.none());
       for (RMContainer container : node.getCopiedListOfRunningContainers()) {
         Resource containerResource = container.getAllocatedResource();
         FSAppAttempt app =
             scheduler.getSchedulerApp(container.getApplicationAttemptId());
+
+        // Check if the app's allocation will be over its fairshare even
+        // after preempting this container
         if (Resources.fitsIn(containerResource,
             Resources.subtract(app.getResourceUsage(), app.getFairShare()))) {
           Resources.addTo(potential, containerResource);
         }
+
+        // TODO (KK): Should we go through other app reservations if the
+        // containers alone are not enough to meet the starvedApp's requirements
+
+        // Check if we have already identified enough containers
         if (Resources.fitsIn(requestCapability, potential)) {
           break;
         }
       }
+
+      // Set targetNode if this node has enough containers to preempt
       if (Resources.fitsIn(requestCapability, potential)) {
         targetNode = node;
         break;
       }
     }
 
-    if (Resources.fitsIn(requestCapability, potential)) {
+    if (targetNode != null) {
+      // Reserve resources on the target node so it doesn't go to other nodes
       starvedApp.reserve(targetNode, requestCapability);
       return containers;
     } else {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message