hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [19/50] [abbrv] hadoop git commit: YARN-5165. Fix NoOvercommitPolicy to take advantage of RLE representation of plan. (Carlo Curino via asuresh)
Date Tue, 07 Jun 2016 20:26:11 GMT
YARN-5165. Fix NoOvercommitPolicy to take advantage of RLE representation of plan. (Carlo Curino
via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db54670e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db54670e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db54670e

Branch: refs/heads/YARN-4757
Commit: db54670e83a84c1d7deff2c225725687cf9e5f14
Parents: f10ebc6
Author: Arun Suresh <asuresh@apache.org>
Authored: Fri Jun 3 14:49:32 2016 -0700
Committer: Arun Suresh <asuresh@apache.org>
Committed: Fri Jun 3 14:49:32 2016 -0700

----------------------------------------------------------------------
 .../reservation/NoOverCommitPolicy.java         | 38 ++++++++------------
 .../planning/TestSimpleCapacityReplanner.java   |  2 +-
 2 files changed, 15 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/db54670e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
index 119520b..814d4b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
@@ -21,11 +21,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation;
 import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ReservationId;
-import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.MismatchedUserException;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ResourceOverCommitException;
-import org.apache.hadoop.yarn.util.resource.Resources;
 
 /**
  * This policy enforce a simple physical cluster capacity constraints, by
@@ -52,29 +50,21 @@ public class NoOverCommitPolicy implements SharingPolicy {
               + oldReservation.getUser() + " != " + reservation.getUser());
     }
 
-    long startTime = reservation.getStartTime();
-    long endTime = reservation.getEndTime();
-    long step = plan.getStep();
+    RLESparseResourceAllocation available = plan.getAvailableResourceOverTime(
+        reservation.getUser(), reservation.getReservationId(),
+        reservation.getStartTime(), reservation.getEndTime());
 
-    // for every instant in time, check we are respecting cluster capacity
-    for (long t = startTime; t < endTime; t += step) {
-      Resource currExistingAllocTot = plan.getTotalCommittedResources(t);
-      Resource currNewAlloc = reservation.getResourcesAtTime(t);
-      Resource currOldAlloc = Resource.newInstance(0, 0);
-      if (oldReservation != null) {
-        oldReservation.getResourcesAtTime(t);
-      }
-      // check the cluster is never over committed
-      // currExistingAllocTot + currNewAlloc - currOldAlloc >
-      // capPlan.getTotalCapacity()
-      if (Resources.greaterThan(plan.getResourceCalculator(), plan
-          .getTotalCapacity(), Resources.subtract(
-          Resources.add(currExistingAllocTot, currNewAlloc), currOldAlloc),
-          plan.getTotalCapacity())) {
-        throw new ResourceOverCommitException("Resources at time " + t
-            + " would be overcommitted by " + "accepting reservation: "
-            + reservation.getReservationId());
-      }
+    // test the reservation does not exceed what is available
+    try {
+      RLESparseResourceAllocation
+          .merge(plan.getResourceCalculator(), plan.getTotalCapacity(),
+              available, reservation.getResourcesOverTime(),
+              RLESparseResourceAllocation.RLEOperator.subtractTestNonNegative,
+              reservation.getStartTime(), reservation.getEndTime());
+    } catch (PlanningException p) {
+      throw new ResourceOverCommitException(
+          "Resources at time " + " would be overcommitted by "
+              + "accepting reservation: " + reservation.getReservationId());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db54670e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
index d762b36..e01608c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
@@ -53,7 +53,7 @@ public class TestSimpleCapacityReplanner {
   @Test
   public void testReplanningPlanCapacityLoss() throws PlanningException {
 
-    Resource clusterCapacity = Resource.newInstance(100 * 1024, 10);
+    Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
     Resource minAlloc = Resource.newInstance(1024, 1);
     Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message