hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From haiboc...@apache.org
Subject [1/2] hadoop git commit: YARN-4511. Common scheduler changes to support scheduler-specific oversubscription implementations.
Date Thu, 02 Nov 2017 16:45:52 GMT
Repository: hadoop
Updated Branches:
  refs/heads/YARN-1011 e5a996e85 -> 0c8fd8973


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c8fd897/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 740ef33..c56be29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -563,7 +563,7 @@ public class TestNodeLabelContainerAllocation {
       int numContainers) {
     CapacityScheduler cs = (CapacityScheduler) rm.getRMContext().getScheduler();
     SchedulerNode node = cs.getSchedulerNode(nodeId);
-    Assert.assertEquals(numContainers, node.getNumContainers());
+    Assert.assertEquals(numContainers, node.getNumGuaranteedContainers());
   }
 
   /**
@@ -1065,7 +1065,7 @@ public class TestNodeLabelContainerAllocation {
     for (int i = 0; i < 50; i++) {
       cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
       cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
-      if (schedulerNode1.getNumContainers() == 0) {
+      if (schedulerNode1.getNumGuaranteedContainers() == 0) {
         cycleWaited++;
       }
     }
@@ -1131,7 +1131,7 @@ public class TestNodeLabelContainerAllocation {
             CSAMContainerLaunchDiagnosticsConstants.LAST_NODE_PROCESSED_MSG
                 + nodeIdStr + " ( Partition : [x]"));
     Assert.assertEquals(0, cs.getSchedulerNode(nm1.getNodeId())
-        .getNumContainers());
+        .getNumGuaranteedContainers());
     
     rm1.close();
   }
@@ -1215,7 +1215,7 @@ public class TestNodeLabelContainerAllocation {
     }
     
     // app1 gets all resource in partition=x
-    Assert.assertEquals(10, schedulerNode1.getNumContainers());
+    Assert.assertEquals(10, schedulerNode1.getNumGuaranteedContainers());
 
     // check non-exclusive containers of LeafQueue is correctly updated
     LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
@@ -1943,7 +1943,7 @@ public class TestNodeLabelContainerAllocation {
     }
 
     // app1 gets all resource in partition=x
-    Assert.assertEquals(5, schedulerNode1.getNumContainers());
+    Assert.assertEquals(5, schedulerNode1.getNumGuaranteedContainers());
 
     SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
         .getNodeReport(nm1.getNodeId());
@@ -2043,7 +2043,7 @@ public class TestNodeLabelContainerAllocation {
     }
 
     // app1 gets all resource in partition=x (non-exclusive)
-    Assert.assertEquals(3, schedulerNode1.getNumContainers());
+    Assert.assertEquals(3, schedulerNode1.getNumGuaranteedContainers());
 
     SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
         .getNodeReport(nm1.getNodeId());
@@ -2074,7 +2074,7 @@ public class TestNodeLabelContainerAllocation {
     cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
 
     // app1 gets all resource in default partition
-    Assert.assertEquals(2, schedulerNode2.getNumContainers());
+    Assert.assertEquals(2, schedulerNode2.getNumGuaranteedContainers());
 
     // 3GB is used from label x quota. 2GB used from default label.
     // So total 2.5 GB is remaining.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c8fd897/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
index 854a65c..d432991 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestContinuousScheduling.java
@@ -20,16 +20,26 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.NoOpSystemMetricPublisher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ClusterNodeTracker;
 import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -48,7 +58,9 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.Matchers.isA;
 import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -319,7 +331,9 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
           for (FSSchedulerNode node : clusterNodeTracker.getAllNodes()) {
             int i = ThreadLocalRandom.current().nextInt(-30, 30);
             synchronized (scheduler) {
-              node.deductUnallocatedResource(Resource.newInstance(i * 1024, i));
+              RMContainer container = createRMContainer(
+                  node.getRMNode(), Resource.newInstance(i * 1024, 1));
+              node.allocateContainer(container);
             }
           }
         }
@@ -333,6 +347,32 @@ public class TestContinuousScheduling extends FairSchedulerTestBase {
     }
   }
 
+  private static RMContainer createRMContainer(
+      RMNode node, Resource resource) {
+    ApplicationAttemptId appAttemptId = ApplicationAttemptId.
+        newInstance(ApplicationId.newInstance(0, 0), 0);
+    ContainerId cId =
+        ContainerId.newContainerId(appAttemptId, 0);
+    Container container = Container.newInstance(
+        cId, node.getNodeID(), node.getNodeAddress(), resource,
+        Priority.newInstance(0), null, ExecutionType.GUARANTEED);
+
+    Dispatcher dispatcher = new AsyncDispatcher();
+    RMContext rmContext = mock(RMContext.class);
+    when(rmContext.getDispatcher()).thenReturn(dispatcher);
+    when(rmContext.getSystemMetricsPublisher()).
+        thenReturn(new NoOpSystemMetricPublisher());
+    when(rmContext.getYarnConfiguration()).
+        thenReturn(new YarnConfiguration());
+    when(rmContext.getContainerAllocationExpirer()).
+        thenReturn(new ContainerAllocationExpirer(dispatcher));
+    when(rmContext.getRMApplicationHistoryWriter()).
+        thenReturn(new RMApplicationHistoryWriter());
+    return new RMContainerImpl(container, null,
+        container.getId().getApplicationAttemptId(),
+        node.getNodeID(), "test", rmContext);
+  }
+
   @Test
   public void testFairSchedulerContinuousSchedulingInitTime() throws Exception {
     scheduler.start();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c8fd897/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
index 0e3d344..6726f17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerNode.java
@@ -87,7 +87,7 @@ public class TestFSSchedulerNode {
     while (!Resources.isNone(schedulerNode.getUnallocatedResource())) {
       createDefaultContainer();
       schedulerNode.allocateContainer(containers.get(containers.size() - 1));
-      schedulerNode.containerStarted(containers.get(containers.size() - 1).
+      schedulerNode.containerLaunched(containers.get(containers.size() - 1).
           getContainerId());
     }
   }
@@ -183,9 +183,9 @@ public class TestFSSchedulerNode {
     assertEquals("Nothing should have been allocated, yet",
         Resources.none(), schedulerNode.getAllocatedResource());
     schedulerNode.allocateContainer(containers.get(0));
-    schedulerNode.containerStarted(containers.get(0).getContainerId());
+    schedulerNode.containerLaunched(containers.get(0).getContainerId());
     schedulerNode.allocateContainer(containers.get(1));
-    schedulerNode.containerStarted(containers.get(1).getContainerId());
+    schedulerNode.containerLaunched(containers.get(1).getContainerId());
     schedulerNode.allocateContainer(containers.get(2));
     assertEquals("Container should be allocated",
         Resources.multiply(containers.get(0).getContainer().getResource(), 3.0),
@@ -225,7 +225,7 @@ public class TestFSSchedulerNode {
     schedulerNode.releaseContainer(containers.get(0).getContainerId(), true);
     allocateContainers(schedulerNode);
     assertEquals("Container should be allocated",
-        schedulerNode.getTotalResource(),
+        schedulerNode.getCapacity(),
         schedulerNode.getAllocatedResource());
 
     // Release all remaining containers
@@ -266,7 +266,7 @@ public class TestFSSchedulerNode {
     schedulerNode.releaseContainer(containers.get(0).getContainerId(), true);
     allocateContainers(schedulerNode);
     assertEquals("Container should be allocated",
-        schedulerNode.getTotalResource(),
+        schedulerNode.getCapacity(),
         schedulerNode.getAllocatedResource());
 
     // Release all remaining containers
@@ -312,7 +312,7 @@ public class TestFSSchedulerNode {
 
     allocateContainers(schedulerNode);
     assertEquals("Container should be allocated",
-        schedulerNode.getTotalResource(),
+        schedulerNode.getCapacity(),
         schedulerNode.getAllocatedResource());
 
     // Release all containers
@@ -360,7 +360,7 @@ public class TestFSSchedulerNode {
     allocateContainers(schedulerNode);
 
     assertEquals("Container should be allocated",
-        schedulerNode.getTotalResource(),
+        schedulerNode.getCapacity(),
         schedulerNode.getAllocatedResource());
 
     // Release all containers
@@ -399,7 +399,7 @@ public class TestFSSchedulerNode {
     when(starvingApp.isStopped()).thenReturn(true);
     allocateContainers(schedulerNode);
     assertNotEquals("Container should be allocated",
-        schedulerNode.getTotalResource(),
+        schedulerNode.getCapacity(),
         schedulerNode.getAllocatedResource());
 
     // Release all containers
@@ -437,7 +437,7 @@ public class TestFSSchedulerNode {
     // Container partially reassigned
     allocateContainers(schedulerNode);
     assertEquals("Container should be allocated",
-        Resources.subtract(schedulerNode.getTotalResource(),
+        Resources.subtract(schedulerNode.getCapacity(),
             Resource.newInstance(512, 0)),
         schedulerNode.getAllocatedResource());
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c8fd897/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 6b2109d..e1f155e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3291,12 +3291,16 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     scheduler.handle(node2UpdateEvent);
     if (invalid) {
       assertEquals(0, app.getLiveContainers().size());
-      assertEquals(0, scheduler.getNode(node2.getNodeID()).getNumContainers());
-      assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+      assertEquals(0,
+          scheduler.getNode(node2.getNodeID()).getNumGuaranteedContainers());
+      assertEquals(0,
+          scheduler.getNode(node1.getNodeID()).getNumGuaranteedContainers());
     } else {
       assertEquals(1, app.getLiveContainers().size());
-      assertEquals(1, scheduler.getNode(node2.getNodeID()).getNumContainers());
-      assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers());
+      assertEquals(1,
+          scheduler.getNode(node2.getNodeID()).getNumGuaranteedContainers());
+      assertEquals(0,
+          scheduler.getNode(node1.getNodeID()).getNumGuaranteedContainers());
     }
   }
 
@@ -4984,7 +4988,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     // Check total resource of scheduler node is also changed to 0 GB 0 core
     Resource totalResource =
         resourceManager.getResourceScheduler()
-            .getSchedulerNode(nm_0.getNodeId()).getTotalResource();
+            .getSchedulerNode(nm_0.getNodeId()).getCapacity();
     Assert.assertEquals(totalResource.getMemorySize(), 0 * GB);
     Assert.assertEquals(totalResource.getVirtualCores(), 0);
     // Check the available resource is 0/0

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c8fd897/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 3f97b59..63da0c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -340,7 +340,7 @@ public class TestFifoScheduler {
     
     // SchedulerNode's total resource and available resource are changed.
     assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID())
-        .getTotalResource().getMemorySize());
+        .getCapacity().getMemorySize());
     assertEquals(1024, scheduler.getNodeTracker().getNode(node0.getNodeID()).
         getUnallocatedResource().getMemorySize(), 1024);
     QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
@@ -1266,7 +1266,7 @@ public class TestFifoScheduler {
     // Check total resource of scheduler node is also changed to 1 GB 1 core
     Resource totalResource =
         resourceManager.getResourceScheduler()
-            .getSchedulerNode(nm_0.getNodeId()).getTotalResource();
+            .getSchedulerNode(nm_0.getNodeId()).getCapacity();
     Assert.assertEquals(totalResource.getMemorySize(), 1 * GB);
     Assert.assertEquals(totalResource.getVirtualCores(), 1);
     // Check the available resource is 0/0


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message