hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1611531 [3/3] - in /hadoop/common/branches/HDFS-6584/hadoop-yarn-project: ./ hadoop-yarn/bin/ hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hado...
Date Fri, 18 Jul 2014 02:21:28 GMT
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1611531&r1=1611530&r2=1611531&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Fri Jul 18 02:21:21 2014
@@ -53,10 +53,13 @@ import org.apache.hadoop.yarn.MockApps;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
@@ -77,11 +80,13 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
@@ -788,14 +793,14 @@ public class TestFairScheduler extends F
     scheduler.reinitialize(conf, resourceManager.getRMContext());
 
     ApplicationAttemptId id11 = createAppAttemptId(1, 1);
-    scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1");
-    scheduler.addApplicationAttempt(id11, false, true);
+    scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1", false);
+    scheduler.addApplicationAttempt(id11, false, false);
     ApplicationAttemptId id21 = createAppAttemptId(2, 1);
-    scheduler.addApplication(id21.getApplicationId(), "root.queue2", "user1");
-    scheduler.addApplicationAttempt(id21, false, true);
+    scheduler.addApplication(id21.getApplicationId(), "root.queue2", "user1", false);
+    scheduler.addApplicationAttempt(id21, false, false);
     ApplicationAttemptId id22 = createAppAttemptId(2, 2);
-    scheduler.addApplication(id22.getApplicationId(), "root.queue2", "user1");
-    scheduler.addApplicationAttempt(id22, false, true);
+    scheduler.addApplication(id22.getApplicationId(), "root.queue2", "user1", false);
+    scheduler.addApplicationAttempt(id22, false, false);
 
     int minReqSize = 
         FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB;
@@ -1556,8 +1561,8 @@ public class TestFairScheduler extends F
     scheduler.handle(nodeEvent2);
     
     ApplicationAttemptId appId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
-    scheduler.addApplication(appId.getApplicationId(), "queue1", "user1");
-    scheduler.addApplicationAttempt(appId, false, true);
+    scheduler.addApplication(appId.getApplicationId(), "queue1", "user1", false);
+    scheduler.addApplicationAttempt(appId, false, false);
     
     // 1 request with 2 nodes on the same rack. another request with 1 node on
     // a different rack
@@ -1838,7 +1843,7 @@ public class TestFairScheduler extends F
 
     ApplicationAttemptId attId =
         ApplicationAttemptId.newInstance(applicationId, this.ATTEMPT_ID++);
-    scheduler.addApplication(attId.getApplicationId(), queue, user);
+    scheduler.addApplication(attId.getApplicationId(), queue, user, false);
 
     numTries = 0;
     while (application.getFinishTime() == 0 && numTries < MAX_TRIES) {
@@ -2715,8 +2720,8 @@ public class TestFairScheduler extends F
     // send application request
     ApplicationAttemptId appAttemptId =
             createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
-    fs.addApplication(appAttemptId.getApplicationId(), "queue11", "user11");
-    fs.addApplicationAttempt(appAttemptId, false, true);
+    fs.addApplication(appAttemptId.getApplicationId(), "queue11", "user11", false);
+    fs.addApplicationAttempt(appAttemptId, false, false);
     List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
     ResourceRequest request =
             createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true);
@@ -2831,6 +2836,87 @@ public class TestFairScheduler extends F
       }
     }
   }
+    
+  @Test(timeout=5000)
+  public void testRecoverRequestAfterPreemption() throws Exception {
+    conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10);
+    
+    MockClock clock = new MockClock();
+    scheduler.setClock(clock);
+    scheduler.init(conf);
+    scheduler.start();
+    scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+    Priority priority = Priority.newInstance(20);
+    String host = "127.0.0.1";
+    int GB = 1024;
+
+    // Create Node and raised Node Added event
+    RMNode node = MockNodes.newNodeInfo(1,
+        Resources.createResource(16 * 1024, 4), 0, host);
+    NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+    scheduler.handle(nodeEvent);
+
+    // Create 3 container requests and place it in ask
+    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
+    ResourceRequest nodeLocalRequest = createResourceRequest(GB, 1, host,
+        priority.getPriority(), 1, true);
+    ResourceRequest rackLocalRequest = createResourceRequest(GB, 1,
+        node.getRackName(), priority.getPriority(), 1, true);
+    ResourceRequest offRackRequest = createResourceRequest(GB, 1,
+        ResourceRequest.ANY, priority.getPriority(), 1, true);
+    ask.add(nodeLocalRequest);
+    ask.add(rackLocalRequest);
+    ask.add(offRackRequest);
+
+    // Create Request and update
+    ApplicationAttemptId appAttemptId = createSchedulingRequest("queueA",
+        "user1", ask);
+    scheduler.update();
+
+    // Sufficient node check-ins to fully schedule containers
+    NodeUpdateSchedulerEvent nodeUpdate = new NodeUpdateSchedulerEvent(node);
+    scheduler.handle(nodeUpdate);
+
+    assertEquals(1, scheduler.getSchedulerApp(appAttemptId).getLiveContainers()
+        .size());
+    FSSchedulerApp app = scheduler.getSchedulerApp(appAttemptId);
+
+    // ResourceRequest will be empty once NodeUpdate is completed
+    Assert.assertNull(app.getResourceRequest(priority, host));
+
+    ContainerId containerId1 = ContainerId.newInstance(appAttemptId, 1);
+    RMContainer rmContainer = app.getRMContainer(containerId1);
+
+    // Create a preempt event and register for preemption
+    scheduler.warnOrKillContainer(rmContainer);
+    
+    // Wait for few clock ticks
+    clock.tick(5);
+    
+    // preempt now
+    scheduler.warnOrKillContainer(rmContainer);
+
+    List<ResourceRequest> requests = rmContainer.getResourceRequests();
+    // Once recovered, resource request will be present again in app
+    Assert.assertEquals(3, requests.size());
+    for (ResourceRequest request : requests) {
+      Assert.assertEquals(1,
+          app.getResourceRequest(priority, request.getResourceName())
+              .getNumContainers());
+    }
+
+    // Send node heartbeat
+    scheduler.update();
+    scheduler.handle(nodeUpdate);
+
+    List<Container> containers = scheduler.allocate(appAttemptId,
+        Collections.<ResourceRequest> emptyList(),
+        Collections.<ContainerId> emptyList(), null, null).getContainers();
+
+    // Now with updated ResourceRequest, a container is allocated for AM.
+    Assert.assertTrue(containers.size() == 1);
+  }
   
   @SuppressWarnings("resource")
   @Test

Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1611531&r1=1611530&r2=1611531&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
(original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Fri Jul 18 02:21:21 2014
@@ -2707,3 +2707,223 @@ Server: Jetty(6.1.26)
 +---+
 
 
+* Cluster {Delegation Tokens API}
+
+  The Delegation Tokens API can be used to create, renew and cancel YARN ResourceManager
delegation tokens. All delegation token requests must be carried out on a Kerberos authenticated
connection(using SPNEGO). Carrying out operations on a non-kerberos connection will result
in a FORBIDDEN response. In case of renewing a token, only the renewer specified when creating
the token can renew the token. Other users(including the owner) are forbidden from renewing
tokens. It should be noted that when cancelling or renewing a token, the token to be cancelled
or renewed is specified by setting a header.
+
+  This feature is currently in the alpha stage and may change in the future.
+
+** URI
+
+  Use the following URI to create and cancel delegation tokens.
+
+------
+  * http://<rm http address:port>/ws/v1/cluster/delegation-token
+------
+
+  Use the following URI to renew delegation tokens.
+
+------
+  * http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration
+------
+
+** HTTP Operations Supported
+
+------
+  * POST
+  * DELETE
+------
+
+** Query Parameters Supported
+
+------
+  None
+------
+
+** Elements of the <delegation-token> object
+
+  The response from the delegation tokens API contains one of the fields listed below.
+
+*---------------+--------------+-------------------------------+
+|| Item         || Data Type   || Description                   |
+*---------------+--------------+-------------------------------+
+| token         | string       | The delegation token          |
+*---------------+--------------+-------------------------------+
+| renewer       | string       | The user who is allowed to renew the delegation token |
+*---------------+--------------+-------------------------------+
+| owner         | string       | The owner of the delegation token |
+*---------------+--------------+-------------------------------+
+| kind          | string       | The kind of delegation token  |
+*---------------+--------------+-------------------------------+
+| expiration-time | long       | The expiration time of the token |
+*---------------+--------------+-------------------------------+
+| max-validity  | long         | The maximum validity of the token |
+*---------------+--------------+-------------------------------+
+
+** Response Examples
+
+*** Creating a token
+
+  <<JSON response>>
+
+  HTTP Request:
+
+------
+  POST http://<rm http address:port>/ws/v1/cluster/delegation-token
+  Accept: application/json
+  Content-Type: application/json
+  {
+    "renewer" : "test-renewer"
+  }
+------
+
+  Response Header
+
++---+
+  HTTP/1.1 200 OK
+  WWW-Authenticate: Negotiate ...
+  Date: Sat, 28 Jun 2014 18:08:11 GMT
+  Server: Jetty(6.1.26)
+  Set-Cookie: ...
+  Content-Type: application/json
++---+
+
+  Response body
+
++---+
+  {
+    "token":"MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUckiEZpigFHSJTKaQECFN9EMM9BzfPoDxu572EVUpzqhnSGE1JNX0RFTEVHQVRJT05fVE9LRU4A",
+    "renewer":"test-renewer",
+    "owner":"client@EXAMPLE.COM",
+    "kind":"RM_DELEGATION_TOKEN",
+    "expiration-time":"1405153616489",
+    "max-validity":"1405672016489"
+  }
++---+
+
+  <<XML response>>
+
+  HTTP Request
+
+------
+  POST http://<rm http address:port>/ws/v1/cluster/delegation-token
+  Accept: application/xml
+  Content-Type: application/xml
+  <delegation-token>
+    <renewer>test-renewer</renewer>
+  </delegation-token>
+------
+
+  Response Header
+
++---+
+  HTTP/1.1 200 OK
+  WWW-Authenticate: Negotiate ...
+  Date: Sat, 28 Jun 2014 18:08:11 GMT
+  Content-Length: 423
+  Server: Jetty(6.1.26)
+  Set-Cookie: ...
+  Content-Type: application/xml
++---+
+
+  Response Body
+
++---+
+  <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+  <delegation-token>
+    <token>MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUckgZ8yigFHSI4jMgcCFDTG8X6XFFn2udQngzSXQL8vWaKIE1JNX0RFTEVHQVRJT05fVE9LRU4A</token>
+    <renewer>test-renewer</renewer>
+    <owner>client@EXAMPLE.COM</owner>
+    <kind>RM_DELEGATION_TOKEN</kind>
+    <expiration-time>1405153180466</expiration-time>
+    <max-validity>1405671580466</max-validity>
+  </delegation-token>
++---+
+
+*** Renewing a token
+
+  <<JSON response>>
+
+  HTTP Request:
+
+------
+  POST http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration
+  Accept: application/json
+  Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A
+  Content-Type: application/json
+------
+
+  Response Header
+
++---+
+  HTTP/1.1 200 OK
+  WWW-Authenticate: Negotiate ...
+  Date: Sat, 28 Jun 2014 18:08:11 GMT
+  Server: Jetty(6.1.26)
+  Set-Cookie: ...
+  Content-Type: application/json
++---+
+
+  Response body
+
++---+
+  {
+    "expiration-time":"1404112520402"
+  }
++---+
+
+  <<XML response>>
+
+  HTTP Request
+
+------
+  POST http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration
+  Accept: application/xml
+  Content-Type: application/xml
+  Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A
+------
+
+  Response Header
+
++---+
+  HTTP/1.1 200 OK
+  WWW-Authenticate: Negotiate ...
+  Date: Sat, 28 Jun 2014 18:08:11 GMT
+  Content-Length: 423
+  Server: Jetty(6.1.26)
+  Set-Cookie: ...
+  Content-Type: application/xml
++---+
+
+  Response Body
+
++---+
+  <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+  <delegation-token>
+    <expiration-time>1404112520402</expiration-time>
+  </delegation-token>
++---+
+
+*** Cancelling a token
+
+  HTTP Request
+
+-----
+DELETE http://<rm http address:port>/ws/v1/cluster/delegation-token
+Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A
+Accept: application/xml
+-----
+
+  Response Header
+
++---+
+  HTTP/1.1 200 OK
+  WWW-Authenticate: Negotiate ...
+  Date: Sun, 29 Jun 2014 07:25:18 GMT
+  Transfer-Encoding: chunked
+  Server: Jetty(6.1.26)
+  Set-Cookie: ...
+  Content-Type: application/xml
++---+
+
+  No response body.



Mime
View raw message