hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From naganarasimha...@apache.org
Subject [29/50] [abbrv] hadoop git commit: YARN-6257. Fix CapacityScheduler REST API JSON output. Contributed By Tao Yang
Date Mon, 02 Apr 2018 07:18:45 GMT
YARN-6257. Fix CapacityScheduler REST API JSON output.
           Contributed By Tao Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8fb00c3f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8fb00c3f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8fb00c3f

Branch: refs/heads/YARN-3409
Commit: 8fb00c3fce96b445a71fb1a5d98b6a55d6ebfd10
Parents: 9d7a903
Author: Eric Yang <eyang@apache.org>
Authored: Thu Mar 29 17:36:34 2018 -0400
Committer: Eric Yang <eyang@apache.org>
Committed: Thu Mar 29 17:36:34 2018 -0400

----------------------------------------------------------------------
 .../webapp/dao/CapacitySchedulerHealthInfo.java |  35 +++--
 .../webapp/TestRMWebServicesCapacitySched.java  |   4 +
 .../src/site/markdown/ResourceManagerRest.md    | 132 +++++++++++++++++++
 3 files changed, 158 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fb00c3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
index 652c91b..3683055 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
@@ -25,15 +25,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 @XmlAccessorType(XmlAccessType.FIELD)
 public class CapacitySchedulerHealthInfo {
 
   @XmlAccessorType(XmlAccessType.FIELD)
   public static class OperationInformation {
+    String operation;
     String nodeId;
     String containerId;
     String queue;
@@ -41,7 +40,9 @@ public class CapacitySchedulerHealthInfo {
     OperationInformation() {
     }
 
-    OperationInformation(SchedulerHealth.DetailedInformation di) {
+    OperationInformation(String operation,
+        SchedulerHealth.DetailedInformation di) {
+      this.operation = operation;
       this.nodeId = di.getNodeId() == null ? "N/A" : di.getNodeId().toString();
       this.containerId =
           di.getContainerId() == null ? "N/A" : di.getContainerId().toString();
@@ -59,6 +60,10 @@ public class CapacitySchedulerHealthInfo {
     public String getQueue() {
       return queue;
     }
+
+    public String getOperation() {
+      return operation;
+    }
   }
 
   @XmlAccessorType(XmlAccessType.FIELD)
@@ -90,7 +95,7 @@ public class CapacitySchedulerHealthInfo {
   }
 
   long lastrun;
-  Map<String, OperationInformation> operationsInfo;
+  List<OperationInformation> operationsInfo;
   List<LastRunDetails> lastRunDetails;
 
   CapacitySchedulerHealthInfo() {
@@ -100,18 +105,22 @@ public class CapacitySchedulerHealthInfo {
     return lastrun;
   }
 
+  public List<OperationInformation> getOperationsInfo() {
+    return operationsInfo;
+  }
+
   CapacitySchedulerHealthInfo(CapacityScheduler cs) {
     SchedulerHealth ht = cs.getSchedulerHealth();
     lastrun = ht.getLastSchedulerRunTime();
-    operationsInfo = new HashMap<>();
-    operationsInfo.put("last-allocation",
-      new OperationInformation(ht.getLastAllocationDetails()));
-    operationsInfo.put("last-release",
-      new OperationInformation(ht.getLastReleaseDetails()));
-    operationsInfo.put("last-preemption",
-      new OperationInformation(ht.getLastPreemptionDetails()));
-    operationsInfo.put("last-reservation",
-      new OperationInformation(ht.getLastReservationDetails()));
+    operationsInfo = new ArrayList<>();
+    operationsInfo.add(new OperationInformation("last-allocation",
+        ht.getLastAllocationDetails()));
+    operationsInfo.add(
+        new OperationInformation("last-release", ht.getLastReleaseDetails()));
+    operationsInfo.add(new OperationInformation("last-preemption",
+        ht.getLastPreemptionDetails()));
+    operationsInfo.add(new OperationInformation("last-reservation",
+        ht.getLastReservationDetails()));
 
     lastRunDetails = new ArrayList<>();
     lastRunDetails.add(new LastRunDetails("releases", ht.getReleaseCount(), ht

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fb00c3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 86a3943..e37f76f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -329,6 +329,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
     JSONObject health = info.getJSONObject("health");
     assertNotNull(health);
     assertEquals("incorrect number of elements", 3, health.length());
+    JSONArray operationsInfo = health.getJSONArray("operationsInfo");
+    assertEquals("incorrect number of elements", 4, operationsInfo.length());
+    JSONArray lastRunDetails = health.getJSONArray("lastRunDetails");
+    assertEquals("incorrect number of elements", 3, lastRunDetails.length());
 
     JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
     assertEquals("incorrect number of elements", 2, arr.length());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fb00c3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
index c43fe14..caeaf3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
@@ -311,6 +311,7 @@ The capacity scheduler supports hierarchical queues. This one request
will print
 | maxCapacity | float | Configured maximum queue capacity in percentage relative to its parent
queue |
 | queueName | string | Name of the queue |
 | queues | array of queues(JSON)/zero or more queue objects(XML) | A collection of queue
resources |
+| health | A single health object | The health metrics of capacity scheduler. This metrics
existed since 2.8.0, but the output was not well formatted. Hence users can not make use of
this field cleanly, this is optimized from 3.2.0 onwards. |
 
 ### Elements of the queues object for a Parent queue
 
@@ -361,6 +362,31 @@ The capacity scheduler supports hierarchical queues. This one request
will print
 | memory | int | The amount of memory used (in MB) |
 | vCores | int | The number of virtual cores |
 
+### Elements of the health object in schedulerInfo:
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| lastrun | long | The time in which application started (in ms since epoch) |
+| operationsInfo | array of operations(JSON)/operation objects(XML) | A collection of operation
objects |
+| lastRunDetails | array of lastRunDetails(JSON)/lastRunDetail objects(XML) | A collection
of lastRunDetail objects |
+
+### Elements of the operation object in health:
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| operation | string | The type of operation |
+| nodeId | string | The id of the node to which the operation relates |
+| containerId | string | The id of the container to which the operation relates |
+| queue | string | The name of the queue to which the operation relates |
+
+### Elements of the lastRunDetail object in health:
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| operation | string | The type of operation  |
+| count | long | The id of the node to which the operation relates |
+| resources | A single resource object | The resources to which the operation relates |
+
 #### Response Examples
 
 **JSON response**
@@ -632,6 +658,61 @@ Response Body:
                     }
                 ]
             },
+            "health": {
+                "lastrun": 1326381444693,
+                "operationsInfo": [
+                    {
+                        "operation": "last-allocation",
+                        "nodeId": "N/A",
+                        "containerId": "N/A",
+                        "queue": "N/A"
+                    },
+                    {
+                        "operation": "last-release",
+                        "nodeId": "host.domain.com:8041",
+                        "containerId": "container_1326821518301_0005_01_000001",
+                        "queue": "root.default"
+                    },
+                    {
+                        "operation": "last-preemption",
+                        "nodeId": "N/A",
+                        "containerId": "N/A",
+                        "queue": "N/A"
+                    },
+                    {
+                        "operation": "last-reservation",
+                        "nodeId": "N/A",
+                        "containerId": "N/A",
+                        "queue": "N/A"
+                    }
+                ],
+                "lastRunDetails": [
+                    {
+                        "operation": "releases",
+                        "count": 0,
+                        "resources": {
+                            "memory": 0,
+                            "vCores": 0
+                        }
+                    },
+                    {
+                        "operation": "allocations",
+                        "count": 0,
+                        "resources": {
+                            "memory": 0,
+                            "vCores": 0
+                        }
+                    },
+                    {
+                        "operation": "reservations",
+                        "count": 0,
+                        "resources": {
+                            "memory": 0,
+                            "vCores": 0
+                        }
+                    }
+                ]
+            },
             "type": "capacityScheduler",
             "usedCapacity": 0.0
         }
@@ -894,6 +975,57 @@ Response Body:
         </resourcesUsed>
       </queue>
     </queues>
+    <health>
+      <lastrun>1326381444693</lastrun>
+      <operationsInfo>
+        <operation>last-allocation</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-release</operation>
+        <nodeId>host.domain.com:8041</nodeId>
+        <containerId>container_1326821518301_0005_01_000001</containerId>
+        <queue>root.default</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-preemption</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <operationsInfo>
+        <operation>last-reservation</operation>
+        <nodeId>N/A</nodeId>
+        <containerId>N/A</containerId>
+        <queue>N/A</queue>
+      </operationsInfo>
+      <lastRunDetails>
+        <operation>releases</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+        </resources>
+      </lastRunDetails>
+      <lastRunDetails>
+        <operation>allocations</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+        </resources>
+      </lastRunDetails>
+      <lastRunDetails>
+        <operation>reservations</operation>
+        <count>0</count>
+        <resources>
+          <memory>0</memory>
+          <vCores>0</vCores>
+        </resources>
+      </lastRunDetails>
+    </health>
   </schedulerInfo>
 </scheduler>
 ```


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message