hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject [1/3] hadoop git commit: YARN-7817. Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. (Sunil G via wangda)
Date Fri, 26 Jan 2018 08:11:05 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 2e5865606 -> 8b5b045bd


YARN-7817. Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore
resource usages. (Sunil G via wangda)

Change-Id: Ia7ceeabd82046645ddeaf487c763288f36cfbdee


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0cfb0a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0cfb0a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0cfb0a3

Branch: refs/heads/trunk
Commit: e0cfb0a31a1c6ad0f9dcf0705b44229593a0915f
Parents: 2e58656
Author: Wangda Tan <wangda@apache.org>
Authored: Fri Jan 26 15:43:27 2018 +0800
Committer: Wangda Tan <wangda@apache.org>
Committed: Fri Jan 26 15:43:27 2018 +0800

----------------------------------------------------------------------
 .../webapp/JAXBContextResolver.java             |  2 +-
 .../webapp/dao/ClusterMetricsInfo.java          | 24 ++++++++++
 .../resourcemanager/webapp/dao/NodeInfo.java    | 20 ++++++++
 .../webapp/dao/ResourceInfo.java                | 18 ++++++--
 .../webapp/dao/ResourceInformationsInfo.java    | 48 ++++++++++++++++++++
 .../main/webapp/app/models/cluster-metric.js    |  4 +-
 .../src/main/webapp/app/models/yarn-rm-node.js  |  4 +-
 .../main/webapp/app/serializers/yarn-rm-node.js |  4 +-
 8 files changed, 114 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
index 2f50a24..2e4204e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
@@ -55,7 +55,7 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext>
{
             UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
             StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
             FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
-            AppTimeoutInfo.class };
+            AppTimeoutInfo.class, ResourceInformationsInfo.class };
     // these dao classes need root unwrapping
     final Class[] rootUnwrappedTypes =
         { NewApplication.class, ApplicationSubmissionContextInfo.class,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 3214cb9..84f70d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -61,6 +61,12 @@ public class ClusterMetricsInfo {
   private int activeNodes;
   private int shutdownNodes;
 
+  // Total used resource of the cluster, including all partitions
+  private ResourceInfo totalUsedResourcesAcrossPartition;
+
+  // Total registered resources of the cluster, including all partitions
+  private ResourceInfo totalClusterResourcesAcrossPartition;
+
   public ClusterMetricsInfo() {
   } // JAXB needs this
 
@@ -92,9 +98,20 @@ public class ClusterMetricsInfo {
     this.containersReserved = metrics.getReservedContainers();
 
     if (rs instanceof CapacityScheduler) {
+      CapacityScheduler cs = (CapacityScheduler) rs;
       this.totalMB = availableMB + allocatedMB + reservedMB;
       this.totalVirtualCores =
           availableVirtualCores + allocatedVirtualCores + containersReserved;
+      // TODO, add support of other schedulers to get total used resources
+      // across partition.
+      if (cs.getRootQueue() != null
+          && cs.getRootQueue().getQueueResourceUsage() != null
+          && cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
+        totalUsedResourcesAcrossPartition = new ResourceInfo(
+            cs.getRootQueue().getQueueResourceUsage().getAllUsed());
+        totalClusterResourcesAcrossPartition = new ResourceInfo(
+            cs.getClusterResource());
+      }
     } else {
       this.totalMB = availableMB + allocatedMB;
       this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
@@ -310,4 +327,11 @@ public class ClusterMetricsInfo {
     this.shutdownNodes = shutdownNodes;
   }
 
+  public ResourceInfo getTotalUsedResourcesAcrossPartition() {
+    return totalUsedResourcesAcrossPartition;
+  }
+
+  public ResourceInfo getTotalClusterResourcesAcrossPartition() {
+    return totalClusterResourcesAcrossPartition;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 2530c8e..3cec215 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -58,6 +58,8 @@ public class NodeInfo {
   private int numQueuedContainers;
   protected ArrayList<String> nodeLabels = new ArrayList<String>();
   protected ResourceUtilizationInfo resourceUtilization;
+  protected ResourceInfo usedResource;
+  protected ResourceInfo availableResource;
 
   public NodeInfo() {
   } // JAXB needs this
@@ -75,6 +77,8 @@ public class NodeInfo {
       this.usedVirtualCores = report.getUsedResource().getVirtualCores();
       this.availableVirtualCores =
           report.getAvailableResource().getVirtualCores();
+      this.usedResource = new ResourceInfo(report.getUsedResource());
+      this.availableResource = new ResourceInfo(report.getAvailableResource());
     }
     this.id = id.toString();
     this.rack = ni.getRackName();
@@ -183,6 +187,22 @@ public class NodeInfo {
     return this.nodeLabels;
   }
 
+  public ResourceInfo getUsedResource() {
+    return usedResource;
+  }
+
+  public void setUsedResource(ResourceInfo used) {
+    this.usedResource = used;
+  }
+
+  public ResourceInfo getAvailableResource() {
+    return availableResource;
+  }
+
+  public void setAvailableResource(ResourceInfo avail) {
+    this.availableResource = avail;
+  }
+
   public ResourceUtilizationInfo getResourceUtilization() {
     return this.resourceUtilization;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
index e13980a..5bed936 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
@@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlRootElement;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
+
 @XmlRootElement
 @XmlAccessorType(XmlAccessType.NONE)
 public class ResourceInfo {
@@ -34,6 +35,9 @@ public class ResourceInfo {
   long memory;
   @XmlElement
   int vCores;
+  @XmlElement
+  ResourceInformationsInfo resourceInformations =
+      new ResourceInformationsInfo();
 
   private Resource resources;
 
@@ -41,9 +45,13 @@ public class ResourceInfo {
   }
 
   public ResourceInfo(Resource res) {
-    memory = res.getMemorySize();
-    vCores = res.getVirtualCores();
-    resources = Resources.clone(res);
+    // Make sure no NPE.
+    if (res != null) {
+      memory = res.getMemorySize();
+      vCores = res.getVirtualCores();
+      resources = Resources.clone(res);
+      resourceInformations.addAll(res.getAllResourcesListCopy());
+    }
   }
 
   public long getMemorySize() {
@@ -84,4 +92,8 @@ public class ResourceInfo {
   public Resource getResource() {
     return Resource.newInstance(resources);
   }
+
+  public ResourceInformationsInfo getResourcesInformations() {
+    return resourceInformations;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java
new file mode 100644
index 0000000..651d7df
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+
+@XmlRootElement(name = "resourceInformations")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ResourceInformationsInfo {
+
+  @XmlElement(name = "resourceInformation")
+  protected ArrayList<ResourceInformation> resourceInformation =
+      new ArrayList<ResourceInformation>();
+
+  public ResourceInformationsInfo() {
+  } // JAXB needs this
+
+  public ArrayList<ResourceInformation> getApps() {
+    return resourceInformation;
+  }
+
+  public void addAll(List<ResourceInformation> resourcesInformationsInfo) {
+    resourceInformation.addAll(resourcesInformationsInfo);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
index bbc6f08..764723a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
@@ -169,8 +169,8 @@ export default DS.Model.extend({
       && this.get("totalUsedResourcesAcrossPartition")) {
       var usages = [];
 
-      var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations;
-      var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations;
+      var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourceInformations.resourceInformation;
+      var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourceInformations.resourceInformation;
 
       clusterResourceInformations.forEach(function(cluster) {
         var perResourceTypeUsage = {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
index 3b2385f..64fe7b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
@@ -99,7 +99,7 @@ export default DS.Model.extend({
 
     const usedResource = this.get("usedResource");
     const availableResource = this.get("availableResource");
-    var resourceInformations = usedResource ? usedResource.resourcesInformations : [];
+    var resourceInformations = usedResource ? usedResource.resourceInformations.resourceInformation
: [];
     for (var i = 0; i < resourceInformations.length; i++) {
       ri = resourceInformations[i];
       if (ri.name === "yarn.io/gpu") {
@@ -108,7 +108,7 @@ export default DS.Model.extend({
     }
 
     var available = 0;
-    resourceInformations = availableResource ? availableResource.resourcesInformations :
[];
+    resourceInformations = availableResource ? availableResource.resourceInformations.resourceInformation
: [];
     for (i = 0; i < resourceInformations.length; i++) {
       ri = resourceInformations[i];
       if (ri.name === "yarn.io/gpu") {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0cfb0a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
index a3a1d59..c27fd92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
@@ -42,8 +42,8 @@ export default DS.JSONAPISerializer.extend({
         availableVirtualCores: payload.availableVirtualCores,
         version: payload.version,
         nodeLabels: payload.nodeLabels,
-        usedResource: payload.used,
-        availableResource: payload.avail
+        usedResource: payload.usedResource,
+        availableResource: payload.availableResource
       }
     };
     return fixedPayload;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message