hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vinayakum...@apache.org
Subject [01/29] hadoop git commit: YARN-1156. Enhance NodeManager AllocatedGB and AvailableGB metrics for aggregation of decimal values. (Contributed by Tsuyoshi OZAWA)
Date Mon, 08 Dec 2014 09:15:24 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-EC bdc01015c -> 833185345


YARN-1156. Enhance NodeManager AllocatedGB and AvailableGB metrics for aggregation of decimal
values. (Contributed by Tsuyoshi OZAWA)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e65b7c5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e65b7c5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e65b7c5f

Branch: refs/heads/HDFS-EC
Commit: e65b7c5ff6b0c013e510e750fe5cf59acfefea5f
Parents: 7caa3bc
Author: Junping Du <junping_du@apache.org>
Authored: Wed Dec 3 04:11:18 2014 -0800
Committer: Junping Du <junping_du@apache.org>
Committed: Wed Dec 3 04:11:18 2014 -0800

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                  |  3 +++
 .../nodemanager/metrics/NodeManagerMetrics.java  | 19 ++++++++++++++-----
 .../metrics/TestNodeManagerMetrics.java          | 17 ++++++++++++-----
 3 files changed, 29 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e65b7c5f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1e336b7..421e5ea 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -112,6 +112,9 @@ Release 2.7.0 - UNRELEASED
     YARN-2136. Changed RMStateStore to ignore store opearations when fenced.
     (Varun Saxena via jianhe)
 
+    YARN-1156. Enhance NodeManager AllocatedGB and AvailableGB metrics 
+    for aggregation of decimal values. (Tsuyoshi OZAWA via junping_du)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e65b7c5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
index a3637d5..beaafe1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
@@ -47,6 +47,9 @@ public class NodeManagerMetrics {
   @Metric("Container launch duration")
       MutableRate containerLaunchDuration;
 
+  private long allocatedMB;
+  private long availableMB;
+
   public static NodeManagerMetrics create() {
     return create(DefaultMetricsSystem.instance());
   }
@@ -92,22 +95,27 @@ public class NodeManagerMetrics {
 
   public void allocateContainer(Resource res) {
     allocatedContainers.incr();
-    allocatedGB.incr(res.getMemory() / 1024);
-    availableGB.decr(res.getMemory() / 1024);
+    allocatedMB = allocatedMB + res.getMemory();
+    allocatedGB.set((int)Math.ceil(allocatedMB/1024d));
+    availableMB = availableMB - res.getMemory();
+    availableGB.set((int)Math.floor(availableMB/1024d));
     allocatedVCores.incr(res.getVirtualCores());
     availableVCores.decr(res.getVirtualCores());
   }
 
   public void releaseContainer(Resource res) {
     allocatedContainers.decr();
-    allocatedGB.decr(res.getMemory() / 1024);
-    availableGB.incr(res.getMemory() / 1024);
+    allocatedMB = allocatedMB - res.getMemory();
+    allocatedGB.set((int)Math.ceil(allocatedMB/1024d));
+    availableMB = availableMB + res.getMemory();
+    availableGB.set((int)Math.floor(availableMB/1024d));
     allocatedVCores.decr(res.getVirtualCores());
     availableVCores.incr(res.getVirtualCores());
   }
 
   public void addResource(Resource res) {
-    availableGB.incr(res.getMemory() / 1024);
+    availableMB = availableMB + res.getMemory();
+    availableGB.incr((int)Math.floor(availableMB/1024d));
     availableVCores.incr(res.getVirtualCores());
   }
 
@@ -118,4 +126,5 @@ public class NodeManagerMetrics {
   public int getRunningContainers() {
     return containersRunning.value();
   }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e65b7c5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
index 9906e36..d2a0691 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
@@ -33,13 +33,14 @@ public class TestNodeManagerMetrics {
     total.setMemory(8*GiB);
     total.setVirtualCores(16);
     Resource resource = Records.newRecord(Resource.class);
-    resource.setMemory(1*GiB);
+    resource.setMemory(512); //512MiB
     resource.setVirtualCores(2);
 
 
     metrics.addResource(total);
 
-    for (int i = 5; i-- > 0;) {
+    for (int i = 10; i-- > 0;) {
+      // allocate 10 containers(allocatedGB: 5GiB, availableGB: 3GiB)
       metrics.launchedContainer();
       metrics.allocateContainer(resource);
     }
@@ -48,6 +49,7 @@ public class TestNodeManagerMetrics {
     metrics.endInitingContainer();
     metrics.runningContainer();
     metrics.endRunningContainer();
+    // Releasing 3 containers(allocatedGB: 3.5GiB, availableGB: 4.5GiB)
     metrics.completedContainer();
     metrics.releaseContainer(resource);
 
@@ -61,12 +63,17 @@ public class TestNodeManagerMetrics {
     metrics.runningContainer();
     metrics.addContainerLaunchDuration(1);
 
-    checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6, 4, 12);
+    // availableGB is expected to be floored,
+    // while allocatedGB is expected to be ceiled.
+    // allocatedGB: 3.5GB allocated memory is shown as 4GB
+    // availableGB: 4.5GB available memory is shown as 4GB
+    checkMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2);
   }
 
   private void checkMetrics(int launched, int completed, int failed, int killed,
-                            int initing, int running, int allocatedGB,
-                            int allocatedContainers, int availableGB, int allocatedVCores,
int availableVCores) {
+      int initing, int running, int allocatedGB,
+      int allocatedContainers, int availableGB, int allocatedVCores,
+      int availableVCores) {
     MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics");
     assertCounter("ContainersLaunched", launched, rb);
     assertCounter("ContainersCompleted", completed, rb);


Mime
View raw message