hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asur...@apache.org
Subject [50/50] [abbrv] hadoop git commit: YARN-5982. Simplify opportunistic container parameters and metrics. (Konstantinos Karanasos via asuresh)
Date Mon, 12 Dec 2016 16:09:26 GMT
YARN-5982. Simplify opportunistic container parameters and metrics. (Konstantinos Karanasos
via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0aace21
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0aace21
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0aace21

Branch: refs/heads/YARN-5085
Commit: b0aace21b1ef3436ba9d516186208fee9a9ceef2
Parents: 55f5886
Author: Arun Suresh <asuresh@apache.org>
Authored: Fri Dec 9 16:41:25 2016 -0800
Committer: Arun Suresh <asuresh@apache.org>
Committed: Fri Dec 9 16:41:25 2016 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/conf/YarnConfiguration.java     | 41 ------------
 .../src/main/resources/yarn-default.xml         | 57 -----------------
 .../scheduler/ContainerScheduler.java           | 10 +--
 .../nodemanager/metrics/NodeManagerMetrics.java | 65 ++++++++++----------
 ...pportunisticContainerAllocatorAMService.java | 60 ++++--------------
 .../scheduler/SchedulerNode.java                |  6 +-
 .../resourcemanager/webapp/NodesPage.java       |  6 +-
 .../resourcemanager/webapp/dao/NodeInfo.java    | 10 +--
 .../webapp/TestRMWebServicesNodes.java          | 10 +--
 9 files changed, 68 insertions(+), 197 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 4934964..69c7b00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -323,47 +323,6 @@ public class YarnConfiguration extends Configuration {
   public static final boolean
       OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED_DEFAULT = false;
 
-  /** Minimum memory (in MB) used for allocating an opportunistic container. */
-  public static final String OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB =
-      YARN_PREFIX + "opportunistic-containers.min-memory-mb";
-  public static final int OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB_DEFAULT = 512;
-
-  /** Minimum virtual CPU cores used for allocating an opportunistic container.
-   * */
-  public static final String OPPORTUNISTIC_CONTAINERS_MIN_VCORES =
-      YARN_PREFIX + "opportunistic-containers.min-vcores";
-  public static final int OPPORTUNISTIC_CONTAINERS_MIN_VCORES_DEFAULT = 1;
-
-  /** Maximum memory (in MB) used for allocating an opportunistic container. */
-  public static final String OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB =
-      YARN_PREFIX + "opportunistic-containers.max-memory-mb";
-  public static final int OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB_DEFAULT = 2048;
-
-  /** Maximum virtual CPU cores used for allocating an opportunistic container.
-   * */
-  public static final String OPPORTUNISTIC_CONTAINERS_MAX_VCORES =
-      YARN_PREFIX + "opportunistic-containers.max-vcores";
-  public static final int OPPORTUNISTIC_CONTAINERS_MAX_VCORES_DEFAULT = 4;
-
-  /** Incremental memory (in MB) used for allocating an opportunistic container.
-   * */
-  public static final String OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB =
-      YARN_PREFIX + "opportunistic-containers.incr-memory-mb";
-  public static final int OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB_DEFAULT =
-      512;
-
-  /** Incremental virtual CPU cores used for allocating an opportunistic
-   * container. */
-  public static final String OPPORTUNISTIC_CONTAINERS_INCR_VCORES =
-      YARN_PREFIX + "opportunistic-containers.incr-vcores";
-  public static final int OPPORTUNISTIC_CONTAINERS_INCR_VCORES_DEFAULT = 1;
-
-  /** Container token expiry for opportunistic containers. */
-  public static final String OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS =
-      YARN_PREFIX + "opportunistic-containers.container-token-expiry-ms";
-  public static final int OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT =
-      600000;
-
   /** Number of nodes to be used by the Opportunistic Container allocator for
    * dispatching containers during container allocation. */
   public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 47d12d1..c8c4edd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2766,63 +2766,6 @@
 
   <property>
     <description>
-      Minimum memory (in MB) used for allocating an opportunistic container.
-    </description>
-    <name>yarn.opportunistic-containers.min-memory-mb</name>
-    <value>512</value>
-  </property>
-
-  <property>
-    <description>
-      Minimum virtual CPU cores used for allocating an opportunistic container.
-    </description>
-    <name>yarn.opportunistic-containers.min-vcores</name>
-    <value>1</value>
-  </property>
-
-  <property>
-    <description>
-    Maximum memory (in MB) used for allocating an opportunistic container.
-    </description>
-    <name>yarn.opportunistic-containers.max-memory-mb</name>
-    <value>2048</value>
-  </property>
-
-  <property>
-    <description>
-    Maximum virtual CPU cores used for allocating an opportunistic container.
-    </description>
-    <name>yarn.opportunistic-containers.max-vcores</name>
-    <value>4</value>
-  </property>
-
-  <property>
-    <description>
-    Incremental memory (in MB) used for allocating an opportunistic container.
-    </description>
-    <name>yarn.opportunistic-containers.incr-memory-mb</name>
-    <value>512</value>
-  </property>
-
-  <property>
-    <description>
-    Incremental virtual CPU cores used for allocating an opportunistic
-      container.
-    </description>
-    <name>yarn.opportunistic-containers.incr-vcores</name>
-    <value>1</value>
-  </property>
-
-  <property>
-    <description>
-    Container token expiry for opportunistic containers.
-    </description>
-    <name>yarn.opportunistic-containers.container-token-expiry-ms</name>
-    <value>600000</value>
-  </property>
-
-  <property>
-    <description>
     Number of nodes to be used by the Opportunistic Container Allocator for
     dispatching containers during container allocation.
     </description>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 0c2b1ac..753bf04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -170,11 +170,11 @@ public class ContainerScheduler extends AbstractService implements
     this.opportunisticContainersStatus.setWaitQueueLength(
         getNumQueuedContainers());
     this.opportunisticContainersStatus.setOpportMemoryUsed(
-        metrics.getOpportMemoryUsed());
+        metrics.getAllocatedOpportunisticGB());
     this.opportunisticContainersStatus.setOpportCoresUsed(
-        metrics.getOpportCoresUsed());
+        metrics.getAllocatedOpportunisticVCores());
     this.opportunisticContainersStatus.setRunningOpportContainers(
-        metrics.getRunningOpportContainers());
+        metrics.getRunningOpportunisticContainers());
     return this.opportunisticContainersStatus;
   }
 
@@ -196,7 +196,7 @@ public class ContainerScheduler extends AbstractService implements
       this.utilizationTracker.subtractContainerResource(container);
       if (container.getContainerTokenIdentifier().getExecutionType() ==
           ExecutionType.OPPORTUNISTIC) {
-        this.metrics.opportunisticContainerCompleted(container);
+        this.metrics.completeOpportunisticContainer(container.getResource());
       }
       startPendingContainers();
     }
@@ -298,7 +298,7 @@ public class ContainerScheduler extends AbstractService implements
     this.utilizationTracker.addContainerResources(container);
     if (container.getContainerTokenIdentifier().getExecutionType() ==
         ExecutionType.OPPORTUNISTIC) {
-      this.metrics.opportunisticContainerStarted(container);
+      this.metrics.startOpportunisticContainer(container.getResource());
     }
     container.sendLaunchEvent();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
index b001b63..291b488 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
@@ -29,8 +29,6 @@ import org.apache.hadoop.metrics2.source.JvmMetrics;
 import org.apache.hadoop.yarn.api.records.Resource;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
-    .Container;
 
 @Metrics(about="Metrics for node manager", context="yarn")
 public class NodeManagerMetrics {
@@ -64,12 +62,12 @@ public class NodeManagerMetrics {
   @Metric("Disk utilization % on good log dirs")
       MutableGaugeInt goodLogDirsDiskUtilizationPerc;
 
-  @Metric("Memory used by Opportunistic Containers in MB")
-      MutableGaugeLong opportMemoryUsed;
-  @Metric("# of Virtual Cores used by opportunistic containers")
-      MutableGaugeInt opportCoresUsed;
+  @Metric("Current allocated memory by opportunistic containers in GB")
+      MutableGaugeLong allocatedOpportunisticGB;
+  @Metric("Current allocated Virtual Cores by opportunistic containers")
+      MutableGaugeInt allocatedOpportunisticVCores;
   @Metric("# of running opportunistic containers")
-      MutableGaugeInt runningOpportContainers;
+      MutableGaugeInt runningOpportunisticContainers;
 
   // CHECKSTYLE:ON:VisibilityModifier
 
@@ -77,6 +75,7 @@ public class NodeManagerMetrics {
 
   private long allocatedMB;
   private long availableMB;
+  private long allocatedOpportunisticMB;
 
   public NodeManagerMetrics(JvmMetrics jvmMetrics) {
     this.jvmMetrics = jvmMetrics;
@@ -141,30 +140,6 @@ public class NodeManagerMetrics {
     containersReIniting.decr();
   }
 
-  public long getOpportMemoryUsed() {
-    return opportMemoryUsed.value();
-  }
-
-  public int getOpportCoresUsed() {
-    return opportCoresUsed.value();
-  }
-
-  public int getRunningOpportContainers() {
-    return runningOpportContainers.value();
-  }
-
-  public void opportunisticContainerCompleted(Container container) {
-    opportMemoryUsed.decr(container.getResource().getMemorySize());
-    opportCoresUsed.decr(container.getResource().getVirtualCores());
-    runningOpportContainers.decr();
-  }
-
-  public void opportunisticContainerStarted(Container container) {
-    opportMemoryUsed.incr(container.getResource().getMemorySize());
-    opportCoresUsed.incr(container.getResource().getVirtualCores());
-    runningOpportContainers.incr();
-  }
-
   public void allocateContainer(Resource res) {
     allocatedContainers.incr();
     allocatedMB = allocatedMB + res.getMemorySize();
@@ -196,6 +171,22 @@ public class NodeManagerMetrics {
     availableVCores.decr(deltaVCores);
   }
 
+  public void startOpportunisticContainer(Resource res) {
+    runningOpportunisticContainers.incr();
+    allocatedOpportunisticMB = allocatedOpportunisticMB + res.getMemorySize();
+    allocatedOpportunisticGB
+        .set((int) Math.ceil(allocatedOpportunisticMB / 1024d));
+    allocatedOpportunisticVCores.incr(res.getVirtualCores());
+  }
+
+  public void completeOpportunisticContainer(Resource res) {
+    runningOpportunisticContainers.decr();
+    allocatedOpportunisticMB = allocatedOpportunisticMB - res.getMemorySize();
+    allocatedOpportunisticGB
+        .set((int) Math.ceil(allocatedOpportunisticMB / 1024d));
+    allocatedOpportunisticVCores.decr(res.getVirtualCores());
+  }
+
   public void addResource(Resource res) {
     availableMB = availableMB + res.getMemorySize();
     availableGB.incr((int)Math.floor(availableMB/1024d));
@@ -272,4 +263,16 @@ public class NodeManagerMetrics {
   public int getContainersRolledbackOnFailure() {
     return containersRolledBackOnFailure.value();
   }
+
+  public long getAllocatedOpportunisticGB() {
+    return allocatedOpportunisticGB.value();
+  }
+
+  public int getAllocatedOpportunisticVCores() {
+    return allocatedOpportunisticVCores.value();
+  }
+
+  public int getRunningOpportunisticContainers() {
+    return runningOpportunisticContainers.value();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index 7814b84..7b7963f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRespons
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
 
-import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -199,11 +198,12 @@ public class OpportunisticContainerAllocatorAMService
         }
       });
       int tokenExpiryInterval = getConfig()
-          .getInt(YarnConfiguration.OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS,
-              YarnConfiguration.
-                  OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT);
-      opCtx.updateAllocationParams(createMinContainerResource(),
-          createMaxContainerResource(), createIncrContainerResource(),
+          .getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
+              YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS);
+      opCtx.updateAllocationParams(
+          rmContext.getScheduler().getMinimumResourceCapability(),
+          rmContext.getScheduler().getMaximumResourceCapability(),
+          rmContext.getScheduler().getMinimumResourceCapability(),
           tokenExpiryInterval);
       appAttempt.setOpportunisticContainerContext(opCtx);
     }
@@ -273,14 +273,14 @@ public class OpportunisticContainerAllocatorAMService
     RegisterDistributedSchedulingAMResponse dsResp = recordFactory
         .newRecordInstance(RegisterDistributedSchedulingAMResponse.class);
     dsResp.setRegisterResponse(response);
-    dsResp.setMinContainerResource(createMinContainerResource());
-    dsResp.setMaxContainerResource(createMaxContainerResource());
-    dsResp.setIncrContainerResource(createIncrContainerResource());
+    dsResp.setMinContainerResource(
+        rmContext.getScheduler().getMinimumResourceCapability());
+    dsResp.setMaxContainerResource(
+        rmContext.getScheduler().getMaximumResourceCapability());
     dsResp.setContainerTokenExpiryInterval(
         getConfig().getInt(
-            YarnConfiguration.OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS,
-            YarnConfiguration.
-                OPPORTUNISTIC_CONTAINERS_TOKEN_EXPIRY_MS_DEFAULT));
+            YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
+            YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS));
     dsResp.setContainerIdStart(
         this.rmContext.getEpoch() << ResourceManager.EPOCH_BIT_SHIFT);
 
@@ -384,18 +384,6 @@ public class OpportunisticContainerAllocatorAMService
     return nodeMonitor.getThresholdCalculator();
   }
 
-  private Resource createIncrContainerResource() {
-    return Resource.newInstance(
-        getConfig().getInt(
-            YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB,
-            YarnConfiguration.
-                OPPORTUNISTIC_CONTAINERS_INCR_MEMORY_MB_DEFAULT),
-        getConfig().getInt(
-            YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES,
-            YarnConfiguration.OPPORTUNISTIC_CONTAINERS_INCR_VCORES_DEFAULT)
-    );
-  }
-
   private synchronized List<RemoteNode> getLeastLoadedNodes() {
     long currTime = System.currentTimeMillis();
     if ((currTime - lastCacheUpdateTime > cacheRefreshInterval)
@@ -425,30 +413,6 @@ public class OpportunisticContainerAllocatorAMService
         : null;
   }
 
-  private Resource createMaxContainerResource() {
-    return Resource.newInstance(
-        getConfig().getInt(
-            YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB,
-            YarnConfiguration
-                .OPPORTUNISTIC_CONTAINERS_MAX_MEMORY_MB_DEFAULT),
-        getConfig().getInt(
-            YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES,
-            YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MAX_VCORES_DEFAULT)
-    );
-  }
-
-  private Resource createMinContainerResource() {
-    return Resource.newInstance(
-        getConfig().getInt(
-            YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB,
-            YarnConfiguration.
-                OPPORTUNISTIC_CONTAINERS_MIN_MEMORY_MB_DEFAULT),
-        getConfig().getInt(
-            YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES,
-            YarnConfiguration.OPPORTUNISTIC_CONTAINERS_MIN_VCORES_DEFAULT)
-    );
-  }
-
   private static ApplicationAttemptId getAppAttemptId() throws YarnException {
     AMRMTokenIdentifier amrmTokenIdentifier =
         YarnServerSecurityUtils.authorizeRequest();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 995a7b0..6744c2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -149,10 +149,10 @@ public abstract class SchedulerNode {
    */
   public synchronized void allocateContainer(RMContainer rmContainer) {
     Container container = rmContainer.getContainer();
-    if (rmContainer.getExecutionType() != ExecutionType.OPPORTUNISTIC) {
+    if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
       deductUnallocatedResource(container.getResource());
+      ++numContainers;
     }
-    ++numContainers;
 
     launchedContainers.put(container.getId(), rmContainer);
 
@@ -251,8 +251,8 @@ public abstract class SchedulerNode {
       Container container) {
     if (container.getExecutionType() == ExecutionType.GUARANTEED) {
       addUnallocatedResource(container.getResource());
+      --numContainers;
     }
-    --numContainers;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index 7b68b84..c03df63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -49,6 +49,7 @@ class NodesPage extends RmView {
   static class NodesBlock extends HtmlBlock {
     final ResourceManager rm;
     private static final long BYTES_IN_MB = 1024 * 1024;
+    private static final long BYTES_IN_GB = 1024 * 1024 * 1024;
     private static boolean opportunisticContainersEnabled;
 
     @Inject
@@ -181,8 +182,9 @@ class NodesPage extends RmView {
           nodeTableData
               .append(String.valueOf(info.getNumRunningOpportContainers()))
               .append("\",\"").append("<br title='")
-              .append(String.valueOf(info.getUsedMemoryOpport())).append("'>")
-              .append(StringUtils.byteDesc(info.getUsedMemoryOpport()))
+              .append(String.valueOf(info.getUsedMemoryOpportGB())).append("'>")
+              .append(StringUtils.byteDesc(
+                  info.getUsedMemoryOpportGB() * BYTES_IN_GB))
               .append("\",\"")
               .append(String.valueOf(info.getUsedVirtualCoresOpport()))
               .append("\",\"")

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index aee7717..3416e52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -51,7 +51,7 @@ public class NodeInfo {
   protected long usedVirtualCores;
   protected long availableVirtualCores;
   private int numRunningOpportContainers;
-  private long usedMemoryOpport;    // Memory in bytes.
+  private long usedMemoryOpportGB;
   private long usedVirtualCoresOpport;
   private int numQueuedContainers;
   protected ArrayList<String> nodeLabels = new ArrayList<String>();
@@ -85,7 +85,7 @@ public class NodeInfo {
 
     // Status of opportunistic containers.
     this.numRunningOpportContainers = 0;
-    this.usedMemoryOpport = 0;
+    this.usedMemoryOpportGB = 0;
     this.usedVirtualCoresOpport = 0;
     this.numQueuedContainers = 0;
     OpportunisticContainersStatus opportStatus =
@@ -93,7 +93,7 @@ public class NodeInfo {
     if (opportStatus != null) {
       this.numRunningOpportContainers =
           opportStatus.getRunningOpportContainers();
-      this.usedMemoryOpport = opportStatus.getOpportMemoryUsed();
+      this.usedMemoryOpportGB = opportStatus.getOpportMemoryUsed();
       this.usedVirtualCoresOpport = opportStatus.getOpportCoresUsed();
       this.numQueuedContainers = opportStatus.getQueuedOpportContainers();
     }
@@ -165,8 +165,8 @@ public class NodeInfo {
     return numRunningOpportContainers;
   }
 
-  public long getUsedMemoryOpport() {
-    return usedMemoryOpport;
+  public long getUsedMemoryOpportGB() {
+    return usedMemoryOpportGB;
   }
 
   public long getUsedVirtualCoresOpport() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0aace21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index 10aa92a..fb597fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -726,7 +726,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
               "aggregatedContainersVirtualMemoryMB"),
           WebServicesTestUtils.getXmlFloat(element, "containersCPUUsage"),
           WebServicesTestUtils.getXmlInt(element, "numRunningOpportContainers"),
-          WebServicesTestUtils.getXmlLong(element, "usedMemoryOpport"),
+          WebServicesTestUtils.getXmlLong(element, "usedMemoryOpportGB"),
           WebServicesTestUtils.getXmlInt(element, "usedVirtualCoresOpport"),
           WebServicesTestUtils.getXmlInt(element, "numQueuedContainers"));
     }
@@ -753,7 +753,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
         resourceInfo.getInt("aggregatedContainersVirtualMemoryMB"),
         resourceInfo.getDouble("containersCPUUsage"),
         nodeInfo.getInt("numRunningOpportContainers"),
-        nodeInfo.getLong("usedMemoryOpport"),
+        nodeInfo.getLong("usedMemoryOpportGB"),
         nodeInfo.getInt("usedVirtualCoresOpport"),
         nodeInfo.getInt("numQueuedContainers"));
   }
@@ -766,7 +766,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
       int nodePhysicalMemoryMB, int nodeVirtualMemoryMB, double nodeCPUUsage,
       int containersPhysicalMemoryMB, int containersVirtualMemoryMB,
       double containersCPUUsage, int numRunningOpportContainers,
-      long usedMemoryOpport, int usedVirtualCoresOpport,
+      long usedMemoryOpportGB, int usedVirtualCoresOpport,
       int numQueuedContainers)
       throws JSONException, Exception {
 
@@ -827,8 +827,8 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
               numRunningOpportContainers,
           opportunisticStatus.getRunningOpportContainers(),
           numRunningOpportContainers);
-      assertEquals("usedMemoryOpport doesn't match: " + usedMemoryOpport,
-          opportunisticStatus.getOpportMemoryUsed(), usedMemoryOpport);
+      assertEquals("usedMemoryOpportGB doesn't match: " + usedMemoryOpportGB,
+          opportunisticStatus.getOpportMemoryUsed(), usedMemoryOpportGB);
       assertEquals(
           "usedVirtualCoresOpport doesn't match: " + usedVirtualCoresOpport,
           opportunisticStatus.getOpportCoresUsed(), usedVirtualCoresOpport);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message