hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From haiboc...@apache.org
Subject [49/50] [abbrv] hadoop git commit: YARN-5473. Expose per-application over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Date Mon, 29 Jan 2018 05:11:07 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
index ed71ea2..0243443 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
@@ -56,8 +57,9 @@ public class ApplicationAttemptStateDataPBImpl extends
   private Container masterContainer = null;
   private ByteBuffer appAttemptTokens = null;
 
-  private Map<String, Long> resourceSecondsMap;
+  private Map<String, Long> guaranteedResourceSecondsMap;
   private Map<String, Long> preemptedResourceSecondsMap;
+  private Map<String, Long> opportunisticResourceSecondsMap;
 
   public ApplicationAttemptStateDataPBImpl() {
     builder = ApplicationAttemptStateDataProto.newBuilder();
@@ -243,30 +245,72 @@ public class ApplicationAttemptStateDataPBImpl extends
   }
 
   @Override
+  @Deprecated
   public long getMemorySeconds() {
-    ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
-    return p.getMemorySeconds();
+    return getGuaranteedMemorySeconds();
   }
  
   @Override
+  @Deprecated
   public long getVcoreSeconds() {
-    ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
-    return p.getVcoreSeconds();
+    return getGuaranteedVcoreSeconds();
   }
 
   @Override
+  @Deprecated
   public void setMemorySeconds(long memorySeconds) {
-    maybeInitBuilder();
-    builder.setMemorySeconds(memorySeconds);
+    setGuaranteedMemorySeconds(memorySeconds);
   }
  
   @Override
+  @Deprecated
   public void setVcoreSeconds(long vcoreSeconds) {
+    setGuaranteedVcoreSeconds(vcoreSeconds);
+  }
+
+  @Override
+  public long getGuaranteedMemorySeconds() {
+    ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getMemorySeconds();
+  }
+
+  @Override
+  public void setGuaranteedMemorySeconds(long memorySeconds) {
+    maybeInitBuilder();
+    builder.setMemorySeconds(memorySeconds);
+  }
+
+  @Override
+  public long getGuaranteedVcoreSeconds() {
+    ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getVcoreSeconds();
+  }
+
+  @Override
+  public void setGuaranteedVcoreSeconds(long vcoreSeconds) {
     maybeInitBuilder();
     builder.setVcoreSeconds(vcoreSeconds);
   }
 
   @Override
+  public long getOpportunisticMemorySeconds() {
+    Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+    if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
+      return tmp.get(ResourceInformation.MEMORY_MB.getName());
+    }
+    return 0;
+  }
+
+  @Override
+  public long getOpportunisticVcoreSeconds() {
+    Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+    if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
+      return tmp.get(ResourceInformation.VCORES.getName());
+    }
+    return 0;
+  }
+
+  @Override
   public long getPreemptedMemorySeconds() {
     ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
     return p.getPreemptedMemorySeconds();
@@ -410,21 +454,35 @@ public class ApplicationAttemptStateDataPBImpl extends
   }
 
   @Override
+  @Deprecated
   public Map<String, Long> getResourceSecondsMap() {
-    if (this.resourceSecondsMap != null) {
-      return this.resourceSecondsMap;
+    return getGuaranteedResourceSecondsMap();
+  }
+
+  @Override
+  @Deprecated
+  public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+    setGuaranteedResourceSecondsMap(resourceSecondsMap);
+  }
+
+  @Override
+  public Map<String, Long> getGuaranteedResourceSecondsMap() {
+    if (this.guaranteedResourceSecondsMap != null) {
+      return this.guaranteedResourceSecondsMap;
     }
     ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
-    this.resourceSecondsMap = ProtoUtils.convertStringLongMapProtoListToMap(
-        p.getApplicationResourceUsageMapList());
-    return this.resourceSecondsMap;
+    this.guaranteedResourceSecondsMap =
+        ProtoUtils.convertStringLongMapProtoListToMap(
+            p.getApplicationResourceUsageMapList());
+    return this.guaranteedResourceSecondsMap;
   }
 
   @Override
-  public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+  public void setGuaranteedResourceSecondsMap(
+      Map<String, Long> resourceSecondsMap) {
     maybeInitBuilder();
     builder.clearApplicationResourceUsageMap();
-    this.resourceSecondsMap = resourceSecondsMap;
+    this.guaranteedResourceSecondsMap = resourceSecondsMap;
     if (resourceSecondsMap != null) {
       builder.addAllApplicationResourceUsageMap(
           ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
@@ -454,4 +512,28 @@ public class ApplicationAttemptStateDataPBImpl extends
           .convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
     }
   }
+
+  @Override
+  public Map<String, Long> getOpportunisticResourceSecondsMap() {
+    if (this.opportunisticResourceSecondsMap != null) {
+      return this.opportunisticResourceSecondsMap;
+    }
+    ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+    this.opportunisticResourceSecondsMap =
+        ProtoUtils.convertStringLongMapProtoListToMap(
+            p.getApplicationOpportunisticResourceUsageMapList());
+    return this.opportunisticResourceSecondsMap;
+  }
+
+  @Override
+  public void setOpportunisticResourceSecondsMap(
+      Map<String, Long> resourceSecondsMap) {
+    maybeInitBuilder();
+    builder.clearApplicationOpportunisticResourceUsageMap();
+    this.opportunisticResourceSecondsMap = resourceSecondsMap;
+    if (resourceSecondsMap != null) {
+      builder.addAllApplicationOpportunisticResourceUsageMap(
+          ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 61936bb..53728c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -449,6 +449,7 @@ public class RMAppImpl implements RMApp, Recoverable {
     } else {
       this.startTime = startTime;
     }
+    LOG.info(applicationId.toString() + " started at " + this.startTime);
     this.applicationType = StringInterner.weakIntern(applicationType);
     this.applicationTags = applicationTags;
     this.amReqs = amReqs;
@@ -763,10 +764,12 @@ public class RMAppImpl implements RMApp, Recoverable {
         }
 
         RMAppMetrics rmAppMetrics = getRMAppMetrics();
-        appUsageReport
-            .setResourceSecondsMap(rmAppMetrics.getResourceSecondsMap());
+        appUsageReport.setGuaranteedResourceSecondsMap(
+            rmAppMetrics.getGuaranteedResourceSecondsMap());
         appUsageReport.setPreemptedResourceSecondsMap(
             rmAppMetrics.getPreemptedResourceSecondsMap());
+        appUsageReport.setOpportunisticResourceSecondsMap(
+            rmAppMetrics.getOpportunisticResourceSecondsMap());
       }
 
       if (currentApplicationAttemptId == null) {
@@ -1636,8 +1639,9 @@ public class RMAppImpl implements RMApp, Recoverable {
     Resource resourcePreempted = Resource.newInstance(0, 0);
     int numAMContainerPreempted = 0;
     int numNonAMContainerPreempted = 0;
-    Map<String, Long> resourceSecondsMap = new HashMap<>();
-    Map<String, Long> preemptedSecondsMap = new HashMap<>();
+    Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(2);
+    Map<String, Long> preemptedSecondsMap = new HashMap<>(2);
+    Map<String, Long> opportunsiticResourceSecondsMap = new HashMap<>(2);
     this.readLock.lock();
     try {
       for (RMAppAttempt attempt : attempts.values()) {
@@ -1653,20 +1657,15 @@ public class RMAppImpl implements RMApp, Recoverable {
           // for both running and finished containers.
           AggregateAppResourceUsage resUsage =
               attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
-          for (Map.Entry<String, Long> entry : resUsage
-              .getResourceUsageSecondsMap().entrySet()) {
-            long value = RMServerUtils
-                .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
-            value += entry.getValue();
-            resourceSecondsMap.put(entry.getKey(), value);
-          }
-          for (Map.Entry<String, Long> entry : attemptMetrics
-              .getPreemptedResourceSecondsMap().entrySet()) {
-            long value = RMServerUtils
-                .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L);
-            value += entry.getValue();
-            preemptedSecondsMap.put(entry.getKey(), value);
-          }
+          Resources.mergeResourceSecondsMap(
+              resUsage.getGuaranteedResourceUsageSecondsMap(),
+              guaranteedResourceSecondsMap);
+          Resources.mergeResourceSecondsMap(
+              resUsage.getOpportunisticResourceSecondsMap(),
+              opportunsiticResourceSecondsMap);
+          Resources.mergeResourceSecondsMap(
+              attemptMetrics.getPreemptedResourceSecondsMap(),
+              preemptedSecondsMap);
         }
       }
     } finally {
@@ -1674,7 +1673,8 @@ public class RMAppImpl implements RMApp, Recoverable {
     }
 
     return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted,
-        numAMContainerPreempted, resourceSecondsMap, preemptedSecondsMap);
+        numAMContainerPreempted, guaranteedResourceSecondsMap,
+        preemptedSecondsMap, opportunsiticResourceSecondsMap);
   }
 
   @Private

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
index 2bb7fd1..76272e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
@@ -28,18 +28,21 @@ public class RMAppMetrics {
   final Resource resourcePreempted;
   final int numNonAMContainersPreempted;
   final int numAMContainersPreempted;
-  private final Map<String, Long> resourceSecondsMap;
+  private final Map<String, Long> guaranteedResourceSecondsMap;
   private final Map<String, Long> preemptedResourceSecondsMap;
+  private final Map<String, Long> opportunisticResourceSecondsMap;
 
   public RMAppMetrics(Resource resourcePreempted,
       int numNonAMContainersPreempted, int numAMContainersPreempted,
-      Map<String, Long> resourceSecondsMap,
-      Map<String, Long> preemptedResourceSecondsMap) {
+      Map<String, Long> guaranteedResourceSecondsMap,
+      Map<String, Long> preemptedResourceSecondsMap,
+      Map<String, Long> opportunisticResourceSecondsMap) {
     this.resourcePreempted = resourcePreempted;
     this.numNonAMContainersPreempted = numNonAMContainersPreempted;
     this.numAMContainersPreempted = numAMContainersPreempted;
-    this.resourceSecondsMap = resourceSecondsMap;
+    this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
     this.preemptedResourceSecondsMap = preemptedResourceSecondsMap;
+    this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
   }
 
   public Resource getResourcePreempted() {
@@ -54,17 +57,25 @@ public class RMAppMetrics {
     return numAMContainersPreempted;
   }
 
-  public long getMemorySeconds() {
-    return RMServerUtils.getOrDefault(resourceSecondsMap,
+  public long getGuaranteedMemorySeconds() {
+    return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
         ResourceInformation.MEMORY_MB.getName(), 0L);
   }
 
-  public long getVcoreSeconds() {
-    return RMServerUtils
-        .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
-            0L);
+  public long getGuaranteedVcoreSeconds() {
+    return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+        ResourceInformation.VCORES.getName(), 0L);
+  }
+
+  public long getOpportunisticMemorySeconds() {
+    return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+        ResourceInformation.MEMORY_MB.getName(), 0L);
   }
 
+  public long getOpportunisticVcoreSeconds() {
+    return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+        ResourceInformation.VCORES.getName(), 0L);
+  }
   public long getPreemptedMemorySeconds() {
     return RMServerUtils.getOrDefault(preemptedResourceSecondsMap,
         ResourceInformation.MEMORY_MB.getName(), 0L);
@@ -75,12 +86,15 @@ public class RMAppMetrics {
         ResourceInformation.VCORES.getName(), 0L);
   }
 
-  public Map<String, Long> getResourceSecondsMap() {
-    return resourceSecondsMap;
+  public Map<String, Long> getGuaranteedResourceSecondsMap() {
+    return guaranteedResourceSecondsMap;
   }
 
   public Map<String, Long> getPreemptedResourceSecondsMap() {
     return preemptedResourceSecondsMap;
   }
 
+  public Map<String, Long> getOpportunisticResourceSecondsMap() {
+    return opportunisticResourceSecondsMap;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
index b858712..a75cc57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
@@ -27,30 +27,55 @@ import java.util.Map;
 
 @Private
 public class AggregateAppResourceUsage {
-  private Map<String, Long> resourceSecondsMap = new HashMap<>();
+  private Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+  private Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
 
-  public AggregateAppResourceUsage(Map<String, Long> resourceSecondsMap) {
-    this.resourceSecondsMap.putAll(resourceSecondsMap);
+  public AggregateAppResourceUsage(
+      Map<String, Long> guaranteedResourceSecondsMap,
+      Map<String, Long> opportunisticResourceSecondsMap) {
+    this.guaranteedResourceSecondsMap.putAll(
+        guaranteedResourceSecondsMap);
+    this.opportunisticResourceSecondsMap.putAll(
+        opportunisticResourceSecondsMap);
   }
 
   /**
-   * @return the memorySeconds
+   * @return the guaranteed memory seconds
    */
-  public long getMemorySeconds() {
-    return RMServerUtils.getOrDefault(resourceSecondsMap,
+  public long getGuaranteedMemorySeconds() {
+    return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
         ResourceInformation.MEMORY_MB.getName(), 0L);
   }
 
   /**
-   * @return the vcoreSeconds
+   * @return the guaranteed vcore seconds
    */
-  public long getVcoreSeconds() {
-    return RMServerUtils
-        .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
-            0L);
+  public long getGuaranteedVcoreSeconds() {
+    return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+        ResourceInformation.VCORES.getName(), 0L);
   }
 
-  public Map<String, Long> getResourceUsageSecondsMap() {
-    return resourceSecondsMap;
+  /**
+   * @return the opportunistic memory seconds
+   */
+  public long getOpportunisticMemorySeconds() {
+    return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+        ResourceInformation.MEMORY_MB.getName(), 0L);
+  }
+
+  /**
+   * @return the opportunistic vcore seconds
+   */
+  public long getOpportunisticVcoreSeconds() {
+    return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+        ResourceInformation.VCORES.getName(), 0L);
+  }
+
+  public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+    return guaranteedResourceSecondsMap;
+  }
+
+  public Map<String, Long> getOpportunisticResourceSecondsMap() {
+    return opportunisticResourceSecondsMap;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index cf10be4..6f46b33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -934,15 +934,18 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
     this.readLock.lock();
     try {
       ApplicationResourceUsageReport report =
-          scheduler.getAppResourceUsageReport(this.getAppAttemptId());
+          scheduler.getAppActiveResourceUsageReport(this.getAppAttemptId());
       if (report == null) {
         report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
       }
       AggregateAppResourceUsage resUsage =
           this.attemptMetrics.getAggregateAppResourceUsage();
-      report.setResourceSecondsMap(resUsage.getResourceUsageSecondsMap());
+      report.setGuaranteedResourceSecondsMap(
+          resUsage.getGuaranteedResourceUsageSecondsMap());
       report.setPreemptedResourceSecondsMap(
           this.attemptMetrics.getPreemptedResourceSecondsMap());
+      report.setOpportunisticResourceSecondsMap(
+          resUsage.getOpportunisticResourceSecondsMap());
       return report;
     } finally {
       this.readLock.unlock();
@@ -979,10 +982,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
     this.finalStatus = attemptState.getFinalApplicationStatus();
     this.startTime = attemptState.getStartTime();
     this.finishTime = attemptState.getFinishTime();
-    this.attemptMetrics
-        .updateAggregateAppResourceUsage(attemptState.getResourceSecondsMap());
+    this.attemptMetrics.updateAggregateAppGuaranteedResourceUsage(
+        attemptState.getGuaranteedResourceSecondsMap());
     this.attemptMetrics.updateAggregatePreemptedAppResourceUsage(
         attemptState.getPreemptedResourceSecondsMap());
+    this.attemptMetrics.updateAggregateAppOpportunisticResourceUsage(
+        attemptState.getOpportunisticResourceSecondsMap());
   }
 
   public void transferStateFromAttempt(RMAppAttempt attempt) {
@@ -1360,9 +1365,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
     ApplicationAttemptStateData attemptState = ApplicationAttemptStateData
         .newInstance(applicationAttemptId, getMasterContainer(),
             rmStore.getCredentialsFromAppAttempt(this), startTime,
-            stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus,
-            getFinishTime(), resUsage.getResourceUsageSecondsMap(),
-            this.attemptMetrics.getPreemptedResourceSecondsMap());
+            stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus,
+            exitStatus, getFinishTime(),
+            resUsage.getGuaranteedResourceUsageSecondsMap(),
+            this.attemptMetrics.getPreemptedResourceSecondsMap(),
+            resUsage.getOpportunisticResourceSecondsMap());
     LOG.info("Updating application attempt " + applicationAttemptId
         + " with final state: " + targetedFinalState + ", and exit status: "
         + exitStatus);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 015cff7..192722b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -53,8 +53,11 @@ public class RMAppAttemptMetrics {
   
   private ReadLock readLock;
   private WriteLock writeLock;
-  private Map<String, AtomicLong> resourceUsageMap = new HashMap<>();
+  private Map<String, AtomicLong> guaranteedResourceUsageMap
+      = new HashMap<>();
   private Map<String, AtomicLong> preemptedResourceMap = new HashMap<>();
+  private Map<String, AtomicLong> opportunisticResourceUsageMap
+      = new HashMap<>();
   private RMContext rmContext;
 
   private int[][] localityStatistics =
@@ -129,8 +132,10 @@ public class RMAppAttemptMetrics {
   }
 
   public AggregateAppResourceUsage getAggregateAppResourceUsage() {
-    Map<String, Long> resourcesUsed =
-        convertAtomicLongMaptoLongMap(resourceUsageMap);
+    Map<String, Long> guaranteedResourcesUsed =
+        convertAtomicLongMaptoLongMap(guaranteedResourceUsageMap);
+    Map<String, Long> opportunisticResourcesUsed =
+        convertAtomicLongMaptoLongMap(opportunisticResourceUsageMap);
 
     // Only add in the running containers if this is the active attempt.
     RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId());
@@ -139,27 +144,25 @@ public class RMAppAttemptMetrics {
       if (currentAttempt != null
           && currentAttempt.getAppAttemptId().equals(attemptId)) {
         ApplicationResourceUsageReport appResUsageReport =
-            rmContext.getScheduler().getAppResourceUsageReport(attemptId);
+            rmContext.getScheduler().getAppActiveResourceUsageReport(attemptId);
         if (appResUsageReport != null) {
-          Map<String, Long> tmp = appResUsageReport.getResourceSecondsMap();
-          for (Map.Entry<String, Long> entry : tmp.entrySet()) {
-            Long value = resourcesUsed.get(entry.getKey());
-            if (value != null) {
-              value += entry.getValue();
-            } else {
-              value = entry.getValue();
-            }
-            resourcesUsed.put(entry.getKey(), value);
-          }
+          Resources.mergeResourceSecondsMap(
+              appResUsageReport.getGuaranteedResourceSecondsMap(),
+              guaranteedResourcesUsed);
+          Resources.mergeResourceSecondsMap(
+              appResUsageReport.getOpportunisticResourceSecondsMap(),
+              opportunisticResourcesUsed);
         }
       }
     }
-    return new AggregateAppResourceUsage(resourcesUsed);
+
+    return new AggregateAppResourceUsage(guaranteedResourcesUsed,
+        opportunisticResourcesUsed);
   }
 
   public void updateAggregateAppResourceUsage(Resource allocated,
       long deltaUsedMillis) {
-    updateUsageMap(allocated, deltaUsedMillis, resourceUsageMap);
+    updateUsageMap(allocated, deltaUsedMillis, guaranteedResourceUsageMap);
   }
 
   public void updateAggregatePreemptedAppResourceUsage(Resource allocated,
@@ -167,9 +170,14 @@ public class RMAppAttemptMetrics {
     updateUsageMap(allocated, deltaUsedMillis, preemptedResourceMap);
   }
 
-  public void updateAggregateAppResourceUsage(
+  public void updateAggregateAppGuaranteedResourceUsage(
       Map<String, Long> resourceSecondsMap) {
-    updateUsageMap(resourceSecondsMap, resourceUsageMap);
+    updateUsageMap(resourceSecondsMap, guaranteedResourceUsageMap);
+  }
+
+  public void updateAggregateAppOpportunisticResourceUsage(
+      Map<String, Long> resourceSecondsMap) {
+    updateUsageMap(resourceSecondsMap, opportunisticResourceUsageMap);
   }
 
   public void updateAggregatePreemptedAppResourceUsage(
@@ -199,10 +207,9 @@ public class RMAppAttemptMetrics {
       if (!targetMap.containsKey(entry.getKey())) {
         resourceUsed = new AtomicLong(0);
         targetMap.put(entry.getKey(), resourceUsed);
-
       }
       resourceUsed = targetMap.get(entry.getKey());
-      resourceUsed.set(entry.getValue());
+      resourceUsed.addAndGet(entry.getValue());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
new file mode 100644
index 0000000..c91dae8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+import java.util.Map;
+
+/**
+ * The resource usage information of a <code>Container</code>.
+ */
+@Private
+public class ContainerResourceUsageReport {
+  private final Map<String, Long> guaranteedResourceSecondsMap;
+  private final Map<String, Long> opportunisticResourceSecondsMap;
+
+  public ContainerResourceUsageReport(
+      Map<String, Long> guaranteedResourceSecondsMap,
+      Map<String, Long> opportunisticResourceSecondsMap) {
+    this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
+    this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
+  }
+
+  public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+    return guaranteedResourceSecondsMap;
+  }
+
+  public Map<String, Long> getOpportunisticResourceSecondsMap() {
+    return opportunisticResourceSecondsMap;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
index f3cbf63..4c43c5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
 
-import java.util.List;
-
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -29,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest;
 import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -85,7 +82,9 @@ public interface RMContainer extends EventHandler<RMContainerEvent>,
   ContainerState getContainerState();
   
   ContainerReport createContainerReport();
-  
+
+  ContainerResourceUsageReport getResourceUsageReport();
+
   boolean isAMContainer();
 
   ContainerRequest getContainerRequest();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index e26689e..8f51d1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
 
 import java.util.Collections;
 import java.util.EnumSet;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
+import org.apache.commons.lang.time.DateUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -39,7 +41,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -176,7 +178,7 @@ public class RMContainerImpl implements RMContainer {
   private NodeId reservedNode;
   private SchedulerRequestKey reservedSchedulerKey;
   private long creationTime;
-  private long finishTime;
+  private long finishTime = -1L;
   private ContainerStatus finishedStatus;
   private boolean isAMContainer;
   private ContainerRequest containerRequestForRecovery;
@@ -189,6 +191,10 @@ public class RMContainerImpl implements RMContainer {
   private boolean isExternallyAllocated;
   private SchedulerRequestKey allocatedSchedulerKey;
 
+  // This is updated whenever the container instance is updated in
+  // the case of container increase/decrease/promotion/demotion
+  private long lastContainerUpdateTime;
+
   public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
       ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
       RMContext rmContext) {
@@ -230,6 +236,7 @@ public class RMContainerImpl implements RMContainer {
     this.appAttemptId = appAttemptId;
     this.user = user;
     this.creationTime = creationTime;
+    this.lastContainerUpdateTime = creationTime;
     this.rmContext = rmContext;
     this.eventHandler = rmContext.getDispatcher().getEventHandler();
     this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
@@ -277,9 +284,87 @@ public class RMContainerImpl implements RMContainer {
   }
 
   public void setContainer(Container container) {
+    // containers are updated by resetting the underlying Container instance,
+    // always update RMContainer resource usage before the update.
+    updateAppAttemptMetrics(true);
     this.container = container;
   }
 
+  /**
+   * Collect the resource usage information of the current underlying Container
+   * instance and update its RMAppAttemptMetrics. This is called whenever the
+   * RMContainer is updated (by updating its underlying container) or finishes.
+   * @param containerUpdated if it is called upon container update event
+   */
+  private void updateAppAttemptMetrics(boolean containerUpdated) {
+    RMAppAttempt rmAttempt = rmContext.getRMApps()
+        .get(getApplicationAttemptId().getApplicationId())
+        .getCurrentAppAttempt();
+
+    if (rmAttempt != null) {
+      // collect resource usage information of the current Container instance
+      ContainerResourceUsageReport resourceUsage =
+          getCurrentContainerResourceUsage(containerUpdated);
+
+      // If this is a preempted container, update preemption metrics
+      if (finishedStatus != null &&
+          ContainerExitStatus.PREEMPTED == finishedStatus.getExitStatus()) {
+        rmAttempt.getRMAppAttemptMetrics()
+            .updatePreemptionInfo(container.getResource(), this);
+        rmAttempt.getRMAppAttemptMetrics()
+            .updateAggregateAppOpportunisticResourceUsage(
+                resourceUsage.getGuaranteedResourceUsageSecondsMap());
+      }
+      rmAttempt.getRMAppAttemptMetrics()
+          .updateAggregateAppOpportunisticResourceUsage(
+              resourceUsage.getOpportunisticResourceSecondsMap());
+      rmAttempt.getRMAppAttemptMetrics()
+          .updateAggregateAppGuaranteedResourceUsage(
+              resourceUsage.getGuaranteedResourceUsageSecondsMap());
+    }
+  }
+
+  /**
+   * Get resource usage of the current underlying Container instance.
+   * @param containerUpdated if this is called upon container update
+   * @return resource usage of the current <code>container</code>
+   */
+  private ContainerResourceUsageReport getCurrentContainerResourceUsage(
+      boolean containerUpdated) {
+    ContainerResourceUsageReport report;
+
+    // A container generates usage until it finishes which is indicated
+    // by a positive finish timestamp that is set once it finishes
+    final long currentTimeMillis =
+        finishTime < 0 ? System.currentTimeMillis() : finishTime;
+    final long usedSeconds = (currentTimeMillis -
+        lastContainerUpdateTime) / DateUtils.MILLIS_PER_SECOND;
+    Resource resource = container.getResource();
+
+    if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+      Map<String, Long> guaranteedResourceSeconds = new HashMap<>(2);
+      for (ResourceInformation entry : resource.getResources()) {
+        guaranteedResourceSeconds.put(
+            entry.getName(), entry.getValue() * usedSeconds);
+      }
+      report = new ContainerResourceUsageReport(guaranteedResourceSeconds,
+          Collections.emptyMap());
+    } else {
+      Map<String, Long> opportunisticResourceSeconds = new HashMap<>(2);
+      for (ResourceInformation entry : resource.getResources()) {
+        opportunisticResourceSeconds.put(
+            entry.getName(), entry.getValue() * usedSeconds);
+      }
+      report = new ContainerResourceUsageReport(Collections.emptyMap(),
+          opportunisticResourceSeconds);
+    }
+
+    if (containerUpdated) {
+      lastContainerUpdateTime = currentTimeMillis;
+    }
+    return report;
+  }
+
   @Override
   public RMContainerState getState() {
     this.readLock.lock();
@@ -683,7 +768,7 @@ public class RMContainerImpl implements RMContainer {
       // Inform AppAttempt
       // container.getContainer() can return null when a RMContainer is a
       // reserved container
-      updateAttemptMetrics(container);
+      container.updateAppAttemptMetrics(false);
 
       container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
         container.appAttemptId, finishedEvent.getRemoteContainerStatus(),
@@ -705,27 +790,6 @@ public class RMContainerImpl implements RMContainer {
       }
 
     }
-
-    private static void updateAttemptMetrics(RMContainerImpl container) {
-      Resource resource = container.getContainer().getResource();
-      RMAppAttempt rmAttempt = container.rmContext.getRMApps()
-          .get(container.getApplicationAttemptId().getApplicationId())
-          .getCurrentAppAttempt();
-
-      if (rmAttempt != null) {
-        long usedMillis = container.finishTime - container.creationTime;
-        rmAttempt.getRMAppAttemptMetrics()
-            .updateAggregateAppResourceUsage(resource, usedMillis);
-        // If this is a preempted container, update preemption metrics
-        if (ContainerExitStatus.PREEMPTED == container.finishedStatus
-            .getExitStatus()) {
-          rmAttempt.getRMAppAttemptMetrics()
-              .updatePreemptionInfo(resource, container);
-          rmAttempt.getRMAppAttemptMetrics()
-              .updateAggregatePreemptedAppResourceUsage(resource, usedMillis);
-        }
-      }
-    }
   }
 
   private static final class KillTransition extends FinishedTransition {
@@ -764,6 +828,11 @@ public class RMContainerImpl implements RMContainer {
   }
 
   @Override
+  public ContainerResourceUsageReport getResourceUsageReport() {
+    return getCurrentContainerResourceUsage(false);
+  }
+
+  @Override
   public String getNodeHttpAddress() {
     try {
       readLock.lock();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index a755a4c..b78e93e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -412,7 +412,7 @@ public abstract class AbstractYarnScheduler
   }
 
   @Override
-  public ApplicationResourceUsageReport getAppResourceUsageReport(
+  public ApplicationResourceUsageReport getAppActiveResourceUsageReport(
       ApplicationAttemptId appAttemptId) {
     SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId);
     if (attempt == null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index f02f113..af4ef54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.time.DateUtils;
 import org.apache.commons.lang.time.FastDateFormat;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -60,12 +59,12 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerError;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.server.api.ContainerType;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -107,7 +106,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
 
   private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000;
   protected long lastMemoryAggregateAllocationUpdateTime = 0;
-  private Map<String, Long> lastResourceSecondsMap = new HashMap<>();
+  private Map<String, Long> lastGuaranteedResourceSecondsMap =
+      new HashMap<>();
+  private Map<String, Long> lastOpportunisticResourceSecondsMap =
+      new HashMap<>();
   protected final AppSchedulingInfo appSchedulingInfo;
   protected ApplicationAttemptId attemptId;
   protected Map<ContainerId, RMContainer> liveContainers =
@@ -1053,34 +1055,47 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
     // recently.
     if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime)
         > MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) {
-      Map<String, Long> resourceSecondsMap = new HashMap<>();
+      Map<String, Long> aggregatedGuaranteedResourceSecondsMap =
+          new HashMap<>();
+      Map<String, Long> aggregatedOpportunisticResourceSecondsMap =
+          new HashMap<>();
       for (RMContainer rmContainer : this.liveContainers.values()) {
-        long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
-        Resource resource = rmContainer.getContainer().getResource();
-        for (ResourceInformation entry : resource.getResources()) {
-          long value = RMServerUtils
-              .getOrDefault(resourceSecondsMap, entry.getName(), 0L);
-          value += entry.getValue() * usedMillis
-              / DateUtils.MILLIS_PER_SECOND;
-          resourceSecondsMap.put(entry.getName(), value);
-        }
+        ContainerResourceUsageReport containerResourceUsageReport =
+            rmContainer.getResourceUsageReport();
+        Resources.mergeResourceSecondsMap(
+            containerResourceUsageReport.getGuaranteedResourceUsageSecondsMap(),
+            aggregatedGuaranteedResourceSecondsMap);
+        Resources.mergeResourceSecondsMap(
+            containerResourceUsageReport.getOpportunisticResourceSecondsMap(),
+            aggregatedOpportunisticResourceSecondsMap);
       }
-
+      lastGuaranteedResourceSecondsMap =
+          aggregatedGuaranteedResourceSecondsMap;
+      lastOpportunisticResourceSecondsMap =
+          aggregatedOpportunisticResourceSecondsMap;
       lastMemoryAggregateAllocationUpdateTime = currentTimeMillis;
-      lastResourceSecondsMap = resourceSecondsMap;
     }
-    return new AggregateAppResourceUsage(lastResourceSecondsMap);
+    return new AggregateAppResourceUsage(lastGuaranteedResourceSecondsMap,
+        lastOpportunisticResourceSecondsMap);
   }
 
+  /**
+   * Get the resources that are actively being used by the app attempt.
+   * @return active resource usage
+   */
   public ApplicationResourceUsageReport getResourceUsageReport() {
     try {
       writeLock.lock();
       AggregateAppResourceUsage runningResourceUsage =
           getRunningAggregateAppResourceUsage();
-      Resource usedResourceClone = Resources.clone(
+      Resource guaranteedResourceUsedClone = Resources.clone(
           attemptResourceUsage.getAllUsed());
+      Resource opportunisticResourceUsedClone = Resources.clone(
+          attemptOpportunisticResourceUsage.getAllUsed());
       Resource reservedResourceClone = Resources.clone(
           attemptResourceUsage.getReserved());
+      Resource neededResource = Resources.add(opportunisticResourceUsedClone,
+          Resources.add(guaranteedResourceUsedClone, reservedResourceClone));
       Resource cluster = rmContext.getScheduler().getClusterResource();
       ResourceCalculator calc =
           rmContext.getScheduler().getResourceCalculator();
@@ -1095,18 +1110,20 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
         float queueCapacityPerc = queue.getQueueInfo(false, false)
             .getCapacity();
         if (queueCapacityPerc != 0) {
-          queueUsagePerc = calc.divide(cluster, usedResourceClone,
+          queueUsagePerc = calc.divide(cluster, guaranteedResourceUsedClone,
               Resources.multiply(cluster, queueCapacityPerc)) * 100;
         }
         clusterUsagePerc =
-            calc.divide(cluster, usedResourceClone, cluster) * 100;
+            calc.divide(cluster, guaranteedResourceUsedClone, cluster) * 100;
       }
       return ApplicationResourceUsageReport
           .newInstance(liveContainers.size(), reservedContainers.size(),
-              usedResourceClone, reservedResourceClone,
-              Resources.add(usedResourceClone, reservedResourceClone),
-              runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc,
-              clusterUsagePerc, preemptedResourceSecondsMaps);
+              guaranteedResourceUsedClone, reservedResourceClone,
+              neededResource,
+              runningResourceUsage.getGuaranteedResourceUsageSecondsMap(),
+              queueUsagePerc, clusterUsagePerc, preemptedResourceSecondsMaps,
+              opportunisticResourceUsedClone,
+              runningResourceUsage.getOpportunisticResourceSecondsMap());
     } finally {
       writeLock.unlock();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 93ca7c2..a8ec8b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -171,7 +171,7 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
    */
   @LimitedPrivate("yarn")
   @Evolving
-  ApplicationResourceUsageReport getAppResourceUsageReport(
+  ApplicationResourceUsageReport getAppActiveResourceUsageReport(
       ApplicationAttemptId appAttemptId);
   
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index cf651bb..75b2f81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -1017,7 +1017,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
             .getEffectiveCapacity(getAppAMNodePartitionName());
         if (!effCap.equals(Resources.none())) {
           queueUsagePerc = calc.divide(totalPartitionRes,
-              report.getUsedResources(), effCap) * 100;
+              report.getGuaranteedResourcesUsed(), effCap) * 100;
         }
         report.setQueueUsagePercentage(queueUsagePerc);
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 0719ca6..9ef4d31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -169,8 +169,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
       queue.getMetrics().releaseResources(
           rmContainer.getNodeLabelExpression(),
           getUser(), 1, containerResource);
-      this.attemptResourceUsage.decUsed(containerResource);
-      getQueue().decUsedGuaranteedResource(containerResource);
+      if (rmContainer.getExecutionType().equals(ExecutionType.GUARANTEED)) {
+        this.attemptResourceUsage.decUsed(containerResource);
+        getQueue().decUsedGuaranteedResource(containerResource);
+      } else {
+        this.attemptOpportunisticResourceUsage.decUsed(containerResource);
+      }
+
 
       // Clear resource utilization metrics cache.
       lastMemoryAggregateAllocationUpdateTime = -1;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index ac88f86..14dab9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -95,6 +95,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
             th(".runningcontainer", "Running Containers").
             th(".allocatedCpu", "Allocated CPU VCores").
             th(".allocatedMemory", "Allocated Memory MB").
+            th(".allocatedOpportunisticCpu",
+                "Allocated Opportunistic CPU Vcores").
+            th(".allocatedOpportunisticMemory",
+                "Allocated Opportunistic Memory MB").
             th(".reservedCpu", "Reserved CPU VCores").
             th(".reservedMemory", "Reserved Memory MB").
             th(".progress", "Progress").
@@ -144,6 +148,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
         .valueOf(appInfo.getAllocatedVCores())).append("\",\"")
       .append(appInfo.getAllocatedMB() == -1 ? "N/A" : String
         .valueOf(appInfo.getAllocatedMB())).append("\",\"")
+      .append(appInfo.getAllocatedOpportunisticVCores() == -1 ? "N/A" : String
+          .valueOf(appInfo.getAllocatedOpportunisticVCores())).append("\",\"")
+      .append(appInfo.getAllocatedOpportunisticMB() == -1 ? "N/A" : String
+          .valueOf(appInfo.getAllocatedOpportunisticMB())).append("\",\"")
       .append(appInfo.getReservedVCores() == -1 ? "N/A" : String
         .valueOf(appInfo.getReservedVCores())).append("\",\"")
       .append(appInfo.getReservedMB() == -1 ? "N/A" : String

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index 8553d8c..6cd10f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -107,9 +107,12 @@ public class RMAppBlock extends AppBlock{
           attemptResourcePreempted)
         .__("Number of Non-AM Containers Preempted from Current Attempt:",
           attemptNumNonAMContainerPreempted)
-        .__("Aggregate Resource Allocation:", appMetrics == null ? "N/A" :
-            StringHelper
-                .getResourceSecondsString(appMetrics.getResourceSecondsMap()))
+        .__("Aggregate Guaranteed Resource Allocation:", appMetrics == null ?
+            "N/A" : StringHelper.getResourceSecondsString(
+                appMetrics.getGuaranteedResourceSecondsMap()))
+        .__("Aggregate Opportunistic Resource Allocation:", appMetrics == null ?
+            "N/A" : StringHelper.getResourceSecondsString(
+                appMetrics.getOpportunisticResourceSecondsMap()))
         .__("Aggregate Preempted Resource Allocation:",
             appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString(
                 appMetrics.getPreemptedResourceSecondsMap()));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
index d0dccab..0510d62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -69,6 +69,10 @@ public class RMAppsBlock extends AppsBlock {
           .th(".allocatedMemory", "Allocated Memory MB")
           .th(".reservedCpu", "Reserved CPU VCores")
           .th(".reservedMemory", "Reserved Memory MB")
+          .th(".allocatedOpportunisticCpu",
+              "Allocated Opportunistic CPU VCores")
+          .th(".allocatedOpportunisticMemory",
+              "Allocated Opportunistic Memory MB")
           .th(".queuePercentage", "% of Queue")
           .th(".clusterPercentage", "% of Cluster")
           .th(".progress", "Progress")
@@ -155,6 +159,12 @@ public class RMAppsBlock extends AppsBlock {
         .append(app.getReservedMemoryMB() == -1 ? "N/A" :
             String.valueOf(app.getReservedMemoryMB()))
         .append("\",\"")
+        .append(app.getAllocatedOpportunisticCpuVcores() == -1 ? "N/A" : String
+            .valueOf(app.getAllocatedOpportunisticCpuVcores()))
+        .append("\",\"")
+        .append(app.getAllocatedOpportunisticMemoryMB() == -1 ? "N/A" :
+            String.valueOf(app.getAllocatedOpportunisticMemoryMB()))
+        .append("\",\"")
         .append(queuePercent)
         .append("\",\"")
         .append(clusterPercent)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 631c908..e7ac27e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -95,6 +95,8 @@ public class AppInfo {
   private String masterNodeId;
   private long allocatedMB;
   private long allocatedVCores;
+  private long allocatedOpportunisticMB;
+  private long allocatedOpportunisticVCores;
   private long reservedMB;
   private long reservedVCores;
   private int runningContainers;
@@ -103,6 +105,9 @@ public class AppInfo {
   protected float queueUsagePercentage;
   protected float clusterUsagePercentage;
   protected Map<String, Long> resourceSecondsMap;
+  private long opportunisticMemorySeconds;
+  private long opportunisticVcoreSeconds;
+  private Map<String, Long> opportunisticResourceSecondsMap;
 
   // preemption info fields
   private long preemptedResourceMB;
@@ -202,10 +207,17 @@ public class AppInfo {
           ApplicationResourceUsageReport resourceReport =
               attempt.getApplicationResourceUsageReport();
           if (resourceReport != null) {
-            Resource usedResources = resourceReport.getUsedResources();
+            Resource guaranteedResourcesUsed =
+                resourceReport.getGuaranteedResourcesUsed();
+            Resource opportunisticResourceUsed =
+                resourceReport.getOpportunisticResourcesUsed();
             Resource reservedResources = resourceReport.getReservedResources();
-            allocatedMB = usedResources.getMemorySize();
-            allocatedVCores = usedResources.getVirtualCores();
+            allocatedMB = guaranteedResourcesUsed.getMemorySize();
+            allocatedVCores = guaranteedResourcesUsed.getVirtualCores();
+            allocatedOpportunisticMB =
+                opportunisticResourceUsed.getMemorySize();
+            allocatedOpportunisticVCores =
+                opportunisticResourceUsed.getVirtualCores();
             reservedMB = reservedResources.getMemorySize();
             reservedVCores = reservedResources.getVirtualCores();
             runningContainers = resourceReport.getNumUsedContainers();
@@ -239,9 +251,13 @@ public class AppInfo {
       numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted();
       preemptedResourceVCores =
           appMetrics.getResourcePreempted().getVirtualCores();
-      memorySeconds = appMetrics.getMemorySeconds();
-      vcoreSeconds = appMetrics.getVcoreSeconds();
-      resourceSecondsMap = appMetrics.getResourceSecondsMap();
+      memorySeconds = appMetrics.getGuaranteedMemorySeconds();
+      vcoreSeconds = appMetrics.getGuaranteedVcoreSeconds();
+      resourceSecondsMap = appMetrics.getGuaranteedResourceSecondsMap();
+      opportunisticMemorySeconds = appMetrics.getOpportunisticMemorySeconds();
+      opportunisticVcoreSeconds = appMetrics.getOpportunisticVcoreSeconds();
+      opportunisticResourceSecondsMap =
+          appMetrics.getOpportunisticResourceSecondsMap();
       preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds();
       preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds();
       preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap();
@@ -454,6 +470,13 @@ public class AppInfo {
     return this.allocatedVCores;
   }
 
+  public long getAllocatedOpportunisticMB() {
+    return this.allocatedOpportunisticMB;
+  }
+
+  public long getAllocatedOpportunisticVCores() {
+    return this.allocatedOpportunisticVCores;
+  }
   public long getReservedMB() {
     return this.reservedMB;
   }
@@ -478,18 +501,30 @@ public class AppInfo {
     return numAMContainerPreempted;
   }
 
-  public long getMemorySeconds() {
+  public long getGuaranteedMemorySeconds() {
     return memorySeconds;
   }
 
-  public long getVcoreSeconds() {
+  public long getGuaranteedVcoreSeconds() {
     return vcoreSeconds;
   }
 
-  public Map<String, Long> getResourceSecondsMap() {
+  public long getOpportunisticMemorySeconds() {
+    return opportunisticMemorySeconds;
+  }
+
+  public long getOpportunisticVcoreSeconds() {
+    return opportunisticVcoreSeconds;
+  }
+
+  public Map<String, Long> getGuaranteedResourceSecondsMap() {
     return resourceSecondsMap;
   }
 
+  public Map<String, Long> getOpportunisticResourceSecondsMap() {
+    return opportunisticResourceSecondsMap;
+  }
+
   public long getPreemptedMemorySeconds() {
     return preemptedMemorySeconds;
   }
@@ -582,6 +617,15 @@ public class AppInfo {
     this.allocatedVCores = allocatedVCores;
   }
 
+  public void setAllocatedOpportunisticMB(long allocatedOpportunisticMB) {
+    this.allocatedOpportunisticMB = allocatedOpportunisticMB;
+  }
+
+  public void setAllocatedOpportunisticVCores(
+      long allocatedOpportunisticVCores) {
+    this.allocatedOpportunisticVCores = allocatedOpportunisticVCores;
+  }
+
   public void setReservedMB(long reservedMB) {
     this.reservedMB = reservedMB;
   }
@@ -594,12 +638,20 @@ public class AppInfo {
     this.runningContainers = runningContainers;
   }
 
-  public void setMemorySeconds(long memorySeconds) {
-    this.memorySeconds = memorySeconds;
+  public void setGuaranteedMemorySeconds(long guaranteedMemorySeconds) {
+    this.memorySeconds = guaranteedMemorySeconds;
+  }
+
+  public void setGuaranteedVcoreSeconds(long guaranteedVcoreSeconds) {
+    this.vcoreSeconds = guaranteedVcoreSeconds;
+  }
+
+  public void setOpportunisticMemorySeconds(long oppMemorySeconds) {
+    this.opportunisticMemorySeconds = oppMemorySeconds;
   }
 
-  public void setVcoreSeconds(long vcoreSeconds) {
-    this.vcoreSeconds = vcoreSeconds;
+  public void setOpportunisticVcoreSeconds(long oppVcoreSeconds) {
+    this.opportunisticVcoreSeconds = oppVcoreSeconds;
   }
 
   public void setAppId(String appId) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index 39a56a8..1721b1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -89,6 +89,7 @@ message ApplicationAttemptStateDataProto {
     optional int64 preempted_vcore_seconds = 14;
     repeated StringLongMapProto application_resource_usage_map = 15;
     repeated StringLongMapProto preempted_resource_usage_map = 16;
+    repeated StringLongMapProto application_opportunistic_resource_usage_map = 17;
 }
 
 message EpochProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 140c15f..6ad6006 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -1031,12 +1031,20 @@ public class TestAppManager{
     when(app.getState()).thenReturn(RMAppState.RUNNING);
     when(app.getApplicationType()).thenReturn("MAPREDUCE");
     when(app.getSubmitTime()).thenReturn(1000L);
-    Map<String, Long> resourceSecondsMap = new HashMap<>();
-    resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L);
-    resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L);
+    Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+    guaranteedResourceSecondsMap.put(
+        ResourceInformation.MEMORY_MB.getName(), 16384L);
+    guaranteedResourceSecondsMap.put(
+        ResourceInformation.VCORES.getName(), 64L);
+    Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
+    opportunisticResourceSecondsMap.put(
+        ResourceInformation.MEMORY_MB.getName(), 16384L);
+    opportunisticResourceSecondsMap.put(
+        ResourceInformation.VCORES.getName(), 64L);
     RMAppMetrics metrics =
         new RMAppMetrics(Resource.newInstance(1234, 56),
-            10, 1, resourceSecondsMap, new HashMap<>());
+            10, 1, guaranteedResourceSecondsMap,
+            new HashMap<>(), opportunisticResourceSecondsMap);
     when(app.getRMAppMetrics()).thenReturn(metrics);
 
     RMAppManager.ApplicationSummary.SummaryBuilder summary =
@@ -1047,16 +1055,19 @@ public class TestAppManager{
     Assert.assertFalse(msg.contains("\r"));
 
     String escaped = "\\n\\n\\r\\r";
-    assertTrue(msg.contains("Multiline" + escaped +"AppName"));
-    assertTrue(msg.contains("Multiline" + escaped +"UserName"));
-    assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
-    assertTrue(msg.contains("submitTime=1000"));
-    assertTrue(msg.contains("memorySeconds=16384"));
-    assertTrue(msg.contains("vcoreSeconds=64"));
-    assertTrue(msg.contains("preemptedAMContainers=1"));
-    assertTrue(msg.contains("preemptedNonAMContainers=10"));
-    assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
-    assertTrue(msg.contains("applicationType=MAPREDUCE"));
+    Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
+    Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
+    Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
+    Assert.assertTrue(msg.contains("submitTime=1000"));
+    Assert.assertTrue(msg.contains("guaranteedMemorySeconds=16384"));
+    Assert.assertTrue(msg.contains("guaranteedVcoreSeconds=64"));
+    Assert.assertTrue(msg.contains("opportunisticMemorySeconds=16384"));
+    Assert.assertTrue(msg.contains("opportunisticVcoreSeconds=64"));
+    Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
+    Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
+    Assert.assertTrue(
+        msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
+    Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
  }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
index c7ed02c..698ca90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
@@ -426,9 +426,11 @@ public class TestApplicationACLs extends ParameterizedSchedulerTestBase {
     Assert.assertEquals("Enemy should not see app reserved containers",
         -1, usageReport.getNumReservedContainers());
     Assert.assertEquals("Enemy should not see app used resources",
-        -1, usageReport.getUsedResources().getMemorySize());
+        -1, usageReport.getGuaranteedResourcesUsed().getMemorySize());
     Assert.assertEquals("Enemy should not see app reserved resources",
         -1, usageReport.getReservedResources().getMemorySize());
+    Assert.assertEquals("Enemy should not see app used opportunistic resources",
+        -1, usageReport.getOpportunisticResourcesUsed().getMemorySize());
     Assert.assertEquals("Enemy should not see app needed resources",
         -1, usageReport.getNeededResources().getMemorySize());
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d61cc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 1c50dd3..a66a91b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -370,8 +370,8 @@ public class TestClientRMService {
       ApplicationReport report = response.getApplicationReport();
       ApplicationResourceUsageReport usageReport = 
           report.getApplicationResourceUsageReport();
-      Assert.assertEquals(10, usageReport.getMemorySeconds());
-      Assert.assertEquals(3, usageReport.getVcoreSeconds());
+      Assert.assertEquals(10, usageReport.getGuaranteedMemorySeconds());
+      Assert.assertEquals(3, usageReport.getGuaranteedVcoreSeconds());
       Assert.assertEquals("<Not set>", report.getAmNodeLabelExpression());
       Assert.assertEquals("<Not set>", report.getAppNodeLabelExpression());
 
@@ -1402,26 +1402,28 @@ public class TestClientRMService {
     when(asContext.getMaxAppAttempts()).thenReturn(1);
     when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression);
     when(asContext.getPriority()).thenReturn(Priority.newInstance(0));
-    RMAppImpl app =
-        spy(new RMAppImpl(applicationId3, rmContext, config, null, null,
+    ResourceRequest resourceRequest = BuilderUtils.newResourceRequest(
+        RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
+        Resource.newInstance(1024, 1), 1);
+    RMAppImpl rmApp =
+        new RMAppImpl(applicationId3, rmContext, config, null, null,
             queueName, asContext, yarnScheduler, null,
             System.currentTimeMillis(), "YARN", null,
-            Collections.singletonList(BuilderUtils.newResourceRequest(
-                RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
-                Resource.newInstance(1024, 1), 1))){
-                  @Override
-                  public ApplicationReport createAndGetApplicationReport(
-                      String clientUserName, boolean allowAccess) {
-                    ApplicationReport report = super.createAndGetApplicationReport(
-                        clientUserName, allowAccess);
-                    ApplicationResourceUsageReport usageReport = 
-                        report.getApplicationResourceUsageReport();
-                    usageReport.setMemorySeconds(memorySeconds);
-                    usageReport.setVcoreSeconds(vcoreSeconds);
-                    report.setApplicationResourceUsageReport(usageReport);
-                    return report;
-                  }
-              });
+            Collections.singletonList(resourceRequest)) {
+          @Override
+          public ApplicationReport createAndGetApplicationReport(
+              String clientUserName, boolean allowAccess) {
+            ApplicationReport report = super.createAndGetApplicationReport(
+                clientUserName, allowAccess);
+            ApplicationResourceUsageReport usageReport =
+                report.getApplicationResourceUsageReport();
+            usageReport.setGuaranteedMemorySeconds(memorySeconds);
+            usageReport.setGuaranteedVcoreSeconds(vcoreSeconds);
+            report.setApplicationResourceUsageReport(usageReport);
+            return report;
+          }
+        };
+    RMAppImpl app = spy(rmApp);
     app.getAMResourceRequests().get(0)
         .setNodeLabelExpression(amNodeLabelExpression);
     ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
@@ -1474,7 +1476,8 @@ public class TestClientRMService {
     when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(
         Arrays.asList(getApplicationAttemptId(103)));
     ApplicationAttemptId attemptId = getApplicationAttemptId(1);
-    when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
+    when(yarnScheduler.getAppActiveResourceUsageReport(attemptId))
+        .thenReturn(null);
 
     ResourceCalculator rs = mock(ResourceCalculator.class);
     when(yarnScheduler.getResourceCalculator()).thenReturn(rs);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message