hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1619980 - in /hadoop/common/branches/HDFS-6581/hadoop-yarn-project: ./ hadoop-yarn/bin/ hadoop-yarn/dev-support/ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ hadoop-yarn/hadoop-yarn-common/src/test/java/org/apa...
Date Sat, 23 Aug 2014 07:49:45 GMT
Author: arp
Date: Sat Aug 23 07:49:41 2014
New Revision: 1619980

URL: http://svn.apache.org/r1619980
Log:
Merging r1619458 through r1619979 from trunk to branch HDFS-6581

Removed:
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java
Modified:
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/bin/yarn
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
    hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/CHANGES.txt?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/CHANGES.txt Sat Aug 23 07:49:41 2014
@@ -18,6 +18,8 @@ Trunk - Unreleased 
     YARN-2216 TestRMApplicationHistoryWriter sometimes fails in trunk.
     (Zhijie Shen via xgong)
 
+    YARN-2436. [post-HADOOP-9902] yarn application help doesn't work (aw)
+
 Release 2.6.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -53,6 +55,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2174. Enable HTTPs for the writer REST API of TimelineServer.
     (Zhijie Shen via jianhe)
 
+    YARN-2393. FairScheduler: Add the notion of steady fair share. 
+    (Wei Yan via kasha)
+
   IMPROVEMENTS
 
     YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
@@ -229,6 +234,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2424. LCE should support non-cgroups, non-secure mode (Chris Douglas 
     via aw)
 
+    YARN-2434. RM should not recover containers from previously failed attempt
+    when AM restart is not enabled (Jian He via jlowe)
+
 Release 2.5.0 - 2014-08-11
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/bin/yarn
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/bin/yarn?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/bin/yarn (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/bin/yarn Sat Aug 23 07:49:41 2014
@@ -73,6 +73,7 @@ case "${COMMAND}" in
   application|applicationattempt|container)
     CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
     YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+    set -- "${COMMAND}" "$@"
   ;;
   classpath)
     hadoop_finalize

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml Sat Aug 23 07:49:41 2014
@@ -344,4 +344,11 @@
       <Class name="org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider"/>
       <Bug pattern="DC_DOUBLECHECK" />
   </Match>
+
+  <!-- ApplicationClassLoader is deprecated and moved to hadoop-common; ignore
+       warning on the identical name as it should be removed later -->
+  <Match>
+    <Class name="org.apache.hadoop.yarn.util.ApplicationClassLoader"/>
+    <Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS"/>
+  </Match>
 </FindBugsFilter>

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java Sat Aug 23 07:49:41 2014
@@ -18,180 +18,30 @@
 
 package org.apache.hadoop.yarn.util;
 
-import java.io.File;
-import java.io.FilenameFilter;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Splitter;
-
 /**
- * A {@link URLClassLoader} for YARN application isolation. Classes from
- * the application JARs are loaded in preference to the parent loader.
+ * This type has been deprecated in favor of
+ * {@link org.apache.hadoop.util.ApplicationClassLoader}. All new uses of
+ * ApplicationClassLoader should use that type instead.
  */
 @Public
 @Unstable
-public class ApplicationClassLoader extends URLClassLoader {
-
-  private static final Log LOG =
-    LogFactory.getLog(ApplicationClassLoader.class.getName());
-  
-  private static final FilenameFilter JAR_FILENAME_FILTER =
-    new FilenameFilter() {
-      @Override
-      public boolean accept(File dir, String name) {
-        return name.endsWith(".jar") || name.endsWith(".JAR");
-      }
-  };
-  
-  private ClassLoader parent;
-  private List<String> systemClasses;
-
+@Deprecated
+public class ApplicationClassLoader extends
+    org.apache.hadoop.util.ApplicationClassLoader {
   public ApplicationClassLoader(URL[] urls, ClassLoader parent,
       List<String> systemClasses) {
-    super(urls, parent);
-    this.parent = parent;
-    if (parent == null) {
-      throw new IllegalArgumentException("No parent classloader!");
-    }
-    this.systemClasses = systemClasses;
+    super(urls, parent, systemClasses);
   }
-  
+
   public ApplicationClassLoader(String classpath, ClassLoader parent,
       List<String> systemClasses) throws MalformedURLException {
-    this(constructUrlsFromClasspath(classpath), parent, systemClasses);
-  }
-  
-  @VisibleForTesting
-  static URL[] constructUrlsFromClasspath(String classpath)
-      throws MalformedURLException {
-    List<URL> urls = new ArrayList<URL>();
-    for (String element : Splitter.on(File.pathSeparator).split(classpath)) {
-      if (element.endsWith("/*")) {
-        String dir = element.substring(0, element.length() - 1);
-        File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
-        if (files != null) {
-          for (File file : files) {
-            urls.add(file.toURI().toURL());
-          }
-        }
-      } else {
-        File file = new File(element);
-        if (file.exists()) {
-          urls.add(new File(element).toURI().toURL());
-        }
-      }
-    }
-    return urls.toArray(new URL[urls.size()]);
-  }
-
-  @Override
-  public URL getResource(String name) {
-    URL url = null;
-    
-    if (!isSystemClass(name, systemClasses)) {
-      url= findResource(name);
-      if (url == null && name.startsWith("/")) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Remove leading / off " + name);
-        }
-        url= findResource(name.substring(1));
-      }
-    }
-
-    if (url == null) {
-      url= parent.getResource(name);
-    }
-
-    if (url != null) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("getResource("+name+")=" + url);
-      }
-    }
-    
-    return url;
-  }
-
-  @Override
-  public Class<?> loadClass(String name) throws ClassNotFoundException {
-    return this.loadClass(name, false);
-  }
-
-  @Override
-  protected synchronized Class<?> loadClass(String name, boolean resolve)
-      throws ClassNotFoundException {
-    
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Loading class: " + name);
-    }
-
-    Class<?> c = findLoadedClass(name);
-    ClassNotFoundException ex = null;
-
-    if (c == null && !isSystemClass(name, systemClasses)) {
-      // Try to load class from this classloader's URLs. Note that this is like
-      // the servlet spec, not the usual Java 2 behaviour where we ask the
-      // parent to attempt to load first.
-      try {
-        c = findClass(name);
-        if (LOG.isDebugEnabled() && c != null) {
-          LOG.debug("Loaded class: " + name + " ");
-        }
-      } catch (ClassNotFoundException e) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug(e);
-        }
-        ex = e;
-      }
-    }
-
-    if (c == null) { // try parent
-      c = parent.loadClass(name);
-      if (LOG.isDebugEnabled() && c != null) {
-        LOG.debug("Loaded class from parent: " + name + " ");
-      }
-    }
-
-    if (c == null) {
-      throw ex != null ? ex : new ClassNotFoundException(name);
-    }
-
-    if (resolve) {
-      resolveClass(c);
-    }
-
-    return c;
-  }
-
-  @VisibleForTesting
-  public static boolean isSystemClass(String name, List<String> systemClasses) {
-    if (systemClasses != null) {
-      String canonicalName = name.replace('/', '.');
-      while (canonicalName.startsWith(".")) {
-        canonicalName=canonicalName.substring(1);
-      }
-      for (String c : systemClasses) {
-        boolean result = true;
-        if (c.startsWith("-")) {
-          c = c.substring(1);
-          result = false;
-        }
-        if (c.endsWith(".") && canonicalName.startsWith(c)) {
-          return result;
-        } else if (canonicalName.equals(c)) {
-          return result;
-        }
-      }
-    }
-    return false;
+    super(classpath, parent, systemClasses);
   }
-}
\ No newline at end of file
+}

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java Sat Aug 23 07:49:41 2014
@@ -273,6 +273,19 @@ public abstract class AbstractYarnSchedu
       SchedulerApplicationAttempt schedulerAttempt =
           schedulerApp.getCurrentAppAttempt();
 
+      if (!rmApp.getApplicationSubmissionContext()
+        .getKeepContainersAcrossApplicationAttempts()) {
+        // Do not recover containers for stopped attempt or previous attempt.
+        if (schedulerAttempt.isStopped()
+            || !schedulerAttempt.getApplicationAttemptId().equals(
+              container.getContainerId().getApplicationAttemptId())) {
+          LOG.info("Skip recovering container " + container
+              + " for already stopped attempt.");
+          killOrphanContainerOnNode(nm, container);
+          continue;
+        }
+      }
+
       // create container
       RMContainer rmContainer = recoverAndCreateContainer(container, nm);
 

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java Sat Aug 23 07:49:41 2014
@@ -718,12 +718,6 @@ public class FSAppAttempt extends Schedu
   }
 
   @Override
-  public boolean isActive() {
-    return true;
-  }
-
-
-  @Override
   public void updateDemand() {
     demand = Resources.createResource(0);
     // Demand is current consumption plus outstanding requests

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java Sat Aug 23 07:49:41 2014
@@ -35,7 +35,6 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
 
@@ -68,6 +67,16 @@ public class FSParentQueue extends FSQue
     }
   }
 
+  public void recomputeSteadyShares() {
+    policy.computeSteadyShares(childQueues, getSteadyFairShare());
+    for (FSQueue childQueue : childQueues) {
+      childQueue.getMetrics().setSteadyFairShare(childQueue.getSteadyFairShare());
+      if (childQueue instanceof FSParentQueue) {
+        ((FSParentQueue) childQueue).recomputeSteadyShares();
+      }
+    }
+  }
+
   @Override
   public Resource getDemand() {
     return demand;

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Sat Aug 23 07:49:41 2014
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.util.resou
 @Unstable
 public abstract class FSQueue implements Queue, Schedulable {
   private Resource fairShare = Resources.createResource(0, 0);
+  private Resource steadyFairShare = Resources.createResource(0, 0);
   private final String name;
   protected final FairScheduler scheduler;
   private final FSQueueMetrics metrics;
@@ -151,7 +152,17 @@ public abstract class FSQueue implements
     this.fairShare = fairShare;
     metrics.setFairShare(fairShare);
   }
-  
+
+  /** Get the steady fair share assigned to this Schedulable. */
+  public Resource getSteadyFairShare() {
+    return steadyFairShare;
+  }
+
+  public void setSteadyFairShare(Resource steadyFairShare) {
+    this.steadyFairShare = steadyFairShare;
+    metrics.setSteadyFairShare(steadyFairShare);
+  }
+
   public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
     return scheduler.getAllocationConfiguration().hasAccess(name, acl, user);
   }
@@ -161,7 +172,7 @@ public abstract class FSQueue implements
    * queue's current share
    */
   public abstract void recomputeShares();
-  
+
   /**
    * Gets the children of this queue, if any.
    */
@@ -194,7 +205,9 @@ public abstract class FSQueue implements
     return true;
   }
 
-  @Override
+  /**
+   * Returns true if queue has at least one app running.
+   */
   public boolean isActive() {
     return getNumRunnableApps() > 0;
   }

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java Sat Aug 23 07:49:41 2014
@@ -33,6 +33,8 @@ public class FSQueueMetrics extends Queu
 
   @Metric("Fair share of memory in MB") MutableGaugeInt fairShareMB;
   @Metric("Fair share of CPU in vcores") MutableGaugeInt fairShareVCores;
+  @Metric("Steady fair share of memory in MB") MutableGaugeInt steadyFairShareMB;
+  @Metric("Steady fair share of CPU in vcores") MutableGaugeInt steadyFairShareVCores;
   @Metric("Minimum share of memory in MB") MutableGaugeInt minShareMB;
   @Metric("Minimum share of CPU in vcores") MutableGaugeInt minShareVCores;
   @Metric("Maximum share of memory in MB") MutableGaugeInt maxShareMB;
@@ -55,7 +57,20 @@ public class FSQueueMetrics extends Queu
   public int getFairShareVirtualCores() {
     return fairShareVCores.value();
   }
-  
+
+  public void setSteadyFairShare(Resource resource) {
+    steadyFairShareMB.set(resource.getMemory());
+    steadyFairShareVCores.set(resource.getVirtualCores());
+  }
+
+  public int getSteadyFairShareMB() {
+    return steadyFairShareMB.value();
+  }
+
+  public int getSteadyFairShareVCores() {
+    return steadyFairShareVCores.value();
+  }
+
   public void setMinShare(Resource resource) {
     minShareMB.set(resource.getMemory());
     minShareVCores.set(resource.getVirtualCores());

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Sat Aug 23 07:49:41 2014
@@ -851,6 +851,8 @@ public class FairScheduler extends
     Resources.addTo(clusterResource, node.getTotalCapability());
     updateRootQueueMetrics();
 
+    queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
+    queueMgr.getRootQueue().recomputeSteadyShares();
     LOG.info("Added node " + node.getNodeAddress() +
         " cluster capacity: " + clusterResource);
   }
@@ -885,6 +887,8 @@ public class FairScheduler extends
     }
 
     nodes.remove(rmNode.getNodeID());
+    queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
+    queueMgr.getRootQueue().recomputeSteadyShares();
     LOG.info("Removed node " + rmNode.getNodeAddress() +
         " cluster capacity: " + clusterResource);
   }

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Sat Aug 23 07:49:41 2014
@@ -118,6 +118,11 @@ public class QueueManager {
       if (queue == null && create) {
         // if the queue doesn't exist,create it and return
         queue = createQueue(name, queueType);
+
+        // Update steady fair share for all queues
+        if (queue != null) {
+          rootQueue.recomputeSteadyShares();
+        }
       }
       return queue;
     }
@@ -190,7 +195,7 @@ public class QueueManager {
         parent = newParent;
       }
     }
-    
+
     return parent;
   }
 
@@ -376,5 +381,8 @@ public class QueueManager {
             + queue.getName(), ex);
       }
     }
+
+    // Update steady fair shares for all queues
+    rootQueue.recomputeSteadyShares();
   }
 }

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java Sat Aug 23 07:49:41 2014
@@ -24,7 +24,6 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.util.resource.Resources;
 
 /**
  * A Schedulable represents an entity that can be scheduled such as an
@@ -102,10 +101,4 @@ public interface Schedulable {
 
   /** Assign a fair share to this Schedulable. */
   public void setFairShare(Resource fairShare);
-
-  /**
-   * Returns true if queue has atleast one app running. Always returns true for
-   * AppSchedulables.
-   */
-  public boolean isActive();
 }

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java Sat Aug 23 07:49:41 2014
@@ -17,10 +17,6 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.concurrent.ConcurrentHashMap;
-
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.util.ReflectionUtils;
@@ -29,6 +25,10 @@ import org.apache.hadoop.yarn.server.res
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
 
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.concurrent.ConcurrentHashMap;
+
 @Public
 @Evolving
 public abstract class SchedulingPolicy {
@@ -131,8 +131,10 @@ public abstract class SchedulingPolicy {
   public abstract Comparator<Schedulable> getComparator();
 
   /**
-   * Computes and updates the shares of {@link Schedulable}s as per the
-   * {@link SchedulingPolicy}, to be used later at schedule time.
+   * Computes and updates the shares of {@link Schedulable}s as per
+   * the {@link SchedulingPolicy}, to be used later for scheduling decisions.
+   * The shares computed are instantaneous and only consider queues with
+   * running applications.
    * 
    * @param schedulables {@link Schedulable}s whose shares are to be updated
    * @param totalResources Total {@link Resource}s in the cluster
@@ -141,6 +143,19 @@ public abstract class SchedulingPolicy {
       Collection<? extends Schedulable> schedulables, Resource totalResources);
 
   /**
+   * Computes and updates the steady shares of {@link FSQueue}s as per the
+   * {@link SchedulingPolicy}. The steady share does not differentiate
+   * between queues with and without running applications under them. The
+   * steady share is not used for scheduling, it is displayed on the Web UI
+   * for better visibility.
+   *
+   * @param queues {@link FSQueue}s whose shares are to be updated
+   * @param totalResources Total {@link Resource}s in the cluster
+   */
+  public abstract void computeSteadyShares(
+      Collection<? extends FSQueue> queues, Resource totalResources);
+
+  /**
    * Check if the resource usage is over the fair share under this policy
    *
    * @param usage {@link Resource} the resource usage

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java Sat Aug 23 07:49:41 2014
@@ -22,6 +22,7 @@ import java.util.Collection;
 
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
 
 /**
@@ -49,14 +50,29 @@ public class ComputeFairShares {
       ResourceType type) {
     Collection<Schedulable> activeSchedulables = new ArrayList<Schedulable>();
     for (Schedulable sched : schedulables) {
-      if (sched.isActive()) {
-        activeSchedulables.add(sched);
-      } else {
+      if ((sched instanceof FSQueue) && !((FSQueue) sched).isActive()) {
         setResourceValue(0, sched.getFairShare(), type);
+      } else {
+        activeSchedulables.add(sched);
       }
     }
 
-    computeSharesInternal(activeSchedulables, totalResources, type);
+    computeSharesInternal(activeSchedulables, totalResources, type, false);
+  }
+
+  /**
+   * Compute the steady fair share of the given queues. The steady fair
+   * share is an allocation of shares considering all queues, i.e.,
+   * active and inactive.
+   *
+   * @param queues
+   * @param totalResources
+   * @param type
+   */
+  public static void computeSteadyShares(
+      Collection<? extends FSQueue> queues, Resource totalResources,
+      ResourceType type) {
+    computeSharesInternal(queues, totalResources, type, true);
   }
 
   /**
@@ -102,7 +118,7 @@ public class ComputeFairShares {
    */
   private static void computeSharesInternal(
       Collection<? extends Schedulable> schedulables, Resource totalResources,
-      ResourceType type) {
+      ResourceType type, boolean isSteadyShare) {
     if (schedulables.isEmpty()) {
       return;
     }
@@ -145,7 +161,13 @@ public class ComputeFairShares {
     }
     // Set the fair shares based on the value of R we've converged to
     for (Schedulable sched : schedulables) {
-      setResourceValue(computeShare(sched, right, type), sched.getFairShare(), type);
+      if (isSteadyShare) {
+        setResourceValue(computeShare(sched, right, type),
+            ((FSQueue) sched).getSteadyFairShare(), type);
+      } else {
+        setResourceValue(
+            computeShare(sched, right, type), sched.getFairShare(), type);
+      }
     }
   }
 

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java Sat Aug 23 07:49:41 2014
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
 import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
 import org.apache.hadoop.yarn.util.resource.Resources;
@@ -68,6 +69,14 @@ public class DominantResourceFairnessPol
       ComputeFairShares.computeShares(schedulables, totalResources, type);
     }
   }
+
+  @Override
+  public void computeSteadyShares(Collection<? extends FSQueue> queues,
+      Resource totalResources) {
+    for (ResourceType type : ResourceType.values()) {
+      ComputeFairShares.computeSteadyShares(queues, totalResources, type);
+    }
+  }
   
   @Override
   public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java Sat Aug 23 07:49:41 2014
@@ -25,6 +25,7 @@ import org.apache.hadoop.classification.
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -120,6 +121,13 @@ public class FairSharePolicy extends Sch
   }
 
   @Override
+  public void computeSteadyShares(Collection<? extends FSQueue> queues,
+      Resource totalResources) {
+    ComputeFairShares.computeSteadyShares(queues, totalResources,
+        ResourceType.MEMORY);
+  }
+
+  @Override
   public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
     return Resources.greaterThan(RESOURCE_CALCULATOR, null, usage, fairShare);
   }

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java Sat Aug 23 07:49:41 2014
@@ -24,6 +24,7 @@ import java.util.Comparator;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
 import org.apache.hadoop.yarn.util.resource.Resources;
@@ -88,6 +89,13 @@ public class FifoPolicy extends Scheduli
   }
 
   @Override
+  public void computeSteadyShares(Collection<? extends FSQueue> queues,
+      Resource totalResources) {
+    // Nothing needs to do, as leaf queue doesn't have to calculate steady
+    // fair shares for applications.
+  }
+
+  @Override
   public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
     throw new UnsupportedOperationException(
         "FifoPolicy doesn't support checkIfUsageOverFairshare operation, " +

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java Sat Aug 23 07:49:41 2014
@@ -513,6 +513,19 @@ public class TestWorkPreservingRMRestart
     // just-recovered containers.
     assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
     assertNull(scheduler.getRMContainer(completedContainer.getContainerId()));
+
+    rm2.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
+
+    MockNM nm2 =
+        new MockNM("127.1.1.1:4321", 8192, rm2.getResourceTrackerService());
+    NMContainerStatus previousAttemptContainer =
+        TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 4,
+          ContainerState.RUNNING);
+    nm2.registerNode(Arrays.asList(previousAttemptContainer), null);
+    // Wait for RM to settle down on recovering containers;
+    Thread.sleep(3000);
+    // check containers from previous failed attempt should not be recovered.
+    assertNull(scheduler.getRMContainer(previousAttemptContainer.getContainerId()));
   }
 
   // Apps already completed before RM restart. Restarted RM scheduler should not

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java Sat Aug 23 07:49:41 2014
@@ -101,11 +101,6 @@ public class FakeSchedulable implements 
   }
 
   @Override
-  public boolean isActive() {
-    return true;
-  }
-
-  @Override
   public Resource getDemand() {
     return null;
   }

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Sat Aug 23 07:49:41 2014
@@ -292,14 +292,19 @@ public class TestFairScheduler extends F
     createSchedulingRequest(10 * 1024, "root.default", "user1");
 
     scheduler.update();
+    scheduler.getQueueManager().getRootQueue()
+        .setSteadyFairShare(scheduler.getClusterResource());
+    scheduler.getQueueManager().getRootQueue().recomputeSteadyShares();
 
     Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
     assertEquals(3, queues.size());
     
-    // Divided three ways - betwen the two queues and the default queue
+    // Divided three ways - between the two queues and the default queue
     for (FSLeafQueue p : queues) {
       assertEquals(3414, p.getFairShare().getMemory());
       assertEquals(3414, p.getMetrics().getFairShareMB());
+      assertEquals(3414, p.getSteadyFairShare().getMemory());
+      assertEquals(3414, p.getMetrics().getSteadyFairShareMB());
     }
   }
   
@@ -323,6 +328,9 @@ public class TestFairScheduler extends F
     createSchedulingRequest(10 * 1024, "root.default", "user1");
 
     scheduler.update();
+    scheduler.getQueueManager().getRootQueue()
+        .setSteadyFairShare(scheduler.getClusterResource());
+    scheduler.getQueueManager().getRootQueue().recomputeSteadyShares();
 
     QueueManager queueManager = scheduler.getQueueManager();
     Collection<FSLeafQueue> queues = queueManager.getLeafQueues();
@@ -333,10 +341,16 @@ public class TestFairScheduler extends F
     FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3", true);
     assertEquals(capacity / 2, queue1.getFairShare().getMemory());
     assertEquals(capacity / 2, queue1.getMetrics().getFairShareMB());
+    assertEquals(capacity / 2, queue1.getSteadyFairShare().getMemory());
+    assertEquals(capacity / 2, queue1.getMetrics().getSteadyFairShareMB());
     assertEquals(capacity / 4, queue2.getFairShare().getMemory());
     assertEquals(capacity / 4, queue2.getMetrics().getFairShareMB());
+    assertEquals(capacity / 4, queue2.getSteadyFairShare().getMemory());
+    assertEquals(capacity / 4, queue2.getMetrics().getSteadyFairShareMB());
     assertEquals(capacity / 4, queue3.getFairShare().getMemory());
     assertEquals(capacity / 4, queue3.getMetrics().getFairShareMB());
+    assertEquals(capacity / 4, queue3.getSteadyFairShare().getMemory());
+    assertEquals(capacity / 4, queue3.getMetrics().getSteadyFairShareMB());
   }
 
   @Test
@@ -771,6 +785,9 @@ public class TestFairScheduler extends F
     createSchedulingRequest(10 * 1024, "root.default", "user3");
 
     scheduler.update();
+    scheduler.getQueueManager().getRootQueue()
+        .setSteadyFairShare(scheduler.getClusterResource());
+    scheduler.getQueueManager().getRootQueue().recomputeSteadyShares();
 
     Collection<FSLeafQueue> leafQueues = scheduler.getQueueManager()
         .getLeafQueues();
@@ -780,12 +797,128 @@ public class TestFairScheduler extends F
           || leaf.getName().equals("root.parentq.user2")) {
         // assert that the fair share is 1/4th node1's capacity
         assertEquals(capacity / 4, leaf.getFairShare().getMemory());
+        // assert that the steady fair share is 1/4th node1's capacity
+        assertEquals(capacity / 4, leaf.getSteadyFairShare().getMemory());
         // assert weights are equal for both the user queues
         assertEquals(1.0, leaf.getWeights().getWeight(ResourceType.MEMORY), 0);
       }
     }
   }
-  
+
+  @Test
+  public void testSteadyFairShareWithReloadAndNodeAddRemove() throws Exception {
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
+    out.println("<queue name=\"root\">");
+    out.println("  <schedulingPolicy>drf</schedulingPolicy>");
+    out.println("  <queue name=\"child1\">");
+    out.println("    <weight>1</weight>");
+    out.println("  </queue>");
+    out.println("  <queue name=\"child2\">");
+    out.println("    <weight>1</weight>");
+    out.println("  </queue>");
+    out.println("</queue>");
+    out.println("</allocations>");
+    out.close();
+
+    scheduler.init(conf);
+    scheduler.start();
+    scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+    // The steady fair share for all queues should be 0
+    QueueManager queueManager = scheduler.getQueueManager();
+    assertEquals(0, queueManager.getLeafQueue("child1", false)
+        .getSteadyFairShare().getMemory());
+    assertEquals(0, queueManager.getLeafQueue("child2", false)
+        .getSteadyFairShare().getMemory());
+
+    // Add one node
+    RMNode node1 =
+        MockNodes
+            .newNodeInfo(1, Resources.createResource(6144), 1, "127.0.0.1");
+    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+    scheduler.handle(nodeEvent1);
+    assertEquals(6144, scheduler.getClusterResource().getMemory());
+
+    // The steady fair shares for all queues should be updated
+    assertEquals(2048, queueManager.getLeafQueue("child1", false)
+        .getSteadyFairShare().getMemory());
+    assertEquals(2048, queueManager.getLeafQueue("child2", false)
+        .getSteadyFairShare().getMemory());
+
+    // Reload the allocation configuration file
+    out = new PrintWriter(new FileWriter(ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
+    out.println("<queue name=\"root\">");
+    out.println("  <schedulingPolicy>drf</schedulingPolicy>");
+    out.println("  <queue name=\"child1\">");
+    out.println("    <weight>1</weight>");
+    out.println("  </queue>");
+    out.println("  <queue name=\"child2\">");
+    out.println("    <weight>2</weight>");
+    out.println("  </queue>");
+    out.println("  <queue name=\"child3\">");
+    out.println("    <weight>2</weight>");
+    out.println("  </queue>");
+    out.println("</queue>");
+    out.println("</allocations>");
+    out.close();
+    scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+    // The steady fair shares for all queues should be updated
+    assertEquals(1024, queueManager.getLeafQueue("child1", false)
+        .getSteadyFairShare().getMemory());
+    assertEquals(2048, queueManager.getLeafQueue("child2", false)
+        .getSteadyFairShare().getMemory());
+    assertEquals(2048, queueManager.getLeafQueue("child3", false)
+        .getSteadyFairShare().getMemory());
+
+    // Remove the node, steady fair shares should back to 0
+    NodeRemovedSchedulerEvent nodeEvent2 = new NodeRemovedSchedulerEvent(node1);
+    scheduler.handle(nodeEvent2);
+    assertEquals(0, scheduler.getClusterResource().getMemory());
+    assertEquals(0, queueManager.getLeafQueue("child1", false)
+        .getSteadyFairShare().getMemory());
+    assertEquals(0, queueManager.getLeafQueue("child2", false)
+        .getSteadyFairShare().getMemory());
+  }
+
+  @Test
+  public void testSteadyFairShareWithQueueCreatedRuntime() throws Exception {
+    conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+        SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
+    conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
+    scheduler.init(conf);
+    scheduler.start();
+    scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+    // Add one node
+    RMNode node1 =
+        MockNodes
+            .newNodeInfo(1, Resources.createResource(6144), 1, "127.0.0.1");
+    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+    scheduler.handle(nodeEvent1);
+    assertEquals(6144, scheduler.getClusterResource().getMemory());
+    assertEquals(6144, scheduler.getQueueManager().getRootQueue()
+        .getSteadyFairShare().getMemory());
+    assertEquals(6144, scheduler.getQueueManager()
+        .getLeafQueue("default", false).getSteadyFairShare().getMemory());
+
+    // Submit one application
+    ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1);
+    createApplicationWithAMResource(appAttemptId1, "default", "user1", null);
+    assertEquals(3072, scheduler.getQueueManager()
+        .getLeafQueue("default", false).getSteadyFairShare().getMemory());
+    assertEquals(3072, scheduler.getQueueManager()
+        .getLeafQueue("user1", false).getSteadyFairShare().getMemory());
+  }
+
   /**
    * Make allocation requests and ensure they are reflected in queue demand.
    */
@@ -873,7 +1006,7 @@ public class TestFairScheduler extends F
   }
 
   @Test
-  public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException, 
+  public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException,
       AllocationConfigurationException, ParserConfigurationException {
     conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
 

Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java Sat Aug 23 07:49:41 2014
@@ -109,13 +109,15 @@ public class TestFairSchedulerFairShare 
 
     for (FSLeafQueue leaf : leafQueues) {
       if (leaf.getName().startsWith("root.parentA")) {
-        assertEquals(0, (double) leaf.getFairShare().getMemory() / nodeCapacity
-            * 100, 0);
+        assertEquals(0, (double) leaf.getFairShare().getMemory() / nodeCapacity,
+            0);
       } else if (leaf.getName().startsWith("root.parentB")) {
-        assertEquals(0, (double) leaf.getFairShare().getMemory() / nodeCapacity
-            * 100, 0.1);
+        assertEquals(0, (double) leaf.getFairShare().getMemory() / nodeCapacity,
+            0);
       }
     }
+
+    verifySteadyFairShareMemory(leafQueues, nodeCapacity);
   }
 
   @Test
@@ -135,14 +137,15 @@ public class TestFairSchedulerFairShare 
         100,
         (double) scheduler.getQueueManager()
             .getLeafQueue("root.parentA.childA1", false).getFairShare()
-            .getMemory()
-            / nodeCapacity * 100, 0.1);
+            .getMemory() / nodeCapacity * 100, 0.1);
     assertEquals(
         0,
         (double) scheduler.getQueueManager()
             .getLeafQueue("root.parentA.childA2", false).getFairShare()
-            .getMemory()
-            / nodeCapacity * 100, 0.1);
+            .getMemory() / nodeCapacity, 0.1);
+
+    verifySteadyFairShareMemory(scheduler.getQueueManager().getLeafQueues(),
+        nodeCapacity);
   }
 
   @Test
@@ -167,6 +170,9 @@ public class TestFairSchedulerFairShare 
               .getMemory()
               / nodeCapacity * 100, .9);
     }
+
+    verifySteadyFairShareMemory(scheduler.getQueueManager().getLeafQueues(),
+        nodeCapacity);
   }
 
   @Test
@@ -206,6 +212,9 @@ public class TestFairSchedulerFairShare 
             .getLeafQueue("root.parentB.childB1", false).getFairShare()
             .getMemory()
             / nodeCapacity * 100, .9);
+
+    verifySteadyFairShareMemory(scheduler.getQueueManager().getLeafQueues(),
+        nodeCapacity);
   }
 
   @Test
@@ -253,6 +262,9 @@ public class TestFairSchedulerFairShare 
             .getLeafQueue("root.parentA.childA2", false).getFairShare()
             .getMemory()
             / nodeCapacity * 100, 0.1);
+
+    verifySteadyFairShareMemory(scheduler.getQueueManager().getLeafQueues(),
+        nodeCapacity);
   }
 
   @Test
@@ -304,5 +316,45 @@ public class TestFairSchedulerFairShare 
             .getLeafQueue("root.parentB.childB1", false).getFairShare()
             .getVirtualCores()
             / nodeVCores * 100, .9);
+    Collection<FSLeafQueue> leafQueues = scheduler.getQueueManager()
+        .getLeafQueues();
+
+    for (FSLeafQueue leaf : leafQueues) {
+      if (leaf.getName().startsWith("root.parentA")) {
+        assertEquals(0.2,
+            (double) leaf.getSteadyFairShare().getMemory() / nodeMem, 0.001);
+        assertEquals(0.2,
+            (double) leaf.getSteadyFairShare().getVirtualCores() / nodeVCores,
+            0.001);
+      } else if (leaf.getName().startsWith("root.parentB")) {
+        assertEquals(0.05,
+            (double) leaf.getSteadyFairShare().getMemory() / nodeMem, 0.001);
+        assertEquals(0.1,
+            (double) leaf.getSteadyFairShare().getVirtualCores() / nodeVCores,
+            0.001);
+      }
+    }
+  }
+
+  /**
+   * Verify whether steady fair shares for all leaf queues still follow
+   * their weight, not related to active/inactive status.
+   *
+   * @param leafQueues
+   * @param nodeCapacity
+   */
+  private void verifySteadyFairShareMemory(Collection<FSLeafQueue> leafQueues,
+      int nodeCapacity) {
+    for (FSLeafQueue leaf : leafQueues) {
+      if (leaf.getName().startsWith("root.parentA")) {
+        assertEquals(0.2,
+            (double) leaf.getSteadyFairShare().getMemory() / nodeCapacity,
+            0.001);
+      } else if (leaf.getName().startsWith("root.parentB")) {
+        assertEquals(0.05,
+            (double) leaf.getSteadyFairShare().getMemory() / nodeCapacity,
+            0.001);
+      }
+    }
   }
 }



Mime
View raw message