Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 03365905F for ; Fri, 10 Feb 2012 01:50:49 +0000 (UTC) Received: (qmail 23703 invoked by uid 500); 10 Feb 2012 01:50:48 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 23627 invoked by uid 500); 10 Feb 2012 01:50:48 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 23570 invoked by uid 99); 10 Feb 2012 01:50:48 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Feb 2012 01:50:48 +0000 X-ASF-Spam-Status: No, hits=-1996.1 required=5.0 tests=ALL_TRUSTED,FILL_THIS_FORM,FILL_THIS_FORM_FRAUD_PHISH,FILL_THIS_FORM_LONG X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Feb 2012 01:50:22 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 71F502388C27; Fri, 10 Feb 2012 01:49:47 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1242635 [8/10] - in /hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ bin/ conf/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/s... Date: Fri, 10 Feb 2012 01:49:30 -0000 To: mapreduce-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120210014947.71F502388C27@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml Fri Feb 10 01:49:08 2012 @@ -64,7 +64,6 @@ - Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Fri Feb 10 01:49:08 2012 @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.permission.F import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; @@ -128,6 +129,10 @@ public class DefaultContainerExecutor ex // Create the container log-dirs on all disks createContainerLogDirs(appIdStr, containerIdStr, logDirs); + Path tmpDir = new Path(containerWorkDir, + YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR); + lfs.mkdir(tmpDir, null, false); + // copy launch script to work dir Path launchDst = new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ApplicationPage.java Fri Feb 10 01:49:08 2012 @@ -44,6 +44,10 @@ public class ApplicationPage extends NMV @Override protected void preHead(Page.HTML<_> html) { commonPreHead(html); + + // Per-app information. Helps to refresh automatically. + html.meta_http("refresh", "10"); + set(DATATABLES_ID, "containers"); set(initID(DATATABLES, "containers"), containersTableInit()); setTableStyles(html, "containers"); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java Fri Feb 10 01:49:08 2012 @@ -62,7 +62,6 @@ public class ContainerLogsPage extends N String redirectUrl = $(REDIRECT_URL); if (redirectUrl == null || redirectUrl.isEmpty()) { set(TITLE, join("Logs for ", $(CONTAINER_ID))); - html.meta_http("refresh", "10"); } else { if (redirectUrl.equals("false")) { set(TITLE, join("Failed redirect for ", $(CONTAINER_ID))); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java Fri Feb 10 01:49:08 2012 @@ -40,6 +40,10 @@ public class ContainerPage extends NMVie @Override protected void preHead(Page.HTML<_> html) { commonPreHead(html); + + // Per-container information. Helps to refresh automatically. + html.meta_http("refresh", "10"); + setTitle("Container " + $(CONTAINER_ID)); set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMView.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMView.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMView.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMView.java Fri Feb 10 01:49:08 2012 @@ -33,7 +33,6 @@ public class NMView extends TwoColumnLay } protected void commonPreHead(Page.HTML<_> html) { - html.meta_http("refresh", "10"); set(ACCORDION_ID, "nav"); set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); set(THEMESWITCHER_ID, "themeswitcher"); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java Fri Feb 10 01:49:08 2012 @@ -41,6 +41,10 @@ public class NodePage extends NMView { @Override protected void commonPreHead(HTML<_> html) { super.commonPreHead(html); + + // Node summary page. Helps to refresh automatically. + html.meta_http("refresh", "10"); + set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}"); } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c Fri Feb 10 01:49:08 2012 @@ -315,6 +315,13 @@ char* get_app_log_directory(const char * } /** + * Get the tmp directory under the working directory + */ +char *get_tmp_directory(const char *work_dir) { + return concatenate("%s/%s", "tmp dir", 2, work_dir, TMP_DIR); +} + +/** * Ensure that the given path and all of the parent directories are created * with the desired permissions. */ @@ -357,7 +364,7 @@ int mkdirs(const char* path, mode_t perm * It creates the container work and log directories. */ static int create_container_directories(const char* user, const char *app_id, - const char *container_id, char* const* local_dir, char* const* log_dir) { + const char *container_id, char* const* local_dir, char* const* log_dir, const char *work_dir) { // create dirs as 0750 const mode_t perms = S_IRWXU | S_IRGRP | S_IXGRP; if (app_id == NULL || container_id == NULL || user == NULL) { @@ -409,6 +416,23 @@ static int create_container_directories( } free(combined_name); } + + if (result != 0) { + return result; + } + + result = -1; + // also make the tmp directory + char *tmp_dir = get_tmp_directory(work_dir); + + if (tmp_dir == NULL) { + return -1; + } + if (mkdirs(tmp_dir, perms) == 0) { + result = 0; + } + free(tmp_dir); + return result; } @@ -823,7 +847,7 @@ int launch_container_as_user(const char } if (create_container_directories(user, app_id, container_id, local_dirs, - log_dirs) != 0) { + log_dirs, work_dir) != 0) { fprintf(LOGFILE, "Could not create container dirs"); goto cleanup; } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h Fri Feb 10 01:49:08 2012 @@ -64,6 +64,7 @@ enum errorcodes { #define CREDENTIALS_FILENAME "container_tokens" #define MIN_USERID_KEY "min.user.id" #define BANNED_USERS_KEY "banned.users" +#define TMP_DIR "tmp" extern struct passwd *user_detail; Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Fri Feb 10 01:49:08 2012 @@ -28,6 +28,7 @@ import java.util.concurrent.atomic.Atomi import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.Text; @@ -75,7 +76,6 @@ import org.apache.hadoop.yarn.ipc.YarnRP import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; -import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; @@ -157,6 +157,11 @@ public class ClientRMService extends Abs super.start(); } + @Private + public InetSocketAddress getBindAddress() { + return clientBindAddress; + } + /** * check if the calling user has the access to application information. * @param callerUGI @@ -412,7 +417,7 @@ public class ClientRMService extends Abs SchedulerNodeReport schedulerNodeReport = scheduler.getNodeReport(rmNode.getNodeID()); - Resource used = Resources.none(); + Resource used = BuilderUtils.newResource(0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.getUsedResource(); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java Fri Feb 10 01:49:08 2012 @@ -38,7 +38,7 @@ public class ClusterMetrics { private static AtomicBoolean isInitialized = new AtomicBoolean(false); - @Metric("# of active NMs") MutableGaugeInt numNMs; + @Metric("# of active NMs") MutableGaugeInt numActiveNMs; @Metric("# of decommissioned NMs") MutableGaugeInt numDecommissionedNMs; @Metric("# of lost NMs") MutableGaugeInt numLostNMs; @Metric("# of unhealthy NMs") MutableGaugeInt numUnhealthyNMs; @@ -74,7 +74,7 @@ public class ClusterMetrics { //Active Nodemanagers public int getNumActiveNMs() { - return numNMs.value(); + return numActiveNMs.value(); } //Decommisioned NMs @@ -128,17 +128,12 @@ public class ClusterMetrics { public void decrNumRebootedNMs() { numRebootedNMs.decr(); } - - public void removeNode(RMNodeEventType nodeEventType) { - numNMs.decr(); - switch(nodeEventType){ - case DECOMMISSION: incrDecommisionedNMs(); break; - case EXPIRE: incrNumLostNMs();break; - case REBOOTING: incrNumRebootedNMs();break; - } + + public void incrNumActiveNodes() { + numActiveNMs.incr(); } - - public void addNode() { - numNMs.incr(); + + public void decrNumActiveNodes() { + numActiveNMs.decr(); } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Fri Feb 10 01:49:08 2012 @@ -110,7 +110,7 @@ public class ResourceManager extends Com protected ApplicationACLsManager applicationACLsManager; protected RMDelegationTokenSecretManager rmDTSecretManager; private WebApp webApp; - private RMContext rmContext; + protected RMContext rmContext; private final Store store; protected ResourceTrackerService resourceTracker; @@ -207,6 +207,8 @@ public class ResourceManager extends Com addService(applicationMasterLauncher); + new RMNMInfo(this.rmContext, this.scheduler); + super.init(conf); } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java Fri Feb 10 01:49:08 2012 @@ -111,4 +111,12 @@ public class Resources { public static boolean greaterThanOrEqual(Resource lhs, Resource rhs) { return lhs.getMemory() >= rhs.getMemory(); } + + public static Resource min(Resource lhs, Resource rhs) { + return (lhs.getMemory() < rhs.getMemory()) ? lhs : rhs; + } + + public static Resource max(Resource lhs, Resource rhs) { + return (lhs.getMemory() > rhs.getMemory()) ? lhs : rhs; + } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java Fri Feb 10 01:49:08 2012 @@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.util.Build */ @Private @Unstable +@SuppressWarnings("unchecked") public class RMNodeImpl implements RMNode, EventHandler { private static final Log LOG = LogFactory.getLog(RMNodeImpl.class); @@ -116,11 +117,14 @@ public class RMNodeImpl implements RMNod EnumSet.of(RMNodeState.RUNNING, RMNodeState.UNHEALTHY), RMNodeEventType.STATUS_UPDATE, new StatusUpdateWhenHealthyTransition()) .addTransition(RMNodeState.RUNNING, RMNodeState.DECOMMISSIONED, - RMNodeEventType.DECOMMISSION, new RemoveNodeTransition()) + RMNodeEventType.DECOMMISSION, + new DeactivateNodeTransition(RMNodeState.DECOMMISSIONED)) .addTransition(RMNodeState.RUNNING, RMNodeState.LOST, - RMNodeEventType.EXPIRE, new RemoveNodeTransition()) + RMNodeEventType.EXPIRE, + new DeactivateNodeTransition(RMNodeState.LOST)) .addTransition(RMNodeState.RUNNING, RMNodeState.REBOOTED, - RMNodeEventType.REBOOTING, new RemoveNodeTransition()) + RMNodeEventType.REBOOTING, + new DeactivateNodeTransition(RMNodeState.REBOOTED)) .addTransition(RMNodeState.RUNNING, RMNodeState.RUNNING, RMNodeEventType.CLEANUP_APP, new CleanUpAppTransition()) .addTransition(RMNodeState.RUNNING, RMNodeState.RUNNING, @@ -304,26 +308,50 @@ public class RMNodeImpl implements RMNod writeLock.unlock(); } } - + + private void updateMetricsForRejoinedNode(RMNodeState previousNodeState) { + ClusterMetrics metrics = ClusterMetrics.getMetrics(); + metrics.incrNumActiveNodes(); + + switch (previousNodeState) { + case LOST: + metrics.decrNumLostNMs(); + break; + case REBOOTED: + metrics.decrNumRebootedNMs(); + break; + case DECOMMISSIONED: + metrics.decrDecommisionedNMs(); + break; + case UNHEALTHY: + metrics.decrNumUnhealthyNMs(); + break; + } + } + + private void updateMetricsForDeactivatedNode(RMNodeState finalState) { + ClusterMetrics metrics = ClusterMetrics.getMetrics(); + metrics.decrNumActiveNodes(); + + switch (finalState) { + case DECOMMISSIONED: + metrics.incrDecommisionedNMs(); + break; + case LOST: + metrics.incrNumLostNMs(); + break; + case REBOOTED: + metrics.incrNumRebootedNMs(); + break; + case UNHEALTHY: + metrics.incrNumUnhealthyNMs(); + break; + } + } + public static class AddNodeTransition implements SingleArcTransition { - - private void updateMetrics(RMNodeState nodeState) { - ClusterMetrics metrics = ClusterMetrics.getMetrics(); - switch (nodeState) { - case LOST: - metrics.decrNumLostNMs(); - break; - case REBOOTED: - metrics.decrNumRebootedNMs(); - break; - case DECOMMISSIONED: - metrics.decrDecommisionedNMs(); - break; - } - } - @SuppressWarnings("unchecked") @Override public void transition(RMNodeImpl rmNode, RMNodeEvent event) { // Inform the scheduler @@ -333,12 +361,14 @@ public class RMNodeImpl implements RMNod String host = rmNode.nodeId.getHost(); if (rmNode.context.getInactiveRMNodes().containsKey(host)) { - RMNode node = rmNode.context.getInactiveRMNodes().get(host); + // Old node rejoining + RMNode previouRMNode = rmNode.context.getInactiveRMNodes().get(host); rmNode.context.getInactiveRMNodes().remove(host); - updateMetrics(node.getState()); + rmNode.updateMetricsForRejoinedNode(previouRMNode.getState()); + } else { + // Increment activeNodes explicitly because this is a new node. + ClusterMetrics.getMetrics().incrNumActiveNodes(); } - - ClusterMetrics.getMetrics().addNode(); } } @@ -362,28 +392,33 @@ public class RMNodeImpl implements RMNod } } - public static class RemoveNodeTransition + public static class DeactivateNodeTransition implements SingleArcTransition { - @SuppressWarnings("unchecked") + private final RMNodeState finalState; + public DeactivateNodeTransition(RMNodeState finalState) { + this.finalState = finalState; + } + @Override public void transition(RMNodeImpl rmNode, RMNodeEvent event) { // Inform the scheduler rmNode.context.getDispatcher().getEventHandler().handle( new NodeRemovedSchedulerEvent(rmNode)); - // Remove the node from the system. + // Deactivate the node rmNode.context.getRMNodes().remove(rmNode.nodeId); - LOG.info("Removed Node " + rmNode.nodeId); + LOG.info("Deactivating Node " + rmNode.nodeId + " as it is now " + + finalState); rmNode.context.getInactiveRMNodes().put(rmNode.nodeId.getHost(), rmNode); - //Update the metrics - ClusterMetrics.getMetrics().removeNode(event.getType()); + + //Update the metrics + rmNode.updateMetricsForDeactivatedNode(finalState); } } public static class StatusUpdateWhenHealthyTransition implements MultipleArcTransition { - @SuppressWarnings("unchecked") @Override public RMNodeState transition(RMNodeImpl rmNode, RMNodeEvent event) { @@ -399,7 +434,8 @@ public class RMNodeImpl implements RMNod // Inform the scheduler rmNode.context.getDispatcher().getEventHandler().handle( new NodeRemovedSchedulerEvent(rmNode)); - ClusterMetrics.getMetrics().incrNumUnhealthyNMs(); + // Update metrics + rmNode.updateMetricsForDeactivatedNode(RMNodeState.UNHEALTHY); return RMNodeState.UNHEALTHY; } @@ -458,11 +494,9 @@ public class RMNodeImpl implements RMNod } } - public static class StatusUpdateWhenUnHealthyTransition - implements + public static class StatusUpdateWhenUnHealthyTransition implements MultipleArcTransition { - @SuppressWarnings("unchecked") @Override public RMNodeState transition(RMNodeImpl rmNode, RMNodeEvent event) { RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event; @@ -474,7 +508,8 @@ public class RMNodeImpl implements RMNod if (remoteNodeHealthStatus.getIsNodeHealthy()) { rmNode.context.getDispatcher().getEventHandler().handle( new NodeAddedSchedulerEvent(rmNode)); - ClusterMetrics.getMetrics().decrNumUnhealthyNMs(); + // Update metrics + rmNode.updateMetricsForRejoinedNode(RMNodeState.UNHEALTHY); return RMNodeState.RUNNING; } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java Fri Feb 10 01:49:08 2012 @@ -162,6 +162,13 @@ public class AppSchedulingInfo { asks.put(hostName, request); if (updatePendingResources) { + + // Similarly, deactivate application? + if (request.getNumContainers() <= 0) { + LOG.info("checking for deactivate... "); + checkForDeactivation(); + } + int lastRequestContainers = lastRequest != null ? lastRequest .getNumContainers() : 0; Resource lastRequestCapability = lastRequest != null ? lastRequest @@ -308,19 +315,24 @@ public class AppSchedulingInfo { // Do we have any outstanding requests? // If there is nothing, we need to deactivate this application if (numOffSwitchContainers == 0) { - boolean deactivate = true; - for (Priority priority : getPriorities()) { - ResourceRequest request = getResourceRequest(priority, RMNodeImpl.ANY); - if (request.getNumContainers() > 0) { - deactivate = false; - break; - } - } - if (deactivate) { - activeUsersManager.deactivateApplication(user, applicationId); + checkForDeactivation(); + } + } + + synchronized private void checkForDeactivation() { + boolean deactivate = true; + for (Priority priority : getPriorities()) { + ResourceRequest request = getResourceRequest(priority, RMNodeImpl.ANY); + if (request.getNumContainers() > 0) { + deactivate = false; + break; } } + if (deactivate) { + activeUsersManager.deactivateApplication(user, applicationId); + } } + synchronized private void allocate(Container container) { // Update consumption and track allocations //TODO: fixme sharad Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java Fri Feb 10 01:49:08 2012 @@ -51,20 +51,19 @@ public class QueueMetrics { @Metric("# of apps killed") MutableCounterInt appsKilled; @Metric("# of apps failed") MutableCounterInt appsFailed; - @Metric("Allocated memory in GiB") MutableGaugeInt allocatedGB; + @Metric("Allocated memory in MB") MutableGaugeInt allocatedMB; @Metric("# of allocated containers") MutableGaugeInt allocatedContainers; @Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated; @Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased; - @Metric("Available memory in GiB") MutableGaugeInt availableGB; - @Metric("Pending memory allocation in GiB") MutableGaugeInt pendingGB; + @Metric("Available memory in MB") MutableGaugeInt availableMB; + @Metric("Pending memory allocation in MB") MutableGaugeInt pendingMB; @Metric("# of pending containers") MutableGaugeInt pendingContainers; - @Metric("# of reserved memory in GiB") MutableGaugeInt reservedGB; + @Metric("# of reserved memory in MB") MutableGaugeInt reservedMB; @Metric("# of reserved containers") MutableGaugeInt reservedContainers; @Metric("# of active users") MutableGaugeInt activeUsers; @Metric("# of active users") MutableGaugeInt activeApplications; static final Logger LOG = LoggerFactory.getLogger(QueueMetrics.class); - static final int GB = 1024; // resource.memory is in MB static final MetricsInfo RECORD_INFO = info("QueueMetrics", "Metrics for the resource scheduler"); static final MetricsInfo QUEUE_INFO = info("Queue", "Metrics by queue"); @@ -183,7 +182,7 @@ public class QueueMetrics { * @param limit resource limit */ public void setAvailableResourcesToQueue(Resource limit) { - availableGB.set(limit.getMemory()/GB); + availableMB.set(limit.getMemory()); } /** @@ -219,7 +218,7 @@ public class QueueMetrics { private void _incrPendingResources(int containers, Resource res) { pendingContainers.incr(containers); - pendingGB.incr(res.getMemory()/GB); + pendingMB.incr(res.getMemory()); } public void decrPendingResources(String user, int containers, Resource res) { @@ -235,13 +234,13 @@ public class QueueMetrics { private void _decrPendingResources(int containers, Resource res) { pendingContainers.decr(containers); - pendingGB.decr(res.getMemory()/GB); + pendingMB.decr(res.getMemory()); } public void allocateResources(String user, int containers, Resource res) { allocatedContainers.incr(containers); aggregateContainersAllocated.incr(containers); - allocatedGB.incr(res.getMemory()/GB * containers); + allocatedMB.incr(res.getMemory() * containers); _decrPendingResources(containers, multiply(res, containers)); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { @@ -255,7 +254,7 @@ public class QueueMetrics { public void releaseResources(String user, int containers, Resource res) { allocatedContainers.decr(containers); aggregateContainersReleased.incr(containers); - allocatedGB.decr(res.getMemory()/GB * containers); + allocatedMB.decr(res.getMemory() * containers); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.releaseResources(user, containers, res); @@ -267,7 +266,7 @@ public class QueueMetrics { public void reserveResource(String user, Resource res) { reservedContainers.incr(); - reservedGB.incr(res.getMemory()/GB); + reservedMB.incr(res.getMemory()); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.reserveResource(user, res); @@ -279,7 +278,7 @@ public class QueueMetrics { public void unreserveResource(String user, Resource res) { reservedContainers.decr(); - reservedGB.decr(res.getMemory()/GB); + reservedMB.decr(res.getMemory()); QueueMetrics userMetrics = getUserMetrics(user); if (userMetrics != null) { userMetrics.unreserveResource(user, res); @@ -343,28 +342,28 @@ public class QueueMetrics { return appsFailed.value(); } - public int getAllocatedGB() { - return allocatedGB.value(); + public int getAllocatedMB() { + return allocatedMB.value(); } public int getAllocatedContainers() { return allocatedContainers.value(); } - public int getAvailableGB() { - return availableGB.value(); + public int getAvailableMB() { + return availableMB.value(); } - public int getPendingGB() { - return pendingGB.value(); + public int getPendingMB() { + return pendingMB.value(); } public int getPendingContainers() { return pendingContainers.value(); } - public int getReservedGB() { - return reservedGB.value(); + public int getReservedMB() { + return reservedMB.value(); } public int getReservedContainers() { Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java Fri Feb 10 01:49:08 2012 @@ -99,6 +99,12 @@ extends org.apache.hadoop.yarn.server.re public float getUsedCapacity(); /** + * Set used capacity of the queue. + * @param usedCapacity used capacity of the queue + */ + public void setUsedCapacity(float usedCapacity); + + /** * Get the currently utilized resources in the cluster * by the queue and children (if any). * @return used resources by the queue and it's children @@ -115,6 +121,12 @@ extends org.apache.hadoop.yarn.server.re public float getUtilization(); /** + * Get the current utilization of the queue. + * @param utilization queue utilization + */ + public void setUtilization(float utilization); + + /** * Get the current run-state of the queue * @return current run-state */ Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java Fri Feb 10 01:49:08 2012 @@ -17,7 +17,9 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import org.apache.hadoop.yarn.Lock; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; class CSQueueUtils { @@ -46,17 +48,59 @@ class CSQueueUtils { } public static int computeMaxActiveApplications(Resource clusterResource, - float maxAMResourcePercent, float absoluteCapacity) { + Resource minimumAllocation, float maxAMResourcePercent, + float absoluteMaxCapacity) { return Math.max( - (int)((clusterResource.getMemory() / (float)LeafQueue.DEFAULT_AM_RESOURCE) * - maxAMResourcePercent * absoluteCapacity), + (int)Math.ceil( + ((float)clusterResource.getMemory() / + minimumAllocation.getMemory()) * + maxAMResourcePercent * absoluteMaxCapacity), 1); } public static int computeMaxActiveApplicationsPerUser( int maxActiveApplications, int userLimit, float userLimitFactor) { - return (int)(maxActiveApplications * (userLimit / 100.0f) * userLimitFactor); + return Math.max( + (int)Math.ceil( + maxActiveApplications * (userLimit / 100.0f) * userLimitFactor), + 1); } + @Lock(CSQueue.class) + public static void updateQueueStatistics( + final CSQueue childQueue, final CSQueue parentQueue, + final Resource clusterResource, final Resource minimumAllocation) { + final int clusterMemory = clusterResource.getMemory(); + final int usedMemory = childQueue.getUsedResources().getMemory(); + + float queueLimit = 0.0f; + float utilization = 0.0f; + float usedCapacity = 0.0f; + if (clusterMemory > 0) { + queueLimit = clusterMemory * childQueue.getAbsoluteCapacity(); + final float parentAbsoluteCapacity = + (parentQueue == null) ? 1.0f : parentQueue.getAbsoluteCapacity(); + utilization = (usedMemory / queueLimit); + usedCapacity = (usedMemory / (clusterMemory * parentAbsoluteCapacity)); + } + + childQueue.setUtilization(utilization); + childQueue.setUsedCapacity(usedCapacity); + + int available = + Math.max((roundUp(minimumAllocation, (int)queueLimit) - usedMemory), 0); + childQueue.getMetrics().setAvailableResourcesToQueue( + Resources.createResource(available)); + } + + public static int roundUp(Resource minimumAllocation, int memory) { + int minMemory = minimumAllocation.getMemory(); + return LeafQueue.divideAndCeil(memory, minMemory) * minMemory; + } + + public static int roundDown(Resource minimumAllocation, int memory) { + int minMemory = minimumAllocation.getMemory(); + return (memory / minMemory) * minMemory; + } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Fri Feb 10 01:49:08 2012 @@ -531,8 +531,10 @@ implements ResourceScheduler, CapacitySc private synchronized void nodeUpdate(RMNode nm, List newlyLaunchedContainers, List completedContainers) { - LOG.info("nodeUpdate: " + nm + " clusterResources: " + clusterResource); - + if (LOG.isDebugEnabled()) { + LOG.debug("nodeUpdate: " + nm + " clusterResources: " + clusterResource); + } + SchedulerNode node = getNode(nm.getNodeID()); // Processing the newly launched containers Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Fri Feb 10 01:49:08 2012 @@ -125,8 +125,6 @@ public class LeafQueue implements CSQueu private final ActiveUsersManager activeUsersManager; - final static int DEFAULT_AM_RESOURCE = 2 * 1024; - public LeafQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, Comparator applicationComparator, CSQueue old) { @@ -166,8 +164,9 @@ public class LeafQueue implements CSQueu this.maxAMResourcePercent = cs.getConfiguration().getMaximumApplicationMasterResourcePercent(); int maxActiveApplications = - CSQueueUtils.computeMaxActiveApplications(cs.getClusterResources(), - maxAMResourcePercent, absoluteCapacity); + CSQueueUtils.computeMaxActiveApplications( + cs.getClusterResources(), this.minimumAllocation, + maxAMResourcePercent, absoluteMaxCapacity); int maxActiveApplicationsPerUser = CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveApplications, userLimit, userLimitFactor); @@ -181,7 +180,9 @@ public class LeafQueue implements CSQueu Map acls = cs.getConfiguration().getAcls(getQueuePath()); - setupQueueConfigs(capacity, absoluteCapacity, + setupQueueConfigs( + cs.getClusterResources(), + capacity, absoluteCapacity, maximumCapacity, absoluteMaxCapacity, userLimit, userLimitFactor, maxApplications, maxApplicationsPerUser, @@ -199,6 +200,7 @@ public class LeafQueue implements CSQueu } private synchronized void setupQueueConfigs( + Resource clusterResource, float capacity, float absoluteCapacity, float maximumCapacity, float absoluteMaxCapacity, int userLimit, float userLimitFactor, @@ -236,6 +238,10 @@ public class LeafQueue implements CSQueu for (Map.Entry e : acls.entrySet()) { aclsString.append(e.getKey() + ":" + e.getValue().getAclString()); } + + // Update metrics + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); LOG.info("Initializing " + queueName + "\n" + "capacity = " + capacity + @@ -246,30 +252,39 @@ public class LeafQueue implements CSQueu " [= configuredMaxCapacity ]" + "\n" + "absoluteMaxCapacity = " + absoluteMaxCapacity + " [= 1.0 maximumCapacity undefined, " + - "(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]" + "\n" + + "(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]" + + "\n" + "userLimit = " + userLimit + " [= configuredUserLimit ]" + "\n" + "userLimitFactor = " + userLimitFactor + " [= configuredUserLimitFactor ]" + "\n" + "maxApplications = " + maxApplications + - " [= (int)(configuredMaximumSystemApplications * absoluteCapacity) ]" + "\n" + + " [= (int)(configuredMaximumSystemApplications * absoluteCapacity) ]" + + "\n" + "maxApplicationsPerUser = " + maxApplicationsPerUser + - " [= (int)(maxApplications * (userLimit / 100.0f) * userLimitFactor) ]" + "\n" + + " [= (int)(maxApplications * (userLimit / 100.0f) * " + + "userLimitFactor) ]" + "\n" + "maxActiveApplications = " + maxActiveApplications + " [= max(" + - "(int)((clusterResourceMemory / (float)DEFAULT_AM_RESOURCE) *" + - "maxAMResourcePercent * absoluteCapacity)," + + "(int)ceil((clusterResourceMemory / minimumAllocation) *" + + "maxAMResourcePercent * absoluteMaxCapacity)," + "1) ]" + "\n" + "maxActiveApplicationsPerUser = " + maxActiveApplicationsPerUser + - " [= (int)(maxActiveApplications * (userLimit / 100.0f) * userLimitFactor) ]" + "\n" + + " [= max(" + + "(int)(maxActiveApplications * (userLimit / 100.0f) * " + + "userLimitFactor)," + + "1) ]" + "\n" + "utilization = " + utilization + - " [= usedResourcesMemory / (clusterResourceMemory * absoluteCapacity)]" + "\n" + + " [= usedResourcesMemory / " + + "(clusterResourceMemory * absoluteCapacity)]" + "\n" + "usedCapacity = " + usedCapacity + - " [= usedResourcesMemory / (clusterResourceMemory * parent.absoluteCapacity)]" + "\n" + + " [= usedResourcesMemory / " + + "(clusterResourceMemory * parent.absoluteCapacity)]" + "\n" + "maxAMResourcePercent = " + maxAMResourcePercent + " [= configuredMaximumAMResourcePercent ]" + "\n" + "minimumAllocationFactor = " + minimumAllocationFactor + - " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / maximumAllocationMemory ]" + "\n" + + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " + + "maximumAllocationMemory ]" + "\n" + "numContainers = " + numContainers + " [= currentNumContainers ]" + "\n" + "state = " + state + @@ -378,11 +393,11 @@ public class LeafQueue implements CSQueu return null; } - synchronized void setUtilization(float utilization) { + public synchronized void setUtilization(float utilization) { this.utilization = utilization; } - synchronized void setUsedCapacity(float usedCapacity) { + public synchronized void setUsedCapacity(float usedCapacity) { this.usedCapacity = usedCapacity; } @@ -526,7 +541,9 @@ public class LeafQueue implements CSQueu } LeafQueue leafQueue = (LeafQueue)queue; - setupQueueConfigs(leafQueue.capacity, leafQueue.absoluteCapacity, + setupQueueConfigs( + clusterResource, + leafQueue.capacity, leafQueue.absoluteCapacity, leafQueue.maximumCapacity, leafQueue.absoluteMaxCapacity, leafQueue.userLimit, leafQueue.userLimitFactor, leafQueue.maxApplications, @@ -534,8 +551,6 @@ public class LeafQueue implements CSQueu leafQueue.getMaximumActiveApplications(), leafQueue.getMaximumActiveApplicationsPerUser(), leafQueue.state, leafQueue.acls); - - updateResource(clusterResource); } @Override @@ -606,7 +621,10 @@ public class LeafQueue implements CSQueu addApplication(application, user); } - metrics.submitApp(userName); + int attemptId = application.getApplicationAttemptId().getAttemptId(); + if (attemptId == 1) { + metrics.submitApp(userName); + } // Inform the parent queue try { @@ -635,7 +653,7 @@ public class LeafQueue implements CSQueu user.activateApplication(); activeApplications.add(application); i.remove(); - LOG.info("Application " + application.getApplicationId().getId() + + LOG.info("Application " + application.getApplicationId() + " from user: " + application.getUser() + " activated in queue: " + getQueueName()); } @@ -673,10 +691,13 @@ public class LeafQueue implements CSQueu } public synchronized void removeApplication(SchedulerApp application, User user) { - activeApplications.remove(application); + boolean wasActive = activeApplications.remove(application); + if (!wasActive) { + pendingApplications.remove(application); + } applicationsMap.remove(application.getApplicationAttemptId()); - user.finishApplication(); + user.finishApplication(wasActive); if (user.getTotalApplications() == 0) { users.remove(application.getUser()); } @@ -751,15 +772,15 @@ public class LeafQueue implements CSQueu continue; } - // Compute & set headroom - // Note: We set the headroom with the highest priority request - // as the target. + // Compute user-limit & set headroom + // Note: We compute both user-limit & headroom with the highest + // priority request as the target. // This works since we never assign lower priority requests // before all higher priority ones are serviced. Resource userLimit = - computeAndSetUserResourceLimit(application, clusterResource, - required); - + computeUserLimitAndSetHeadroom(application, clusterResource, + required); + // Check queue max-capacity limit if (!assignToQueue(clusterResource, required)) { return NULL_ASSIGNMENT; @@ -777,13 +798,13 @@ public class LeafQueue implements CSQueu CSAssignment assignment = assignContainersOnNode(clusterResource, node, application, priority, null); - - Resource assigned = assignment.getResource(); - + // Did we schedule or reserve a container? + Resource assigned = assignment.getResource(); if (Resources.greaterThan(assigned, Resources.none())) { - // Book-keeping + // Book-keeping + // Note: Update headroom to account for current allocation too... allocateResource(clusterResource, application, assigned); // Reset scheduling opportunities @@ -854,20 +875,41 @@ public class LeafQueue implements CSQueu } @Lock({LeafQueue.class, SchedulerApp.class}) - private Resource computeAndSetUserResourceLimit(SchedulerApp application, - Resource clusterResource, Resource required) { + private Resource computeUserLimitAndSetHeadroom( + SchedulerApp application, Resource clusterResource, Resource required) { + String user = application.getUser(); - Resource limit = computeUserLimit(application, clusterResource, required); + + /** + * Headroom is min((userLimit, queue-max-cap) - consumed) + */ + + Resource userLimit = // User limit + computeUserLimit(application, clusterResource, required); + + + Resource queueMaxCap = // Queue Max-Capacity + Resources.createResource( + CSQueueUtils.roundDown(minimumAllocation, + (int)(absoluteMaxCapacity * clusterResource.getMemory())) + ); + + Resource userConsumed = getUser(user).getConsumedResources(); Resource headroom = - Resources.subtract(limit, getUser(user).getConsumedResources()); + Resources.subtract(Resources.min(userLimit, queueMaxCap), userConsumed); + + if (LOG.isDebugEnabled()) { + LOG.debug("Headroom calculation for user " + user + ": " + + " userLimit=" + userLimit + + " queueMaxCap=" + queueMaxCap + + " consumed=" + userConsumed + + " headroom=" + headroom); + } + application.setHeadroom(headroom); metrics.setAvailableResourcesToUser(user, headroom); - return limit; - } - - private int roundUp(int memory) { - return divideAndCeil(memory, minimumAllocation.getMemory()) * - minimumAllocation.getMemory(); + + return userLimit; } @Lock(NoLock.class) @@ -883,8 +925,11 @@ public class LeafQueue implements CSQueu // Allow progress for queues with miniscule capacity final int queueCapacity = Math.max( - roundUp((int)(absoluteCapacity * clusterResource.getMemory())), - required.getMemory()); + CSQueueUtils.roundUp( + minimumAllocation, + (int)(absoluteCapacity * clusterResource.getMemory())), + required.getMemory() + ); final int consumed = usedResources.getMemory(); final int currentCapacity = @@ -899,7 +944,8 @@ public class LeafQueue implements CSQueu final int activeUsers = activeUsersManager.getNumActiveUsers(); int limit = - roundUp( + CSQueueUtils.roundUp( + minimumAllocation, Math.min( Math.max(divideAndCeil(currentCapacity, activeUsers), divideAndCeil((int)userLimit*currentCapacity, 100)), @@ -947,7 +993,7 @@ public class LeafQueue implements CSQueu return true; } - private static int divideAndCeil(int a, int b) { + static int divideAndCeil(int a, int b) { if (b == 0) { LOG.info("divideAndCeil called with a=" + a + " b=" + b); return 0; @@ -1281,24 +1327,33 @@ public class LeafQueue implements CSQueu SchedulerApp application, Resource resource) { // Update queue metrics Resources.addTo(usedResources, resource); - updateResource(clusterResource); + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); ++numContainers; // Update user metrics String userName = application.getUser(); User user = getUser(userName); user.assignContainer(resource); + Resources.subtractFrom(application.getHeadroom(), resource); // headroom metrics.setAvailableResourcesToUser(userName, application.getHeadroom()); - LOG.info(getQueueName() + - " used=" + usedResources + " numContainers=" + numContainers + - " user=" + userName + " user-resources=" + user.getConsumedResources()); + + if (LOG.isDebugEnabled()) { + LOG.info(getQueueName() + + " user=" + userName + + " used=" + usedResources + " numContainers=" + numContainers + + " headroom = " + application.getHeadroom() + + " user-resources=" + user.getConsumedResources() + ); + } } synchronized void releaseResource(Resource clusterResource, SchedulerApp application, Resource resource) { // Update queue metrics Resources.subtractFrom(usedResources, resource); - updateResource(clusterResource); + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); --numContainers; // Update user metrics @@ -1316,33 +1371,26 @@ public class LeafQueue implements CSQueu public synchronized void updateClusterResource(Resource clusterResource) { // Update queue properties maxActiveApplications = - CSQueueUtils.computeMaxActiveApplications(clusterResource, maxAMResourcePercent, - absoluteCapacity); + CSQueueUtils.computeMaxActiveApplications( + clusterResource, minimumAllocation, + maxAMResourcePercent, absoluteMaxCapacity); maxActiveApplicationsPerUser = - CSQueueUtils.computeMaxActiveApplicationsPerUser(maxActiveApplications, userLimit, - userLimitFactor); + CSQueueUtils.computeMaxActiveApplicationsPerUser( + maxActiveApplications, userLimit, userLimitFactor); + + // Update metrics + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); // Update application properties for (SchedulerApp application : activeApplications) { synchronized (application) { - computeAndSetUserResourceLimit( - application, clusterResource, Resources.none()); + computeUserLimitAndSetHeadroom(application, clusterResource, + Resources.none()); } } } - private synchronized void updateResource(Resource clusterResource) { - float queueLimit = clusterResource.getMemory() * absoluteCapacity; - setUtilization(usedResources.getMemory() / queueLimit); - setUsedCapacity(usedResources.getMemory() - / (clusterResource.getMemory() * parent.getAbsoluteCapacity())); - - Resource resourceLimit = - Resources.createResource(roundUp((int)queueLimit)); - metrics.setAvailableResourcesToQueue( - Resources.subtractFrom(resourceLimit, usedResources)); - } - @Override public QueueMetrics getMetrics() { return metrics; @@ -1378,8 +1426,13 @@ public class LeafQueue implements CSQueu ++activeApplications; } - public synchronized void finishApplication() { - --activeApplications; + public synchronized void finishApplication(boolean wasActive) { + if (wasActive) { + --activeApplications; + } + else { + --pendingApplications; + } } public synchronized void assignContainer(Resource resource) { Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Fri Feb 10 01:49:08 2012 @@ -97,7 +97,8 @@ public class ParentQueue implements CSQu RecordFactoryProvider.getRecordFactory(null); public ParentQueue(CapacitySchedulerContext cs, - String queueName, Comparator comparator, CSQueue parent, CSQueue old) { + String queueName, Comparator comparator, + CSQueue parent, CSQueue old) { minimumAllocation = cs.getMinimumResourceCapability(); this.parent = parent; @@ -137,7 +138,8 @@ public class ParentQueue implements CSQu this.queueInfo.setQueueName(queueName); this.queueInfo.setChildQueues(new ArrayList()); - setupQueueConfigs(capacity, absoluteCapacity, + setupQueueConfigs(cs.getClusterResources(), + capacity, absoluteCapacity, maximumCapacity, absoluteMaxCapacity, state, acls); this.queueComparator = comparator; @@ -149,9 +151,10 @@ public class ParentQueue implements CSQu } private synchronized void setupQueueConfigs( - float capacity, float absoluteCapacity, - float maximumCapacity, float absoluteMaxCapacity, - QueueState state, Map acls + Resource clusterResource, + float capacity, float absoluteCapacity, + float maximumCapacity, float absoluteMaxCapacity, + QueueState state, Map acls ) { // Sanity check CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); @@ -174,6 +177,10 @@ public class ParentQueue implements CSQu aclsString.append(e.getKey() + ":" + e.getValue().getAclString()); } + // Update metrics + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); + LOG.info(queueName + ", capacity=" + capacity + ", asboluteCapacity=" + absoluteCapacity + @@ -361,6 +368,12 @@ public class ParentQueue implements CSQu ParentQueue parentQueue = (ParentQueue)queue; + // Set new configs + setupQueueConfigs(clusterResource, + parentQueue.capacity, parentQueue.absoluteCapacity, + parentQueue.maximumCapacity, parentQueue.absoluteMaxCapacity, + parentQueue.state, parentQueue.acls); + // Re-configure existing child queues and add new ones // The CS has already checked to ensure all existing child queues are present! Map currentChildQueues = getQueues(childQueues); @@ -382,14 +395,6 @@ public class ParentQueue implements CSQu // Re-sort all queues childQueues.clear(); childQueues.addAll(currentChildQueues.values()); - - // Set new configs - setupQueueConfigs(parentQueue.capacity, parentQueue.absoluteCapacity, - parentQueue.maximumCapacity, parentQueue.absoluteMaxCapacity, - parentQueue.state, parentQueue.acls); - - // Update - updateResource(clusterResource); } Map getQueues(Set queues) { @@ -485,11 +490,11 @@ public class ParentQueue implements CSQu " #applications: " + getNumApplications()); } - synchronized void setUsedCapacity(float usedCapacity) { + public synchronized void setUsedCapacity(float usedCapacity) { this.usedCapacity = usedCapacity; } - synchronized void setUtilization(float utilization) { + public synchronized void setUtilization(float utilization) { this.utilization = utilization; } @@ -674,14 +679,16 @@ public class ParentQueue implements CSQu synchronized void allocateResource(Resource clusterResource, Resource resource) { Resources.addTo(usedResources, resource); - updateResource(clusterResource); + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); ++numContainers; } synchronized void releaseResource(Resource clusterResource, Resource resource) { Resources.subtractFrom(usedResources, resource); - updateResource(clusterResource); + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); --numContainers; } @@ -691,22 +698,12 @@ public class ParentQueue implements CSQu for (CSQueue childQueue : childQueues) { childQueue.updateClusterResource(clusterResource); } + + // Update metrics + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); } - private synchronized void updateResource(Resource clusterResource) { - float queueLimit = clusterResource.getMemory() * absoluteCapacity; - float parentAbsoluteCapacity = - (rootQueue) ? 1.0f : parent.getAbsoluteCapacity(); - setUtilization(usedResources.getMemory() / queueLimit); - setUsedCapacity(usedResources.getMemory() - / (clusterResource.getMemory() * parentAbsoluteCapacity)); - - Resource resourceLimit = - Resources.createResource((int)queueLimit); - metrics.setAvailableResourcesToQueue( - Resources.subtractFrom(resourceLimit, usedResources)); - } - @Override public QueueMetrics getMetrics() { return metrics; Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Fri Feb 10 01:49:08 2012 @@ -230,7 +230,7 @@ public class FifoScheduler implements Re } // Sanity check - SchedulerUtils.normalizeRequests(ask, MINIMUM_MEMORY); + SchedulerUtils.normalizeRequests(ask, minimumAllocation.getMemory()); // Release containers for (ContainerId releasedContainer : release) { @@ -298,7 +298,9 @@ public class FifoScheduler implements Re new SchedulerApp(appAttemptId, user, DEFAULT_QUEUE, activeUsersManager, this.rmContext, null); applications.put(appAttemptId, schedulerApp); - metrics.submitApp(user); + if (appAttemptId.getAttemptId() == 1) { + metrics.submitApp(user); + } LOG.info("Application Submission: " + appAttemptId.getApplicationId() + " from " + user + ", currently active: " + applications.size()); rmContext.getDispatcher().getEventHandler().handle( @@ -590,7 +592,7 @@ public class FifoScheduler implements Re minimumAllocation)) { LOG.debug("Node heartbeat " + rmNode.getNodeID() + " available resource = " + node.getAvailableResource()); - + assignContainers(node); LOG.debug("Node after allocation " + rmNode.getNodeID() + " resource = " Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java Fri Feb 10 01:49:08 2012 @@ -24,6 +24,8 @@ public class AppPage extends RmView { @Override protected void preHead(Page.HTML<_> html) { commonPreHead(html); + // App page is per-app information. Helps to refresh automatically. + html.meta_http("refresh", "10"); } @Override protected Class content() { Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java Fri Feb 10 01:49:08 2012 @@ -19,11 +19,14 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; @@ -49,16 +52,24 @@ class AppsBlock extends HtmlBlock { th(".user", "User"). th(".name", "Name"). th(".queue", "Queue"). + th(".starttime", "StartTime"). + th(".finishtime", "FinishTime"). th(".state", "State"). th(".finalstatus", "FinalStatus"). th(".progress", "Progress"). - th(".ui", "Tracking UI"). - th(".note", "Note")._()._(). + th(".ui", "Tracking UI")._()._(). tbody(); int i = 0; + String reqState = $(APP_STATE); + reqState = (reqState == null ? "" : reqState); for (RMApp app : list.apps.values()) { + if (!reqState.isEmpty() && app.getState() != RMAppState.valueOf(reqState)) { + continue; + } AppInfo appInfo = new AppInfo(app, true); String percent = String.format("%.1f", appInfo.getProgress()); + String startTime = Times.format(appInfo.getStartTime()); + String finishTime = Times.format(appInfo.getFinishTime()); tbody. tr(). td(). @@ -67,6 +78,10 @@ class AppsBlock extends HtmlBlock { td(appInfo.getUser()). td(appInfo.getName()). td(appInfo.getQueue()). + td(). + br().$title(startTime)._()._(startTime)._(). + td(). + br().$title(finishTime)._()._(finishTime)._(). td(appInfo.getState()). td(appInfo.getFinalStatus()). td(). @@ -77,8 +92,7 @@ class AppsBlock extends HtmlBlock { $style(join("width:", percent, '%'))._()._()._(). td(). a(!appInfo.isTrackingUrlReady()? - "#" : appInfo.getTrackingUrlPretty(), appInfo.getTrackingUI())._(). - td(appInfo.getNote())._(); + "#" : appInfo.getTrackingUrlPretty(), appInfo.getTrackingUI())._()._(); if (list.rendering != Render.HTML && ++i >= 20) break; } tbody._()._(); @@ -86,7 +100,7 @@ class AppsBlock extends HtmlBlock { if (list.rendering == Render.JS_ARRAY) { echo("\n"); } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java Fri Feb 10 01:49:08 2012 @@ -31,7 +31,9 @@ import java.util.concurrent.ConcurrentMa import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.Controller.RequestContext; import org.apache.hadoop.yarn.webapp.ToJSON; import org.apache.hadoop.yarn.webapp.view.JQueryUI.Render; @@ -51,11 +53,17 @@ class AppsList implements ToJSON { apps = rmContext.getRMApps(); } - void toDataTableArrays(PrintWriter out) { + void toDataTableArrays(String requiredAppState, PrintWriter out) { out.append('['); boolean first = true; for (RMApp app : apps.values()) { - AppInfo appInfo = new AppInfo(app, false); + if (requiredAppState != null && !requiredAppState.isEmpty() + && app.getState() != RMAppState.valueOf(requiredAppState)) { + continue; + } + AppInfo appInfo = new AppInfo(app, true); + String startTime = Times.format(appInfo.getStartTime()); + String finishTime = Times.format(appInfo.getFinishTime()); if (first) { first = false; } else { @@ -67,15 +75,15 @@ class AppsList implements ToJSON { appInfo.getAppId()).append(_SEP). append(escapeHtml(appInfo.getUser())).append(_SEP). append(escapeJavaScript(escapeHtml(appInfo.getName()))).append(_SEP). - append(escapeHtml(appInfo.getQueue())).append(_SEP). + append(escapeHtml(appInfo.getQueue())).append(_SEP); + appendSortable(out, startTime).append(startTime).append(_SEP); + appendSortable(out, finishTime).append(finishTime).append(_SEP). append(appInfo.getState()).append(_SEP). append(appInfo.getFinalStatus()).append(_SEP); appendProgressBar(out, appInfo.getProgress()).append(_SEP); appendLink(out, appInfo.getTrackingUI(), rc.prefix(), !appInfo.isTrackingUrlReady() ? "#" : appInfo.getTrackingUrlPretty()). - append(_SEP).append(escapeJavaScript(escapeHtml( - appInfo.getNote()))). append("\"]"); } out.append(']'); @@ -84,7 +92,7 @@ class AppsList implements ToJSON { @Override public void toJSON(PrintWriter out) { out.print("{\"aaData\":"); - toDataTableArrays(out); + toDataTableArrays(null, out); out.print("}\n"); } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java Fri Feb 10 01:49:08 2012 @@ -55,15 +55,19 @@ public class MetricsOverviewTable extend //CSS in the correct spot html.style(".metrics {margin-bottom:5px}"); - ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(this.rm, this.rmContext); - + ClusterMetricsInfo clusterMetrics = + new ClusterMetricsInfo(this.rm, this.rmContext); DIV div = html.div().$class("metrics"); - div.table("#metricsoverview"). + div.h3("Cluster Metrics"). + table("#metricsoverview"). thead().$class("ui-widget-header"). tr(). th().$class("ui-state-default")._("Apps Submitted")._(). + th().$class("ui-state-default")._("Apps Pending")._(). + th().$class("ui-state-default")._("Apps Running")._(). + th().$class("ui-state-default")._("Apps Completed")._(). th().$class("ui-state-default")._("Containers Running")._(). th().$class("ui-state-default")._("Memory Used")._(). th().$class("ui-state-default")._("Memory Total")._(). @@ -78,6 +82,14 @@ public class MetricsOverviewTable extend tbody().$class("ui-widget-content"). tr(). td(String.valueOf(clusterMetrics.getAppsSubmitted())). + td(String.valueOf(clusterMetrics.getAppsPending())). + td(String.valueOf(clusterMetrics.getAppsRunning())). + td( + String.valueOf( + clusterMetrics.getAppsCompleted() + + clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled() + ) + ). td(String.valueOf(clusterMetrics.getContainersAllocated())). td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)). td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)). @@ -89,26 +101,38 @@ public class MetricsOverviewTable extend td().a(url("nodes/rebooted"),String.valueOf(clusterMetrics.getRebootedNodes()))._(). _(). _()._(); - + String user = request().getRemoteUser(); if (user != null) { UserMetricsInfo userMetrics = new UserMetricsInfo(this.rm, this.rmContext, user); if (userMetrics.metricsAvailable()) { - div.table("#usermetricsoverview"). + div.h3("User Metrics for " + user). + table("#usermetricsoverview"). thead().$class("ui-widget-header"). tr(). - th().$class("ui-state-default")._("Apps Submitted ("+user+")")._(). - th().$class("ui-state-default")._("Containers Running ("+user+")")._(). - th().$class("ui-state-default")._("Containers Pending ("+user+")")._(). - th().$class("ui-state-default")._("Containers Reserved ("+user+")")._(). - th().$class("ui-state-default")._("Memory Used ("+user+")")._(). - th().$class("ui-state-default")._("Memory Pending ("+user+")")._(). - th().$class("ui-state-default")._("Memory Reserved ("+user+")")._(). + th().$class("ui-state-default")._("Apps Submitted")._(). + th().$class("ui-state-default")._("Apps Pending")._(). + th().$class("ui-state-default")._("Apps Running")._(). + th().$class("ui-state-default")._("Apps Completed")._(). + th().$class("ui-state-default")._("Containers Running")._(). + th().$class("ui-state-default")._("Containers Pending")._(). + th().$class("ui-state-default")._("Containers Reserved")._(). + th().$class("ui-state-default")._("Memory Used")._(). + th().$class("ui-state-default")._("Memory Pending")._(). + th().$class("ui-state-default")._("Memory Reserved")._(). _(). _(). tbody().$class("ui-widget-content"). tr(). td(String.valueOf(userMetrics.getAppsSubmitted())). + td(String.valueOf(userMetrics.getAppsPending())). + td(String.valueOf(userMetrics.getAppsRunning())). + td( + String.valueOf( + (userMetrics.getAppsCompleted() + + userMetrics.getAppsFailed() + userMetrics.getAppsKilled()) + ) + ). td(String.valueOf(userMetrics.getRunningContainers())). td(String.valueOf(userMetrics.getPendingContainers())). td(String.valueOf(userMetrics.getReservedContainers())). @@ -117,6 +141,7 @@ public class MetricsOverviewTable extend td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)). _(). _()._(); + } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java Fri Feb 10 01:49:08 2012 @@ -18,18 +18,32 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.LI; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; public class NavBlock extends HtmlBlock { @Override public void render(Block html) { - html. + UL> mainList = html. div("#nav"). h3("Cluster"). ul(). li().a(url("cluster"), "About")._(). - li().a(url("nodes"), "Nodes")._(). - li().a(url("apps"), "Applications")._(). + li().a(url("nodes"), "Nodes")._(); + UL>>> subAppsList = mainList. + li().a(url("apps"), "Applications"). + ul(); + subAppsList.li()._(); + for (RMAppState state : RMAppState.values()) { + subAppsList. + li().a(url("apps", state.toString()), state.toString())._(); + } + subAppsList._()._(); + mainList. li().a(url("scheduler"), "Scheduler")._()._(). h3("Tools"). ul(). Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java?rev=1242635&r1=1242634&r2=1242635&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java Fri Feb 10 01:49:08 2012 @@ -100,6 +100,12 @@ class NodesPage extends RmView { if(!stateFilter.equals(state)) { continue; } + } else { + // No filter. User is asking for all nodes. Make sure you skip the + // unhealthy nodes. + if (ni.getState() == RMNodeState.UNHEALTHY) { + continue; + } } NodeInfo info = new NodeInfo(ni, sched); int usedMemory = (int)info.getUsedMemory();