hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1440579 - in /hadoop/common/branches/HDFS-347/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-common/ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/c...
Date Wed, 30 Jan 2013 18:32:03 GMT
Author: atm
Date: Wed Jan 30 18:32:00 2013
New Revision: 1440579

URL: http://svn.apache.org/viewvc?rev=1440579&view=rev
Log:
Merge trunk into HDFS-347 branch.

Modified:
    hadoop/common/branches/HDFS-347/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
    hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
    hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
    hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
    hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
    hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm

Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/CHANGES.txt?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/CHANGES.txt Wed Jan 30 18:32:00 2013
@@ -274,10 +274,16 @@ Release 0.23.7 - UNRELEASED
 
   IMPROVEMENTS
 
+    YARN-133 Update web services docs for RM clusterMetrics (Ravi Prakash via
+    kihwal)
+
   OPTIMIZATIONS
 
   BUG FIXES
 
+    YARN-343. Capacity Scheduler maximum-capacity value -1 is invalid (Xuan 
+    Gong via tgraves)
+
 Release 0.23.6 - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -335,6 +341,9 @@ Release 0.23.6 - UNRELEASED
 
     YARN-334. Maven RAT plugin is not checking all source files (tgraves)
 
+    YARN-354. WebAppProxyServer exits immediately after startup (Liang Xie via
+    jlowe)
+
 Release 0.23.5 - 2012-11-28
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
(original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
Wed Jan 30 18:32:00 2013
@@ -93,6 +93,7 @@
         <executions>
           <execution>
             <id>version-info</id>
+            <phase>generate-resources</phase>
             <goals>
               <goal>version-info</goal>
             </goals>

Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
(original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
Wed Jan 30 18:32:00 2013
@@ -100,6 +100,9 @@ public class CapacitySchedulerConfigurat
   public static final float MAXIMUM_CAPACITY_VALUE = 100;
   
   @Private
+  public static final float DEFAULT_MAXIMUM_CAPACITY_VALUE = -1.0f;
+  
+  @Private
   public static final int DEFAULT_USER_LIMIT = 100;
   
   @Private
@@ -206,6 +209,8 @@ public class CapacitySchedulerConfigurat
   public float getMaximumCapacity(String queue) {
     float maxCapacity = getFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY,
         MAXIMUM_CAPACITY_VALUE);
+    maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE) ? 
+        MAXIMUM_CAPACITY_VALUE : maxCapacity;
     return maxCapacity;
   }
   

Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
(original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
Wed Jan 30 18:32:00 2013
@@ -244,6 +244,18 @@ public class TestCapacityScheduler {
   }
   
   @Test
+  public void testMaximumCapacitySetup() {
+    float delta = 0.0000001f;
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getMaximumCapacity(A),delta);
+    conf.setMaximumCapacity(A, 50.0f);
+    assertEquals(50.0f, conf.getMaximumCapacity(A),delta);
+    conf.setMaximumCapacity(A, -1);
+    assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getMaximumCapacity(A),delta);
+  }
+  
+  
+  @Test
   public void testRefreshQueues() throws Exception {
     CapacityScheduler cs = new CapacityScheduler();
     CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();

Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
(original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
Wed Jan 30 18:32:00 2013
@@ -114,4 +114,13 @@ public class WebAppProxy extends Abstrac
     }
     super.stop();
   }
+
+  public void join() {
+    if(proxyServer != null) {
+      try {
+        proxyServer.join();
+      } catch (InterruptedException e) {
+      }
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
(original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java
Wed Jan 30 18:32:00 2013
@@ -73,6 +73,14 @@ public class WebAppProxyServer extends C
         YarnConfiguration.PROXY_PRINCIPAL);
   }
 
+  /**
+   * Wait for service to finish.
+   * (Normally, it runs forever.)
+   */
+  private void join() {
+    proxy.join();
+  }
+
   public static void main(String[] args) {
     Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
     StringUtils.startupShutdownMessage(WebAppProxyServer.class, args, LOG);
@@ -84,6 +92,7 @@ public class WebAppProxyServer extends C
       YarnConfiguration conf = new YarnConfiguration();
       proxy.init(conf);
       proxy.start();
+      proxy.join();
     } catch (Throwable t) {
       LOG.fatal("Error starting Proxy server", t);
       System.exit(-1);

Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1440579&r1=1440578&r2=1440579&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
(original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Wed Jan 30 18:32:00 2013
@@ -177,6 +177,16 @@ ResourceManager REST API's.
 *---------------+--------------+-------------------------------+
 | appsSubmitted | int | The number of applications submitted |
 *---------------+--------------+-------------------------------+
+| appsCompleted | int | The number of applications completed |
+*---------------+--------------+-------------------------------+
+| appsPending | int | The number of applications pending |
+*---------------+--------------+-------------------------------+
+| appsRunning | int | The number of applications running |
+*---------------+--------------+-------------------------------+
+| appsFailed | int | The number of applications failed |
+*---------------+--------------+-------------------------------+
+| appsKilled | int | The number of applications killed |
+*---------------+--------------+-------------------------------+
 | reservedMB    | long         | The amount of memory reserved in MB |
 *---------------+--------------+-------------------------------+
 | availableMB   | long         | The amount of memory available in MB |
@@ -187,6 +197,10 @@ ResourceManager REST API's.
 *---------------+--------------+-------------------------------+
 | containersAllocated | int | The number of containers allocated |
 *---------------+--------------+-------------------------------+
+| containersReserved | int | The number of containers reserved |
+*---------------+--------------+-------------------------------+
+| containersPending | int | The number of containers pending |
+*---------------+--------------+-------------------------------+
 | totalNodes | int | The total number of nodes |
 *---------------+--------------+-------------------------------+
 | activeNodes | int | The number of active nodes |
@@ -223,20 +237,26 @@ ResourceManager REST API's.
 
 +---+
   {
-    "clusterMetrics":
-    {
-      "appsSubmitted":4,
+    "clusterMetrics":{
+      "appsSubmitted":0,
+      "appsCompleted":0,
+      "appsPending":0,
+      "appsRunning":0,
+      "appsFailed":0,
+      "appsKilled":0,
       "reservedMB":0,
-      "availableMB":8192,
+      "availableMB":17408,
       "allocatedMB":0,
-      "totalMB":8192,
       "containersAllocated":0,
+      "containersReserved":0,
+      "containersPending":0,
+      "totalMB":17408,
       "totalNodes":1,
-      "activeNodes":1,
       "lostNodes":0,
       "unhealthyNodes":0,
       "decommissionedNodes":0,
-      "rebootedNodes":0
+      "rebootedNodes":0,
+      "activeNodes":1
     }
   }
 +---+
@@ -264,18 +284,25 @@ ResourceManager REST API's.
 +---+
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <clusterMetrics>
-  <appsSubmitted>4</appsSubmitted>
+  <appsSubmitted>0</appsSubmitted>
+  <appsCompleted>0</appsCompleted>
+  <appsPending>0</appsPending>
+  <appsRunning>0</appsRunning>
+  <appsFailed>0</appsFailed>
+  <appsKilled>0</appsKilled>
   <reservedMB>0</reservedMB>
-  <availableMB>8192</availableMB>
+  <availableMB>17408</availableMB>
   <allocatedMB>0</allocatedMB>
   <containersAllocated>0</containersAllocated>
-  <totalMB>8192</totalMB>
+  <containersReserved>0</containersReserved>
+  <containersPending>0</containersPending>
+  <totalMB>17408</totalMB>
   <totalNodes>1</totalNodes>
-  <activeNodes>1</activeNodes>
   <lostNodes>0</lostNodes>
   <unhealthyNodes>0</unhealthyNodes>
   <decommissionedNodes>0</decommissionedNodes>
   <rebootedNodes>0</rebootedNodes>
+  <activeNodes>1</activeNodes>
 </clusterMetrics>
 +---+
 



Mime
View raw message