hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1407217 [2/2] - in /hadoop/common/branches/branch-trunk-win/hadoop-yarn-project: ./ hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/ hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-app...
Date Thu, 08 Nov 2012 19:09:59 GMT
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java
Thu Nov  8 19:09:46 2012
@@ -69,7 +69,7 @@ public class RmView extends TwoColumnLay
         append("null,{sType:'title-numeric', bSearchable:false}, null]");
 
     // Sort by id upon page load
-    init.append(", aaSorting: [[0, 'asc']]");
+    init.append(", aaSorting: [[0, 'desc']]");
 
     String rows = $("rowlimit");
     int rowLimit = rows.isEmpty() ? MAX_DISPLAY_ROWS : Integer.parseInt(rows);

Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
Thu Nov  8 19:09:46 2012
@@ -17,6 +17,7 @@
 */
 package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
 
+import static org.apache.hadoop.yarn.util.StringHelper.pjoin;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
@@ -85,6 +87,8 @@ public class TestRMAppAttemptTransitions
       LogFactory.getLog(TestRMAppAttemptTransitions.class);
   
   private static final String EMPTY_DIAGNOSTICS = "";
+  private static final String RM_WEBAPP_ADDR =
+      YarnConfiguration.getRMWebAppHostAndPort(new Configuration());
   
   private RMContext rmContext;
   private YarnScheduler scheduler;
@@ -203,7 +207,7 @@ public class TestRMAppAttemptTransitions
     application = mock(RMApp.class);
     applicationAttempt = 
         new RMAppAttemptImpl(applicationAttemptId, null, rmContext, scheduler, 
-            masterService, submissionContext, null);
+            masterService, submissionContext, new Configuration());
     when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
     when(application.getApplicationId()).thenReturn(applicationId);
     
@@ -216,6 +220,11 @@ public class TestRMAppAttemptTransitions
   }
   
 
+  private String getProxyUrl(RMAppAttempt appAttempt) {
+    return pjoin(RM_WEBAPP_ADDR, "proxy",
+        appAttempt.getAppAttemptId().getApplicationId(), "");
+  }
+
   /**
    * {@link RMAppAttemptState#NEW}
    */
@@ -373,8 +382,8 @@ public class TestRMAppAttemptTransitions
     assertEquals(host, applicationAttempt.getHost());
     assertEquals(rpcPort, applicationAttempt.getRpcPort());
     assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
-    assertEquals("null/proxy/"+applicationAttempt.getAppAttemptId().
-        getApplicationId()+"/", applicationAttempt.getTrackingUrl());
+    assertEquals(getProxyUrl(applicationAttempt),
+        applicationAttempt.getTrackingUrl());
     
     // TODO - need to add more checks relevant to this state
   }
@@ -390,8 +399,8 @@ public class TestRMAppAttemptTransitions
         applicationAttempt.getAppAttemptState());
     assertEquals(diagnostics, applicationAttempt.getDiagnostics());
     assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
-    assertEquals("null/proxy/"+applicationAttempt.getAppAttemptId().
-        getApplicationId()+"/", applicationAttempt.getTrackingUrl());
+    assertEquals(getProxyUrl(applicationAttempt),
+        applicationAttempt.getTrackingUrl());
     assertEquals(container, applicationAttempt.getMasterContainer());
     assertEquals(finalStatus, applicationAttempt.getFinalApplicationStatus());
   }
@@ -408,8 +417,8 @@ public class TestRMAppAttemptTransitions
         applicationAttempt.getAppAttemptState());
     assertEquals(diagnostics, applicationAttempt.getDiagnostics());
     assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
-    assertEquals("null/proxy/"+applicationAttempt.getAppAttemptId().
-        getApplicationId()+"/", applicationAttempt.getTrackingUrl());
+    assertEquals(getProxyUrl(applicationAttempt),
+        applicationAttempt.getTrackingUrl());
     assertEquals(finishedContainerCount, applicationAttempt
         .getJustFinishedContainers().size());
     assertEquals(container, applicationAttempt.getMasterContainer());
@@ -596,7 +605,30 @@ public class TestRMAppAttemptTransitions
             diagnostics));
     testAppAttemptFailedState(amContainer, diagnostics);
   }
-  
+
+  @Test
+  public void testRunningToFailed() {
+    Container amContainer = allocateApplicationAttempt();
+    launchApplicationAttempt(amContainer);
+    runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+    String containerDiagMsg = "some error";
+    int exitCode = 123;
+    ContainerStatus cs = BuilderUtils.newContainerStatus(amContainer.getId(),
+        ContainerState.COMPLETE, containerDiagMsg, exitCode);
+    ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
+    applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
+        appAttemptId, cs));
+    assertEquals(RMAppAttemptState.FAILED,
+        applicationAttempt.getAppAttemptState());
+    assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
+    assertEquals(amContainer, applicationAttempt.getMasterContainer());
+    assertEquals(0, applicationAttempt.getRanNodes().size());
+    String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app",
+        applicationAttempt.getAppAttemptId().getApplicationId());
+    assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
+    assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
+  }
+
   @Test 
   public void testUnregisterToKilledFinishing() {
     Container amContainer = allocateApplicationAttempt();

Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
Thu Nov  8 19:09:46 2012
@@ -66,8 +66,8 @@ public class TestCapacityScheduler {
   private static float B_CAPACITY = 89.5f;
   private static float A1_CAPACITY = 30;
   private static float A2_CAPACITY = 70;
-  private static float B1_CAPACITY = 50;
-  private static float B2_CAPACITY = 30;
+  private static float B1_CAPACITY = 79.2f;
+  private static float B2_CAPACITY = 0.8f;
   private static float B3_CAPACITY = 20;
 
   private ResourceManager resourceManager = null;
@@ -378,4 +378,43 @@ public class TestCapacityScheduler {
 
     Assert.assertEquals(4 * GB, cs.getClusterResources().getMemory());
   }
+
+  @Test
+  public void testRefreshQueuesWithNewQueue() throws Exception {
+    CapacityScheduler cs = new CapacityScheduler();
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    cs.setConf(new YarnConfiguration());
+    cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null,
+      null, new RMContainerTokenSecretManager(conf),
+      new ClientToAMTokenSecretManagerInRM()));
+    checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
+
+    // Add a new queue b4
+    String B4 = B + ".b4";
+    float B4_CAPACITY = 10;
+    
+    B3_CAPACITY -= B4_CAPACITY;
+    try {
+      conf.setCapacity(A, 80f);
+      conf.setCapacity(B, 20f);
+      conf.setQueues(B, new String[] {"b1", "b2", "b3", "b4"});
+      conf.setCapacity(B1, B1_CAPACITY);
+      conf.setCapacity(B2, B2_CAPACITY);
+      conf.setCapacity(B3, B3_CAPACITY);
+      conf.setCapacity(B4, B4_CAPACITY);
+      cs.reinitialize(conf,null);
+      checkQueueCapacities(cs, 80f, 20f);
+      
+      // Verify parent for B4
+      CSQueue rootQueue = cs.getRootQueue();
+      CSQueue queueB = findQueue(rootQueue, B);
+      CSQueue queueB4 = findQueue(queueB, B4);
+
+      assertEquals(queueB, queueB4.getParent());
+    } finally {
+      B3_CAPACITY += B4_CAPACITY;
+    }
+  }
+
 }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Thu Nov  8 19:09:46 2012
@@ -1259,7 +1259,7 @@ public class TestLeafQueue {
     assignment = a.assignContainers(clusterResource, node_2);
     verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2), 
         any(Priority.class), any(ResourceRequest.class), any(Container.class));
-    assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset
+    assertEquals(4, app_0.getSchedulingOpportunities(priority)); // should NOT reset
     assertEquals(2, app_0.getTotalRequiredResources(priority));
     assertEquals(NodeType.OFF_SWITCH, assignment.getType());
     
@@ -1408,11 +1408,11 @@ public class TestLeafQueue {
     assertEquals(0, app_0.getSchedulingOpportunities(priority_2));
     assertEquals(1, app_0.getTotalRequiredResources(priority_2));
 
-    // Another off-switch, shouldn allocate OFF_SWITCH P1
+    // Another off-switch, shouldn't allocate OFF_SWITCH P1
     a.assignContainers(clusterResource, node_2);
     verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2), 
         eq(priority_1), any(ResourceRequest.class), any(Container.class));
-    assertEquals(0, app_0.getSchedulingOpportunities(priority_1));
+    assertEquals(3, app_0.getSchedulingOpportunities(priority_1));
     assertEquals(1, app_0.getTotalRequiredResources(priority_1));
     verify(app_0, never()).allocate(any(NodeType.class), eq(node_2), 
         eq(priority_2), any(ResourceRequest.class), any(Container.class));
@@ -1438,7 +1438,7 @@ public class TestLeafQueue {
     assertEquals(0, app_0.getTotalRequiredResources(priority_1));
     verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_1), 
         eq(priority_2), any(ResourceRequest.class), any(Container.class));
-    assertEquals(0, app_0.getSchedulingOpportunities(priority_2));
+    assertEquals(1, app_0.getSchedulingOpportunities(priority_2));
     assertEquals(0, app_0.getTotalRequiredResources(priority_2));
 
   }

Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
Thu Nov  8 19:09:46 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doAnswer;
@@ -494,6 +495,72 @@ public class TestParentQueue {
     verifyQueueMetrics(c, 4*GB, clusterResource);
     reset(a); reset(b); reset(c);
   }
+  
+  @Test (expected=IllegalArgumentException.class)
+  public void testQueueCapacitySettingChildZero() throws Exception {
+    // Setup queue configs
+    setupMultiLevelQueues(csConf);
+    
+    // set child queues capacity to 0 when parents not 0
+    final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+    csConf.setCapacity(Q_B + "." + B1, 0);
+    csConf.setCapacity(Q_B + "." + B2, 0);
+    csConf.setCapacity(Q_B + "." + B3, 0);
+    
+    Map<String, CSQueue> queues = new HashMap<String, CSQueue>(); 
+    CapacityScheduler.parseQueue(csContext, csConf, null, 
+        CapacitySchedulerConfiguration.ROOT, queues, queues, 
+        CapacityScheduler.queueComparator, 
+        CapacityScheduler.applicationComparator,
+        TestUtils.spyHook);
+  }
+  
+  @Test (expected=IllegalArgumentException.class)
+  public void testQueueCapacitySettingParentZero() throws Exception {
+    // Setup queue configs
+    setupMultiLevelQueues(csConf);
+    
+    // set parent capacity to 0 when child not 0
+    final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+    csConf.setCapacity(Q_B, 0);
+    final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
+    csConf.setCapacity(Q_A, 60);
+
+    Map<String, CSQueue> queues = new HashMap<String, CSQueue>(); 
+    CapacityScheduler.parseQueue(csContext, csConf, null, 
+        CapacitySchedulerConfiguration.ROOT, queues, queues, 
+        CapacityScheduler.queueComparator, 
+        CapacityScheduler.applicationComparator,
+        TestUtils.spyHook);
+  }
+  
+  @Test
+  public void testQueueCapacityZero() throws Exception {
+    // Setup queue configs
+    setupMultiLevelQueues(csConf);
+    
+    // set parent and child capacity to 0
+    final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+    csConf.setCapacity(Q_B, 0);
+    csConf.setCapacity(Q_B + "." + B1, 0);
+    csConf.setCapacity(Q_B + "." + B2, 0);
+    csConf.setCapacity(Q_B + "." + B3, 0);
+    
+    final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
+    csConf.setCapacity(Q_A, 60);
+
+    Map<String, CSQueue> queues = new HashMap<String, CSQueue>(); 
+    try {
+      CapacityScheduler.parseQueue(csContext, csConf, null, 
+          CapacitySchedulerConfiguration.ROOT, queues, queues, 
+          CapacityScheduler.queueComparator, 
+          CapacityScheduler.applicationComparator,
+          TestUtils.spyHook);
+    } catch (IllegalArgumentException e) {
+      fail("Failed to create queues with 0 capacity: " + e);
+    }
+    assertTrue("Failed to create queues with 0 capacity", true);
+  }
 
   @Test
   public void testOffSwitchScheduling() throws Exception {

Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
(original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
Thu Nov  8 19:09:46 2012
@@ -144,11 +144,11 @@ public class TestRMWebServicesCapacitySc
     final String B2 = B + ".b2";
     final String B3 = B + ".b3";
     conf.setQueues(B, new String[] { "b1", "b2", "b3" });
-    conf.setCapacity(B1, 50);
+    conf.setCapacity(B1, 60);
     conf.setUserLimitFactor(B1, 100.0f);
-    conf.setCapacity(B2, 30);
+    conf.setCapacity(B2, 39.5f);
     conf.setUserLimitFactor(B2, 100.0f);
-    conf.setCapacity(B3, 20);
+    conf.setCapacity(B3, 0.5f);
     conf.setUserLimitFactor(B3, 100.0f);
     
     conf.setQueues(A1, new String[] {"a1a", "a1b"});



Mime
View raw message