hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject hadoop git commit: YARN-2414. RM web UI: app page will crash if app is failed before any attempt has been created. Contributed by Wangda Tan (cherry picked from commit 81c9d17af84ed87b9ded7057cb726a3855ddd32d)
Date Fri, 28 Aug 2015 01:43:12 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6.1 f83d89894 -> f307b426f


YARN-2414. RM web UI: app page will crash if app is failed before any attempt has been created.
Contributed by Wangda Tan
(cherry picked from commit 81c9d17af84ed87b9ded7057cb726a3855ddd32d)

(cherry picked from commit 242fd0e39ad1c5d51719cd0f6c197166066e3288)
(cherry picked from commit a9d5acd898b34e1050a78f2d70ed62fdb82948a6)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f307b426
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f307b426
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f307b426

Branch: refs/heads/branch-2.6.1
Commit: f307b426f35f708e8ea1f92d4a772c99855192b6
Parents: f83d898
Author: Jason Lowe <jlowe@apache.org>
Authored: Mon Nov 17 21:13:22 2014 +0000
Committer: Vinod Kumar Vavilapalli <vinodkv@apache.org>
Committed: Thu Aug 27 18:36:50 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../server/security/ApplicationACLsManager.java |  7 ++
 .../security/QueueACLsManager.java              |  7 ++
 .../server/resourcemanager/webapp/AppBlock.java | 30 +++++--
 .../resourcemanager/webapp/TestAppPage.java     | 90 ++++++++++++++++++++
 5 files changed, 130 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f307b426/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5e8e4f9..81a4347 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -18,6 +18,9 @@ Release 2.6.1 - UNRELEASED
     YARN-2816. NM fail to start with NPE during container recovery (Zhihai Xu
     via jlowe)
 
+    YARN-2414. RM web UI: app page will crash if app is failed before any
+    attempt has been created (Wangda Tan via jlowe)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f307b426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
index e8e3cb5..4cacfca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.security.AdminACLsManager;
 
+import com.google.common.annotations.VisibleForTesting;
+
 @InterfaceAudience.Private
 public class ApplicationACLsManager {
 
@@ -48,6 +50,11 @@ public class ApplicationACLsManager {
   private final ConcurrentMap<ApplicationId, Map<ApplicationAccessType, AccessControlList>>
applicationACLS
     = new ConcurrentHashMap<ApplicationId, Map<ApplicationAccessType, AccessControlList>>();
 
+  @VisibleForTesting
+  public ApplicationACLsManager() {
+    this(new Configuration());
+  }
+  
   public ApplicationACLsManager(Configuration conf) {
     this.conf = conf;
     this.adminAclsManager = new AdminACLsManager(this.conf);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f307b426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
index eb5037a..fb8279d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
@@ -24,9 +24,16 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 
+import com.google.common.annotations.VisibleForTesting;
+
 public class QueueACLsManager {
   private ResourceScheduler scheduler;
   private boolean isACLsEnable;
+  
+  @VisibleForTesting
+  public QueueACLsManager() {
+    this(null, new Configuration());
+  }
 
   public QueueACLsManager(ResourceScheduler scheduler, Configuration conf) {
     this.scheduler = scheduler;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f307b426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
index c427ccf..a108e43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
 import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
@@ -113,8 +115,23 @@ public class AppBlock extends HtmlBlock {
     setTitle(join("Application ", aid));
 
     RMAppMetrics appMerics = rmApp.getRMAppMetrics();
-    RMAppAttemptMetrics attemptMetrics =
-        rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
+    
+    // Get attempt metrics and fields, it is possible currentAttempt of RMApp is
+    // null. In that case, we will assume resource preempted and number of Non
+    // AM container preempted on that attempt is 0
+    RMAppAttemptMetrics attemptMetrics;
+    if (null == rmApp.getCurrentAppAttempt()) {
+      attemptMetrics = null;
+    } else {
+      attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
+    }
+    Resource attemptResourcePreempted =
+        attemptMetrics == null ? Resources.none() : attemptMetrics
+            .getResourcePreempted();
+    int attemptNumNonAMContainerPreempted =
+        attemptMetrics == null ? 0 : attemptMetrics
+            .getNumNonAMContainersPreempted();
+    
     info("Application Overview")
         ._("User:", app.getUser())
         ._("Name:", app.getName())
@@ -143,13 +160,12 @@ public class AppBlock extends HtmlBlock {
         ._("Total Number of AM Containers Preempted:",
           String.valueOf(appMerics.getNumAMContainersPreempted()))
         ._("Resource Preempted from Current Attempt:",
-          attemptMetrics.getResourcePreempted())
+          attemptResourcePreempted)
         ._("Number of Non-AM Containers Preempted from Current Attempt:",
-          String.valueOf(attemptMetrics
-            .getNumNonAMContainersPreempted()))
+          attemptNumNonAMContainerPreempted)
         ._("Aggregate Resource Allocation:",
-            String.format("%d MB-seconds, %d vcore-seconds", 
-                appMerics.getMemorySeconds(), appMerics.getVcoreSeconds()));
+          String.format("%d MB-seconds, %d vcore-seconds", 
+              appMerics.getMemorySeconds(), appMerics.getVcoreSeconds()));
     pdiv._();
 
     Collection<RMAppAttempt> attempts = rmApp.getAppAttempts().values();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f307b426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
new file mode 100644
index 0000000..9732c19
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
+import org.apache.hadoop.yarn.webapp.test.WebAppTests;
+import org.junit.Test;
+
+import com.google.inject.Binder;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+public class TestAppPage {
+  @Test
+  public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
+    final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0);
+    Injector injector;
+    
+    // init app
+    RMApp app = mock(RMApp.class);
+    when(app.getTrackingUrl()).thenReturn("http://host:123");
+    when(app.getState()).thenReturn(RMAppState.FAILED);
+    when(app.getApplicationId()).thenReturn(APP_ID);
+    when(app.getApplicationType()).thenReturn("Type");
+    when(app.getUser()).thenReturn("user");
+    when(app.getName()).thenReturn("Name");
+    when(app.getQueue()).thenReturn("queue");
+    when(app.getDiagnostics()).thenReturn(new StringBuilder());
+    when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
+    when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
+    when(app.getStartTime()).thenReturn(0L);
+    when(app.getFinishTime()).thenReturn(0L);
+    when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
+    
+    RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
+    when(app.getRMAppMetrics()).thenReturn(appMetrics);
+    
+    // initialize RM Context, and create RMApp, without creating RMAppAttempt
+    final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
+    rmContext.getRMApps().put(APP_ID, app);
+    
+    injector =
+        WebAppTests.createMockInjector(RMContext.class, rmContext,
+            new Module() {
+              @Override
+              public void configure(Binder binder) {
+                try {
+                  binder.bind(ResourceManager.class).toInstance(
+                      TestRMWebApp.mockRm(rmContext));
+                } catch (IOException e) {
+                  throw new IllegalStateException(e);
+                }
+              }
+            });
+    
+    AppBlock instance = injector.getInstance(AppBlock.class);
+    instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString());
+    instance.render();
+  }
+}


Mime
View raw message