Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 48FF71049A for ; Thu, 26 Sep 2013 04:18:40 +0000 (UTC) Received: (qmail 40999 invoked by uid 500); 26 Sep 2013 04:18:29 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 40782 invoked by uid 500); 26 Sep 2013 04:18:28 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 40502 invoked by uid 99); 26 Sep 2013 04:18:23 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 Sep 2013 04:18:23 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 Sep 2013 04:18:21 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 19993238896F; Thu, 26 Sep 2013 04:18:01 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1526371 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hado... Date: Thu, 26 Sep 2013 04:18:00 -0000 To: yarn-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130926041801.19993238896F@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: vinodkv Date: Thu Sep 26 04:18:00 2013 New Revision: 1526371 URL: http://svn.apache.org/r1526371 Log: YARN-1157. Fixed ResourceManager UI to behave correctly when apps like distributed-shell do not set tracking urls. Contributed by Xuan Gong. Removed: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1526371&r1=1526370&r2=1526371&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Thu Sep 26 04:18:00 2013 @@ -83,6 +83,9 @@ Release 2.1.2 - UNRELEASED YARN-49. Improve distributed shell application to work on a secure cluster. (Vinod Kumar Vavilapalli via hitesh) + YARN-1157. Fixed ResourceManager UI to behave correctly when apps like + distributed-shell do not set tracking urls. (Xuan Gong via vinodkv) + Release 2.1.1-beta - 2013-09-23 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java?rev=1526371&r1=1526370&r2=1526371&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java Thu Sep 26 04:18:00 2013 @@ -100,11 +100,22 @@ public abstract class FinishApplicationM public abstract String getTrackingUrl(); /** - * Set the tracking URLfor the ApplicationMaster - * This url if contains scheme then that will be used by resource manager - * web application proxy otherwise it will default to http. - * @param url tracking URLfor the - * ApplicationMaster + * Set the final tracking URLfor the ApplicationMaster. + * This is the web-URL to which ResourceManager or web-application proxy will + * redirect client/users once the application is finished and the + * ApplicationMaster is gone. + *

+ * If the passed url has a scheme then that will be used by the + * ResourceManager and web-application proxy, otherwise the scheme will + * default to http. + *

+ *

+ * Empty, null, "N/A" strings are all valid besides a real URL. In case an url + * isn't explicitly passed, it defaults to "N/A" on the ResourceManager. + *

+ * + * @param url + * tracking URLfor the ApplicationMaster */ @Public @Stable Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java?rev=1526371&r1=1526370&r2=1526371&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java Thu Sep 26 04:18:00 2013 @@ -112,11 +112,22 @@ public abstract class RegisterApplicatio public abstract String getTrackingUrl(); /** - * Set the tracking URL for the ApplicationMaster. - * This url if contains scheme then that will be used by resource manager - * web application proxy otherwise it will default to http. - * @param trackingUrl tracking URL for the - * ApplicationMaster + * Set the tracking URLfor the ApplicationMaster while + * it is running. This is the web-URL to which ResourceManager or + * web-application proxy will redirect client/users while the application and + * the ApplicationMaster are still running. + *

+ * If the passed url has a scheme then that will be used by the + * ResourceManager and web-application proxy, otherwise the scheme will + * default to http. + *

+ *

+ * Empty, null, "N/A" strings are all valid besides a real URL. In case an url + * isn't explicitly passed, it defaults to "N/A" on the ResourceManager. + *

+ * + * @param trackingUrl + * tracking URLfor the ApplicationMaster */ @Public @Stable Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1526371&r1=1526370&r2=1526371&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Thu Sep 26 04:18:00 2013 @@ -994,7 +994,7 @@ public class RMAppAttemptImpl implements } } - static final class AMRegisteredTransition extends BaseTransition { + private static final class AMRegisteredTransition extends BaseTransition { @Override public void transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { @@ -1003,7 +1003,8 @@ public class RMAppAttemptImpl implements = (RMAppAttemptRegistrationEvent) event; appAttempt.host = registrationEvent.getHost(); appAttempt.rpcPort = registrationEvent.getRpcport(); - appAttempt.origTrackingUrl = registrationEvent.getTrackingurl(); + appAttempt.origTrackingUrl = + sanitizeTrackingUrl(registrationEvent.getTrackingurl()); appAttempt.proxiedTrackingUrl = appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl); @@ -1138,7 +1139,8 @@ public class RMAppAttemptImpl implements RMAppAttemptUnregistrationEvent unregisterEvent = (RMAppAttemptUnregistrationEvent) event; appAttempt.diagnostics.append(unregisterEvent.getDiagnostics()); - appAttempt.origTrackingUrl = unregisterEvent.getTrackingUrl(); + appAttempt.origTrackingUrl = + sanitizeTrackingUrl(unregisterEvent.getTrackingUrl()); appAttempt.proxiedTrackingUrl = appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl); appAttempt.finalStatus = unregisterEvent.getFinalApplicationStatus(); @@ -1292,4 +1294,8 @@ public class RMAppAttemptImpl implements appAttempt.rmContext.getAMRMTokenSecretManager() .applicationMasterFinished(appAttempt.getAppAttemptId()); } + + private static String sanitizeTrackingUrl(String url) { + return (url == null || url.trim().isEmpty()) ? "N/A" : url; + } } Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1526371&r1=1526370&r2=1526371&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Thu Sep 26 04:18:00 2013 @@ -30,14 +30,18 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.net.URI; +import java.net.URISyntaxException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; @@ -85,8 +89,10 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -261,8 +267,22 @@ public class TestRMAppAttemptTransitions private String getProxyUrl(RMAppAttempt appAttempt) { - return pjoin(RM_WEBAPP_ADDR, "proxy", - appAttempt.getAppAttemptId().getApplicationId(), ""); + String url = null; + try { + URI trackingUri = + StringUtils.isEmpty(appAttempt.getOriginalTrackingUrl()) ? null : + ProxyUriUtils + .getUriFromAMUrl(appAttempt.getOriginalTrackingUrl()); + String proxy = WebAppUtils.getProxyHostAndPort(conf); + URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy); + URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri, + appAttempt.getAppAttemptId().getApplicationId()); + url = result.toASCIIString().substring( + HttpConfig.getSchemePrefix().length()); + } catch (URISyntaxException ex) { + Assert.fail(); + } + return url; } /** @@ -448,9 +468,9 @@ public class TestRMAppAttemptTransitions assertEquals(container, applicationAttempt.getMasterContainer()); assertEquals(host, applicationAttempt.getHost()); assertEquals(rpcPort, applicationAttempt.getRpcPort()); - assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); if (unmanagedAM) { - assertEquals("oldtrackingurl", applicationAttempt.getTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl()); } else { assertEquals(getProxyUrl(applicationAttempt), applicationAttempt.getTrackingUrl()); @@ -468,7 +488,7 @@ public class TestRMAppAttemptTransitions assertEquals(RMAppAttemptState.FINISHING, applicationAttempt.getAppAttemptState()); assertEquals(diagnostics, applicationAttempt.getDiagnostics()); - assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); assertEquals(getProxyUrl(applicationAttempt), applicationAttempt.getTrackingUrl()); assertEquals(container, applicationAttempt.getMasterContainer()); @@ -487,9 +507,9 @@ public class TestRMAppAttemptTransitions assertEquals(RMAppAttemptState.FINISHED, applicationAttempt.getAppAttemptState()); assertEquals(diagnostics, applicationAttempt.getDiagnostics()); - assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl()); if (unmanagedAM) { - assertEquals("mytrackingurl", applicationAttempt.getTrackingUrl()); + verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl()); } else { assertEquals(getProxyUrl(applicationAttempt), @@ -603,9 +623,7 @@ public class TestRMAppAttemptTransitions trackingUrl, diagnostics); } - - @Test - public void testUnmanagedAMSuccess() { + private void testUnmanagedAMSuccess(String url) { unmanagedAM = true; when(submissionContext.getUnmanagedAM()).thenReturn(true); // submit AM and check it goes to LAUNCHED state @@ -615,7 +633,7 @@ public class TestRMAppAttemptTransitions applicationAttempt.getAppAttemptId()); // launch AM - runApplicationAttempt(null, "host", 8042, "oldtrackingurl", true); + runApplicationAttempt(null, "host", 8042, url, true); // complete a container applicationAttempt.handle(new RMAppAttemptContainerAcquiredEvent( @@ -623,13 +641,12 @@ public class TestRMAppAttemptTransitions applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class))); // complete AM - String trackingUrl = "mytrackingurl"; String diagnostics = "Successful"; FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED; applicationAttempt.handle(new RMAppAttemptUnregistrationEvent( - applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus, + applicationAttempt.getAppAttemptId(), url, finalStatus, diagnostics)); - testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1, + testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1, true); } @@ -824,12 +841,42 @@ public class TestRMAppAttemptTransitions "Killed by user"); } + @Test + public void testTrackingUrlUnmanagedAM() { + testUnmanagedAMSuccess("oldTrackingUrl"); + } @Test - public void testNoTrackingUrl() { + public void testEmptyTrackingUrlUnmanagedAM() { + testUnmanagedAMSuccess(""); + } + + @Test + public void testNullTrackingUrlUnmanagedAM() { + testUnmanagedAMSuccess(null); + } + + @Test + public void testManagedAMWithTrackingUrl() { + testTrackingUrlManagedAM("theTrackingUrl"); + } + + @Test + public void testManagedAMWithEmptyTrackingUrl() { + testTrackingUrlManagedAM(""); + } + + @Test + public void testManagedAMWithNullTrackingUrl() { + testTrackingUrlManagedAM(null); + } + + private void testTrackingUrlManagedAM(String url) { Container amContainer = allocateApplicationAttempt(); launchApplicationAttempt(amContainer); - runApplicationAttempt(amContainer, "host", 8042, "", false); + runApplicationAttempt(amContainer, "host", 8042, url, false); + unregisterApplicationAttempt(amContainer, + FinalApplicationStatus.SUCCEEDED, url, "Successful"); } @Test @@ -927,4 +974,12 @@ public class TestRMAppAttemptTransitions } } } + + private void verifyUrl(String url1, String url2) { + if (url1 == null || url1.trim().isEmpty()) { + assertEquals("N/A", url2); + } else { + assertEquals(url1, url2); + } + } }