tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject tez git commit: TEZ-3494. Support relative url for tez-ui.history-url.base config (hitesh via sree)
Date Wed, 25 Jan 2017 16:24:48 GMT
Repository: tez
Updated Branches:
  refs/heads/master d227b99a4 -> 24a8a95bd


TEZ-3494. Support relative url for tez-ui.history-url.base config (hitesh via sree)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/24a8a95b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/24a8a95b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/24a8a95b

Branch: refs/heads/master
Commit: 24a8a95bd2093377154760cf57ebe4f56c6197f4
Parents: d227b99
Author: Sreenath Somarajapuram <sree@apache.org>
Authored: Wed Jan 25 21:55:20 2017 +0530
Committer: Sreenath Somarajapuram <sree@apache.org>
Committed: Wed Jan 25 21:55:20 2017 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../apache/tez/dag/api/TezConfiguration.java    | 12 ++++++
 .../tez/dag/app/rm/TaskSchedulerManager.java    | 14 +++++--
 .../dag/app/rm/TestTaskSchedulerManager.java    | 42 +++++++++++++++-----
 4 files changed, 56 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4d43bca..d6798ff 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -177,6 +177,7 @@ ALL CHANGES:
   TEZ-3583. Tez UI: UTs are flaky because of a dependency issue
   TEZ-3580. Tez UI: Pagination broken on queries page
   TEZ-3575. RM have started forwarding origin. Use that in AMWebController for CORS support
+  TEZ-3494. Support relative url for tez-ui.history-url.base config
 
 Release 0.8.5: Unreleased
 

http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index e27cdf8..8747616 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1335,6 +1335,18 @@ public class TezConfiguration extends Configuration {
       TEZ_PREFIX + "yarn.ats.max.events.per.batch";
   public static final int YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT = 5;
 
+  /**
+   * Boolean value. Default true.
+   * Whether to fix the history url if it has not been configured correctly i.e. it does
not have a
+   * scheme in the value. By default, the url will be prepended with a scheme (http) if there
is
+   * none present.
+   */
+  @Private
+  @ConfigurationScope(Scope.AM)
+  public static final String TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED =
+      TEZ_PREFIX + "am.ui.history.url.scheme.check.enabled";
+  public static final boolean TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED_DEFAULT = true;
+
 
   /**
    * Int value. Time, in milliseconds, to wait for an event before sending a batch to ATS.

http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
index e68c9b8..7989e5f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
@@ -988,9 +988,17 @@ public class TaskSchedulerManager extends AbstractService implements
           .replaceAll(HISTORY_URL_BASE, historyUrlBase)
           .replaceAll("([^:])/{2,}", "$1/");
 
-      // make sure we have a valid scheme
-      if (!historyUrl.startsWith("http")) {
-        historyUrl = "http://" + historyUrl;
+      // FIXME: we should not need this check in the first place.
+      // Might be better to remove it in the next release and not fix badly configured URLs.
+      boolean checkSchemePrefix = config.getBoolean(
+          TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED,
+          TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED_DEFAULT);
+
+      if (checkSchemePrefix) {
+        // make sure we have a valid scheme
+        if (!historyUrl.startsWith("http")) {
+          historyUrl = "http://" + historyUrl;
+        }
       }
     }
 

http://git-wip-us.apache.org/repos/asf/tez/blob/24a8a95b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
index 791bb7f..66c209a 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
@@ -394,33 +394,55 @@ public class TestTaskSchedulerManager {
 
     // ensure history url is empty when timeline server is not the logging class
     conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9999");
-    assertTrue("http://ui-host:9999/#/tez-app/TEST_APP_ID"
-        .equals(schedulerHandler.getHistoryUrl()));
+    assertEquals("http://ui-host:9999/#/tez-app/TEST_APP_ID",
+        schedulerHandler.getHistoryUrl());
 
     // ensure the trailing / in history url is handled
     conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://ui-host:9998/");
-    assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
-        .equals(schedulerHandler.getHistoryUrl()));
+    assertEquals("http://ui-host:9998/#/tez-app/TEST_APP_ID",
+        schedulerHandler.getHistoryUrl());
 
     // ensure missing scheme in history url is handled
     conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/");
-    Assert.assertTrue("http://ui-host:9998/#/tez-app/TEST_APP_ID"
-        .equals(schedulerHandler.getHistoryUrl()));
+    assertEquals("http://ui-host:9998/#/tez-app/TEST_APP_ID",
+        schedulerHandler.getHistoryUrl());
 
     // handle bad template ex without begining /
     conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
         "__HISTORY_URL_BASE__#/somepath");
-    assertTrue("http://ui-host:9998/#/somepath"
-        .equals(schedulerHandler.getHistoryUrl()));
+    assertEquals("http://ui-host:9998/#/somepath",
+        schedulerHandler.getHistoryUrl());
 
     conf.set(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE,
         "__HISTORY_URL_BASE__?viewPath=tez-app/__APPLICATION_ID__");
     conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://localhost/ui/tez");
-    assertTrue("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID"
-        .equals(schedulerHandler.getHistoryUrl()));
+    assertEquals("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID",
+        schedulerHandler.getHistoryUrl());
 
   }
 
+  @Test (timeout = 5000)
+  public void testHistoryUrlWithoutScheme() throws Exception {
+    Configuration conf = schedulerHandler.appContext.getAMConf();
+    final ApplicationId mockApplicationId = mock(ApplicationId.class);
+    doReturn("TEST_APP_ID").when(mockApplicationId).toString();
+    doReturn(mockApplicationId).when(mockAppContext).getApplicationID();
+
+    conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "/foo/bar/");
+    conf.setBoolean(TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED, false);
+    assertEquals("/foo/bar/#/tez-app/TEST_APP_ID",
+        schedulerHandler.getHistoryUrl());
+
+    conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/foo/bar/");
+    assertEquals("ui-host:9998/foo/bar/#/tez-app/TEST_APP_ID",
+        schedulerHandler.getHistoryUrl());
+
+    conf.setBoolean(TezConfiguration.TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED, true);
+    conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "ui-host:9998/foo/bar/");
+    assertEquals("http://ui-host:9998/foo/bar/#/tez-app/TEST_APP_ID",
+        schedulerHandler.getHistoryUrl());
+  }
+
   @Test(timeout = 5000)
   public void testNoSchedulerSpecified() throws IOException {
     try {


Mime
View raw message