hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject hadoop git commit: YARN-6137. Yarn client implicitly invoke ATS client which accesses HDFS. Contributed by Li Lu
Date Wed, 08 Feb 2017 21:01:24 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 959db3866 -> af65f8bb5


YARN-6137. Yarn client implicitly invoke ATS client which accesses HDFS. Contributed by Li
Lu

(cherry picked from commit 37b4acf7cee1f05599a84bbb1ebf07979a71f82f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af65f8bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af65f8bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af65f8bb

Branch: refs/heads/branch-2.8
Commit: af65f8bb59863089d0b8f2accd44c27785652751
Parents: 959db38
Author: Jason Lowe <jlowe@yahoo-inc.com>
Authored: Wed Feb 8 14:59:09 2017 -0600
Committer: Jason Lowe <jlowe@yahoo-inc.com>
Committed: Wed Feb 8 15:01:10 2017 -0600

----------------------------------------------------------------------
 .../yarn/client/api/impl/YarnClientImpl.java    | 54 +++++++++-----------
 .../yarn/client/api/impl/TestYarnClient.java    | 24 ++++++---
 2 files changed, 40 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/af65f8bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 6d57994..93893f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -134,7 +134,7 @@ public class YarnClientImpl extends YarnClient {
   private long asyncApiPollTimeoutMillis;
   protected AHSClient historyClient;
   private boolean historyServiceEnabled;
-  protected TimelineClient timelineClient;
+  protected volatile TimelineClient timelineClient;
   @VisibleForTesting
   Text timelineService;
   @VisibleForTesting
@@ -174,24 +174,9 @@ public class YarnClientImpl extends YarnClient {
 
     if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
         YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
-      try {
-        timelineServiceEnabled = true;
-        timelineClient = createTimelineClient();
-        timelineClient.init(conf);
-        timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
-        timelineService = TimelineUtils.buildTimelineTokenService(conf);
-      } catch (NoClassDefFoundError error) {
-        // When attempt to initiate the timeline client with
-        // different set of dependencies, it may fail with
-        // NoClassDefFoundError. When some of them are not compatible
-        // with timeline server. This is not necessarily a fatal error
-        // to the client.
-        LOG.warn("Timeline client could not be initialized "
-            + "because dependency missing or incompatible,"
-            + " disabling timeline client.",
-            error);
-        timelineServiceEnabled = false;
-      }
+      timelineServiceEnabled = true;
+      timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
+      timelineService = TimelineUtils.buildTimelineTokenService(conf);
     }
 
     timelineServiceBestEffort = conf.getBoolean(
@@ -212,9 +197,6 @@ public class YarnClientImpl extends YarnClient {
       if (historyServiceEnabled) {
         historyClient.start();
       }
-      if (timelineServiceEnabled) {
-        timelineClient.start();
-      }
     } catch (IOException e) {
       throw new YarnRuntimeException(e);
     }
@@ -229,7 +211,7 @@ public class YarnClientImpl extends YarnClient {
     if (historyServiceEnabled) {
       historyClient.stop();
     }
-    if (timelineServiceEnabled) {
+    if (timelineClient != null) {
       timelineClient.stop();
     }
     super.serviceStop();
@@ -368,16 +350,26 @@ public class YarnClientImpl extends YarnClient {
   @VisibleForTesting
   org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
       getTimelineDelegationToken() throws IOException, YarnException {
-        try {
-          return timelineClient.getDelegationToken(timelineDTRenewer);
-        } catch (Exception e ) {
-          if (timelineServiceBestEffort) {
-            LOG.warn("Failed to get delegation token from the timeline server: "
-                + e.getMessage());
-            return null;
+    try {
+      // Only reachable when both security and timeline service are enabled.
+      if (timelineClient == null) {
+        synchronized (this) {
+          if (timelineClient == null) {
+            timelineClient = createTimelineClient();
+            timelineClient.init(getConfig());
+            timelineClient.start();
           }
-          throw e;
         }
+      }
+      return timelineClient.getDelegationToken(timelineDTRenewer);
+    } catch (Exception e) {
+      if (timelineServiceBestEffort) {
+        LOG.warn("Failed to get delegation token from the timeline server: "
+            + e.getMessage());
+        return null;
+      }
+      throw e;
+    }
   }
 
   private static String getTimelineDelegationTokenRenewer(Configuration conf)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/af65f8bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 1e90a2a..8d199d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.yarn.client.api.impl;
 
-import static org.junit.Assert.assertFalse;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
@@ -157,25 +156,36 @@ public class TestYarnClient {
   }
 
   @Test
-  public void testTimelineClientInitFailure() throws Exception{
+  public void testStartWithTimelineV15Failure() throws Exception{
     Configuration conf = new Configuration();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+    conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
+    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
+        true);
     YarnClient client = YarnClient.createYarnClient();
     if(client instanceof YarnClientImpl) {
       YarnClientImpl impl = (YarnClientImpl) client;
       YarnClientImpl spyClient = spy(impl);
       when(spyClient.createTimelineClient()).thenThrow(
-          new NoClassDefFoundError(
-              "Mock a failure when init timeline instance"));
+          new IOException("ATS v1.5 client initialization failed. "));
       spyClient.init(conf);
       spyClient.start();
-      assertFalse("Timeline client should be disabled when"
-          + "it is failed to init",
-          spyClient.timelineServiceEnabled);
+      spyClient.getTimelineDelegationToken();
       spyClient.stop();
     }
   }
 
+  @Test
+  public void testStartWithTimelineV15() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+    conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
+    YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient();
+    client.init(conf);
+    client.start();
+    client.stop();
+  }
+
   @SuppressWarnings("deprecation")
   @Test (timeout = 30000)
   public void testSubmitApplication() throws Exception {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message