hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject [50/50] [abbrv] hadoop git commit: YARN-3144. Configuration for making delegation token failures to timeline server not-fatal. Contributed by Jonathan Eagles
Date Fri, 06 Feb 2015 21:47:22 GMT
YARN-3144. Configuration for making delegation token failures to timeline server not-fatal.
Contributed by Jonathan Eagles


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d4e9e34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d4e9e34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d4e9e34

Branch: refs/heads/HDFS-EC
Commit: 6d4e9e343f5999998b10dd57afad78b47f918882
Parents: 2bb2df4
Author: Jason Lowe <jlowe@apache.org>
Authored: Fri Feb 6 20:55:00 2015 +0000
Committer: Zhe Zhang <zhz@apache.org>
Committed: Fri Feb 6 13:45:53 2015 -0800

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../hadoop/yarn/conf/YarnConfiguration.java     |  7 +++
 .../yarn/client/api/impl/YarnClientImpl.java    | 22 +++++++-
 .../yarn/client/api/impl/TestYarnClient.java    | 56 +++++++++++++++-----
 .../src/main/resources/yarn-default.xml         |  6 +++
 5 files changed, 80 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d4e9e34/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 51ed31f..79e66a7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -252,6 +252,9 @@ Release 2.7.0 - UNRELEASED
     ApplicationHistoryClientService are uniform when application-attempt is
     not found. (zjshen via acmurthy)
 
+    YARN-3144. Configuration for making delegation token failures to timeline
+    server not-fatal (Jonathan Eagles via jlowe)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d4e9e34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 015baa1..d6f6dee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1385,6 +1385,13 @@ public class YarnConfiguration extends Configuration {
   public static final long
       DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1000;
 
+  /** Timeline client policy for whether connections are fatal */
+  public static final String TIMELINE_SERVICE_CLIENT_BEST_EFFORT =
+      TIMELINE_SERVICE_CLIENT_PREFIX + "best-effort";
+
+  public static final boolean
+      DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT = false;
+
   /** Flag to enable recovery of timeline service */
   public static final String TIMELINE_SERVICE_RECOVERY_ENABLED =
       TIMELINE_SERVICE_PREFIX + "recovery.enabled";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d4e9e34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 975c267..91fbd00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -129,6 +129,7 @@ public class YarnClientImpl extends YarnClient {
   @VisibleForTesting
   String timelineDTRenewer;
   protected boolean timelineServiceEnabled;
+  protected boolean timelineServiceBestEffort;
 
   private static final String ROOT = "root";
 
@@ -163,14 +164,22 @@ public class YarnClientImpl extends YarnClient {
     if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
         YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
       timelineServiceEnabled = true;
-      timelineClient = TimelineClient.createTimelineClient();
+      timelineClient = createTimelineClient();
       timelineClient.init(conf);
       timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
       timelineService = TimelineUtils.buildTimelineTokenService(conf);
     }
+
+    timelineServiceBestEffort = conf.getBoolean(
+        YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
+        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT);
     super.serviceInit(conf);
   }
 
+  TimelineClient createTimelineClient() throws IOException, YarnException {
+    return TimelineClient.createTimelineClient();
+  }
+
   @Override
   protected void serviceStart() throws Exception {
     try {
@@ -325,7 +334,16 @@ public class YarnClientImpl extends YarnClient {
   @VisibleForTesting
   org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
       getTimelineDelegationToken() throws IOException, YarnException {
-    return timelineClient.getDelegationToken(timelineDTRenewer);
+        try {
+          return timelineClient.getDelegationToken(timelineDTRenewer);
+        } catch (Exception e ) {
+          if (timelineServiceBestEffort) {
+            LOG.warn("Failed to get delegation token from the timeline server: "
+                + e.getMessage());
+            return null;
+          }
+          throw e;
+        }
   }
 
   private static String getTimelineDelegationTokenRenewer(Configuration conf)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d4e9e34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index eb8ca48..73e06af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -854,6 +854,42 @@ public class TestYarnClient {
   }
 
   @Test
+  public void testBestEffortTimelineDelegationToken()
+      throws Exception {
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+    SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+
+    YarnClientImpl client = spy(new YarnClientImpl() {
+
+      @Override
+      TimelineClient createTimelineClient() throws IOException, YarnException {
+        timelineClient = mock(TimelineClient.class);
+        when(timelineClient.getDelegationToken(any(String.class)))
+          .thenThrow(new IOException("Best effort test exception"));
+        return timelineClient;
+      }
+    });
+
+    client.init(conf);
+    try {
+      conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, true);
+      client.serviceInit(conf);
+      client.getTimelineDelegationToken();
+    } catch (Exception e) {
+      Assert.fail("Should not have thrown an exception");
+    }
+    try {
+      conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, false);
+      client.serviceInit(conf);
+      client.getTimelineDelegationToken();
+      Assert.fail("Get delegation token should have thrown an exception");
+    } catch (Exception e) {
+      // Success
+    }
+  }
+
+  @Test
   public void testAutomaticTimelineDelegationTokenLoading()
       throws Exception {
     Configuration conf = new YarnConfiguration();
@@ -864,22 +900,18 @@ public class TestYarnClient {
     final Token<TimelineDelegationTokenIdentifier> dToken =
         new Token<TimelineDelegationTokenIdentifier>(
             timelineDT.getBytes(), new byte[0], timelineDT.getKind(), new Text());
-    // crate a mock client
+    // create a mock client
     YarnClientImpl client = spy(new YarnClientImpl() {
+
       @Override
-      protected void serviceInit(Configuration conf) throws Exception {
-        if (getConfig().getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
-            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
-          timelineServiceEnabled = true;
-          timelineClient = mock(TimelineClient.class);
-          when(timelineClient.getDelegationToken(any(String.class)))
-              .thenReturn(dToken);
-          timelineClient.init(getConfig());
-          timelineService = TimelineUtils.buildTimelineTokenService(getConfig());
-        }
-        this.setConfig(conf);
+      TimelineClient createTimelineClient() throws IOException, YarnException {
+        timelineClient = mock(TimelineClient.class);
+        when(timelineClient.getDelegationToken(any(String.class)))
+          .thenReturn(dToken);
+        return timelineClient;
       }
 
+
       @Override
       protected void serviceStart() throws Exception {
         rmClient = mock(ApplicationClientProtocol.class);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d4e9e34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 349e57b..971ce0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1371,6 +1371,12 @@
   </property>
 
   <property>
+    <description>Client policy for whether timeline operations are non-fatal</description>
+    <name>yarn.timeline-service.client.best-effort</name>
+    <value>false</value>
+  </property>
+
+  <property>
     <description>
     Default retry time interval for timeline servive client.
     </description>


Mime
View raw message