From commits-return-5937-archive-asf-public=cust-asf.ponee.io@tez.apache.org Thu Feb 28 03:58:24 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 1B6D7180608 for ; Thu, 28 Feb 2019 04:58:23 +0100 (CET) Received: (qmail 8996 invoked by uid 500); 28 Feb 2019 03:58:23 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 8987 invoked by uid 99); 28 Feb 2019 03:58:22 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Feb 2019 03:58:22 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 50D9182E8A; Thu, 28 Feb 2019 03:58:22 +0000 (UTC) Date: Thu, 28 Feb 2019 03:58:22 +0000 To: "commits@tez.apache.org" Subject: [tez] branch master updated: TEZ-4032. TEZ will throw Client cannot authenticate via:[TOKEN, KERBEROS] when used with HDFS federation(non viewfs, only hdfs schema used). MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <155132630227.21687.12271172665319997059@gitbox.apache.org> From: jeagles@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: tez X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 0093f8b4d9ea38fcdfeb802fe3206c4b2ae8f077 X-Git-Newrev: e88e824fccb0e3dac0ce854738040e885f0d606b X-Git-Rev: e88e824fccb0e3dac0ce854738040e885f0d606b X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. jeagles pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/tez.git The following commit(s) were added to refs/heads/master by this push: new e88e824 TEZ-4032. TEZ will throw Client cannot authenticate via:[TOKEN, KERBEROS] when used with HDFS federation(non viewfs, only hdfs schema used). e88e824 is described below commit e88e824fccb0e3dac0ce854738040e885f0d606b Author: Zhang Butao AuthorDate: Wed Feb 27 21:57:41 2019 -0600 TEZ-4032. TEZ will throw Client cannot authenticate via:[TOKEN, KERBEROS] when used with HDFS federation(non viewfs, only hdfs schema used). --- .../java/org/apache/tez/client/TezClientUtils.java | 19 +++++++++++++++ .../org/apache/tez/common/security/TokenCache.java | 27 ++++++++++++++++++---- .../org/apache/tez/dag/api/TezConfiguration.java | 14 +++++++++++ .../apache/tez/common/security/TestTokenCache.java | 7 +++++- .../tez/mapreduce/hadoop/DeprecatedKeys.java | 4 ++++ .../apache/tez/mapreduce/hadoop/MRJobConfig.java | 2 ++ 6 files changed, 68 insertions(+), 5 deletions(-) diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index 689d947..cd3ae6b 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -27,6 +27,7 @@ import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -495,6 +496,8 @@ public class TezClientUtils { // Add Staging dir creds to the list of session credentials. TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[]{binaryConfPath}, conf); + populateTokenCache(conf, sessionCreds); + // Add session specific credentials to the AM credentials. amLaunchCredentials.mergeAll(sessionCreds); @@ -716,6 +719,22 @@ public class TezClientUtils { return appContext; } + + //get secret keys and tokens and store them into TokenCache + private static void populateTokenCache(TezConfiguration conf, Credentials credentials) + throws IOException{ + // add the delegation tokens from configuration + String[] nameNodes = conf.getStrings(TezConfiguration.TEZ_JOB_FS_SERVERS); + LOG.debug("adding the following namenodes' delegation tokens:" + + Arrays.toString(nameNodes)); + if(nameNodes != null) { + Path[] ps = new Path[nameNodes.length]; + for(int i = 0; i < nameNodes.length; i++) { + ps[i] = new Path(nameNodes[i]); + } + TokenCache.obtainTokensForFileSystems(credentials, ps, conf); + } + } static DAGPlan prepareAndCreateDAGPlan(DAG dag, AMConfiguration amConfig, Map tezJarResources, boolean tezLrsAsArchive, diff --git a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java index fc2c07d..e56ef61 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java @@ -34,6 +34,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.tez.dag.api.TezConfiguration; /** @@ -101,6 +102,20 @@ public class TokenCache { } } + static boolean isTokenRenewalExcluded(FileSystem fs, Configuration conf) { + String[] nns = + conf.getStrings(TezConfiguration.TEZ_JOB_FS_SERVERS_TOKEN_RENEWAL_EXCLUDE); + if (nns != null) { + String host = fs.getUri().getHost(); + for(int i = 0; i < nns.length; i++) { + if (nns[i].equals(host)) { + return true; + } + } + } + return false; + } + /** * get delegation token for a specific FS * @param fs @@ -112,10 +127,14 @@ public class TokenCache { static void obtainTokensForFileSystemsInternal(FileSystem fs, Credentials credentials, Configuration conf) throws IOException { // TODO Change this to use YARN utilities once YARN-1664 is fixed. - String delegTokenRenewer = Master.getMasterPrincipal(conf); - if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { - throw new IOException( - "Can't get Master Kerberos principal for use as renewer"); + // RM skips renewing token with empty renewer + String delegTokenRenewer = ""; + if (!isTokenRenewalExcluded(fs, conf)) { + delegTokenRenewer = Master.getMasterPrincipal(conf); + if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { + throw new IOException( + "Can't get Master Kerberos principal for use as renewer"); + } } final Token tokens[] = fs.addDelegationTokens(delegTokenRenewer, diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 4566600..8ce8f7c 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -2010,4 +2010,18 @@ public class TezConfiguration extends Configuration { public static final String TEZ_SHARED_EXECUTOR_MAX_THREADS = "tez.shared-executor.max-threads"; public static final int TEZ_SHARED_EXECUTOR_MAX_THREADS_DEFAULT = -1; + /** + * Acquire all FileSystems info. e.g., all namenodes info of HDFS federation cluster. + */ + @ConfigurationScope(Scope.AM) + @ConfigurationProperty + public static final String TEZ_JOB_FS_SERVERS = "tez.job.fs-servers"; + + /** + * Skip delegation token renewal for specified FileSystems. + */ + @ConfigurationScope(Scope.AM) + @ConfigurationProperty + public static final String TEZ_JOB_FS_SERVERS_TOKEN_RENEWAL_EXCLUDE = "tez.job.fs-servers.token-renewal.exclude"; + } diff --git a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java index 59488b6..fcb1e98 100644 --- a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java +++ b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java @@ -113,6 +113,11 @@ public class TestTokenCache { conf.setBoolean("fs.test.impl.disable.cache", true); TokenCache.obtainTokensForFileSystemsInternal(creds, paths, conf); verify(TestFileSystem.fs, times(paths.length + 1)).addDelegationTokens(renewer, creds); + + // Excluded filesystem tokens should not be obtained. + conf.set("tez.job.fs-servers.token-renewal.exclude", "dir"); + TokenCache.obtainTokensForFileSystemsInternal(creds, paths, conf); + verify(TestFileSystem.fs, times(paths.length + 1)).addDelegationTokens(renewer, creds); } private Path[] makePaths(int count, String prefix) throws Exception { @@ -127,7 +132,7 @@ public class TestTokenCache { static final FileSystem fs = mock(FileSystem.class); static { try { - when(fs.getUri()).thenReturn(new URI("test:///")); + when(fs.getUri()).thenReturn(new URI("test://dir")); } catch (URISyntaxException e) { throw new RuntimeException(e); } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java index d9b0930..b8d491a 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java @@ -88,6 +88,10 @@ public class DeprecatedKeys { TezConfiguration.TEZ_APPLICATION_TAGS); mrParamToDAGParamMap.put(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, TezConfiguration.TEZ_USER_CLASSPATH_FIRST); + mrParamToDAGParamMap.put(MRJobConfig.JOB_NAMENODES, + TezConfiguration.TEZ_JOB_FS_SERVERS); + mrParamToDAGParamMap.put(MRJobConfig.JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, + TezConfiguration.TEZ_JOB_FS_SERVERS_TOKEN_RENEWAL_EXCLUDE); } // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild. diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java index cd6fd44..ca954d9 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java @@ -302,6 +302,8 @@ public interface MRJobConfig { public static final String JOB_NAMENODES = "mapreduce.job.hdfs-servers"; + public static final String JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE = "mapreduce.job.hdfs-servers.token-renewal.exclude"; + public static final String JOB_JOBTRACKER_ID = "mapreduce.job.kerberos.jtprinicipal"; public static final String JOB_CANCEL_DELEGATION_TOKEN = "mapreduce.job.complete.cancel.delegation.tokens";