Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 366231163B for ; Fri, 28 Mar 2014 19:08:38 +0000 (UTC) Received: (qmail 54525 invoked by uid 500); 28 Mar 2014 19:08:37 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 54487 invoked by uid 500); 28 Mar 2014 19:08:36 -0000 Mailing-List: contact commits-help@tez.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.incubator.apache.org Delivered-To: mailing list commits@tez.incubator.apache.org Received: (qmail 54477 invoked by uid 99); 28 Mar 2014 19:08:34 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 28 Mar 2014 19:08:34 +0000 X-ASF-Spam-Status: No, hits=-2000.5 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Fri, 28 Mar 2014 19:08:32 +0000 Received: (qmail 53945 invoked by uid 99); 28 Mar 2014 19:08:09 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 28 Mar 2014 19:08:09 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 6163F9155AF; Fri, 28 Mar 2014 19:08:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sseth@apache.org To: commits@tez.incubator.apache.org Message-Id: <5924c925f9fb4040a280ab4e0434c221@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: TEZ-990. Add support for a credentials file read by TezClient. (sseth) Date: Fri, 28 Mar 2014 19:08:09 +0000 (UTC) X-Virus-Checked: Checked by ClamAV on apache.org Repository: incubator-tez Updated Branches: refs/heads/master e32252cdc -> 6416102f4 TEZ-990. Add support for a credentials file read by TezClient. (sseth) Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/6416102f Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/6416102f Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/6416102f Branch: refs/heads/master Commit: 6416102f4ffbcde66e3bf4595d268d2539019c2e Parents: e32252c Author: Siddharth Seth Authored: Fri Mar 28 12:07:48 2014 -0700 Committer: Siddharth Seth Committed: Fri Mar 28 12:07:48 2014 -0700 ---------------------------------------------------------------------- .../src/main/java/org/apache/tez/client/TezClient.java | 1 + .../main/java/org/apache/tez/client/TezClientUtils.java | 12 ++++++++++++ .../src/main/java/org/apache/tez/client/TezSession.java | 3 +++ .../main/java/org/apache/tez/common/TezJobConfig.java | 5 +++++ .../org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java | 2 ++ .../java/org/apache/tez/mapreduce/hadoop/MRHelpers.java | 4 ++++ 6 files changed, 27 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/client/TezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java index 61de7e0..f4c6011 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java @@ -96,6 +96,7 @@ public class TezClient { if (credentials == null) { credentials = new Credentials(); } + TezClientUtils.processTezLocalCredentialsFile(credentials, conf); // Add session token for shuffle TezClientUtils.createSessionToken(appId.toString(), http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index ca37313..8b87fb0 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -76,6 +76,7 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.log4j.Level; +import org.apache.tez.common.TezJobConfig; import org.apache.tez.common.TezYARNUtils; import org.apache.tez.common.impl.LogUtils; import org.apache.tez.common.security.JobTokenIdentifier; @@ -198,6 +199,16 @@ public class TezClientUtils { return tezJarResources; } + static void processTezLocalCredentialsFile(Credentials credentials, Configuration conf) + throws IOException { + String path = conf.get(TezJobConfig.TEZ_CREDENTIALS_PATH); + if (path == null) { + return; + } else { + TokenCache.mergeBinaryTokens(credentials, conf, path); + } + } + /** * Verify or create the Staging area directory on the configured Filesystem * @param stagingArea Staging area directory path @@ -316,6 +327,7 @@ public class TezClientUtils { throws IOException, YarnException{ Preconditions.checkNotNull(sessionCreds); + FileSystem fs = TezClientUtils.ensureStagingDirExists(conf, amConfig.getStagingDir()); Path binaryConfPath = new Path(amConfig.getStagingDir(), http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/client/TezSession.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezSession.java b/tez-api/src/main/java/org/apache/tez/client/TezSession.java index 055218e..ff7f00a 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezSession.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezSession.java @@ -94,6 +94,9 @@ public class TezSession { yarnClient = YarnClient.createYarnClient(); yarnClient.init(sessionConfig.getYarnConfiguration()); yarnClient.start(); + + TezClientUtils.processTezLocalCredentialsFile(sessionCredentials, + sessionConfig.getTezConfiguration()); Map tezJarResources = TezClientUtils.setupTezJarsLocalResources( http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java index a45753b..0af0ffb 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java @@ -344,4 +344,9 @@ public class TezJobConfig { /** Defines the ProcessTree implementation which will be used to collect resource utilization. */ public static final String TEZ_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS = "tez.resource.calculator.process-tree.class"; + + /** + * Path to a credentials file located on the local file system with serialized credentials + */ + public static final String TEZ_CREDENTIALS_PATH = "tez.credentials.path"; } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java index 621a1b8..f1c5e14 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java @@ -191,6 +191,8 @@ public class DeprecatedKeys { registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS); registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_RUNTIME_INTERMEDIATE_INPUT_KEY_SECONDARY_COMPARATOR_CLASS); + + registerMRToRuntimeKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezJobConfig.TEZ_CREDENTIALS_PATH); } http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java index d92a563..7538ccf 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java @@ -1026,6 +1026,10 @@ public class MRHelpers { * Uses "mapreduce.job.credentials.binary" property to find location of token file. * @param creds Credentials object to add new tokens to * @param conf Configuration containing location of token file. + * + * TezClient reads credentials from the property - TezJobConfig.TEZ_CREDENTIALS_PATH. This method + * is not required if that property is set. + * * @throws IOException */ public static void mergeMRBinaryTokens(Credentials creds,