tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-990. Add support for a credentials file read by TezClient. (sseth)
Date Fri, 28 Mar 2014 19:08:09 GMT
Repository: incubator-tez
Updated Branches:
  refs/heads/master e32252cdc -> 6416102f4


TEZ-990. Add support for a credentials file read by TezClient. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/6416102f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/6416102f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/6416102f

Branch: refs/heads/master
Commit: 6416102f4ffbcde66e3bf4595d268d2539019c2e
Parents: e32252c
Author: Siddharth Seth <sseth@apache.org>
Authored: Fri Mar 28 12:07:48 2014 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Fri Mar 28 12:07:48 2014 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/tez/client/TezClient.java  |  1 +
 .../main/java/org/apache/tez/client/TezClientUtils.java | 12 ++++++++++++
 .../src/main/java/org/apache/tez/client/TezSession.java |  3 +++
 .../main/java/org/apache/tez/common/TezJobConfig.java   |  5 +++++
 .../org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java |  2 ++
 .../java/org/apache/tez/mapreduce/hadoop/MRHelpers.java |  4 ++++
 6 files changed, 27 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index 61de7e0..f4c6011 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -96,6 +96,7 @@ public class TezClient {
       if (credentials == null) {
         credentials = new Credentials();
       }
+      TezClientUtils.processTezLocalCredentialsFile(credentials, conf);
 
       // Add session token for shuffle
       TezClientUtils.createSessionToken(appId.toString(),

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index ca37313..8b87fb0 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.log4j.Level;
+import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezYARNUtils;
 import org.apache.tez.common.impl.LogUtils;
 import org.apache.tez.common.security.JobTokenIdentifier;
@@ -198,6 +199,16 @@ public class TezClientUtils {
     return tezJarResources;
   }
 
+  static void processTezLocalCredentialsFile(Credentials credentials, Configuration conf)
+      throws IOException {
+    String path = conf.get(TezJobConfig.TEZ_CREDENTIALS_PATH);
+    if (path == null) {
+      return;
+    } else {
+      TokenCache.mergeBinaryTokens(credentials, conf, path);
+    }
+  }
+
   /**
    * Verify or create the Staging area directory on the configured Filesystem
    * @param stagingArea Staging area directory path
@@ -316,6 +327,7 @@ public class TezClientUtils {
           throws IOException, YarnException{
 
     Preconditions.checkNotNull(sessionCreds);
+    
     FileSystem fs = TezClientUtils.ensureStagingDirExists(conf,
         amConfig.getStagingDir());
     Path binaryConfPath =  new Path(amConfig.getStagingDir(),

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/client/TezSession.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezSession.java b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
index 055218e..ff7f00a 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezSession.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
@@ -94,6 +94,9 @@ public class TezSession {
     yarnClient = YarnClient.createYarnClient();
     yarnClient.init(sessionConfig.getYarnConfiguration());
     yarnClient.start();
+    
+    TezClientUtils.processTezLocalCredentialsFile(sessionCredentials,
+        sessionConfig.getTezConfiguration());
 
     Map<String, LocalResource> tezJarResources =
         TezClientUtils.setupTezJarsLocalResources(

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
index a45753b..0af0ffb 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -344,4 +344,9 @@ public class TezJobConfig {
   
   /** Defines the ProcessTree implementation which will be used to collect resource utilization.
*/
   public static final String TEZ_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS = "tez.resource.calculator.process-tree.class";
+  
+  /**
+   * Path to a credentials file located on the local file system with serialized credentials
+   */
+  public static final String TEZ_CREDENTIALS_PATH = "tez.credentials.path";
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index 621a1b8..f1c5e14 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -191,6 +191,8 @@ public class DeprecatedKeys {
     registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS);
     
     registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezJobConfig.TEZ_RUNTIME_INTERMEDIATE_INPUT_KEY_SECONDARY_COMPARATOR_CLASS);
+    
+    registerMRToRuntimeKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezJobConfig.TEZ_CREDENTIALS_PATH);
 
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6416102f/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index d92a563..7538ccf 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -1026,6 +1026,10 @@ public class MRHelpers {
    * Uses "mapreduce.job.credentials.binary" property to find location of token file.
    * @param creds Credentials object to add new tokens to
    * @param conf Configuration containing location of token file.
+   * 
+   * TezClient reads credentials from the property - TezJobConfig.TEZ_CREDENTIALS_PATH. This
method
+   * is not required if that property is set.
+   * 
    * @throws IOException
    */
   public static void mergeMRBinaryTokens(Credentials creds,


Mime
View raw message