tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject git commit: TEZ-983. Support a helper function to extract required additional tokens from a file. (hitesh)
Date Fri, 28 Mar 2014 17:58:34 GMT
Repository: incubator-tez
Updated Branches:
  refs/heads/master cde19354d -> e32252cdc


TEZ-983. Support a helper function to extract required additional tokens from a file. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/e32252cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/e32252cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/e32252cd

Branch: refs/heads/master
Commit: e32252cdcf6e52c1f361c780a7373168242e7c6c
Parents: cde1935
Author: Hitesh Shah <hitesh@apache.org>
Authored: Fri Mar 28 10:58:10 2014 -0700
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Fri Mar 28 10:58:10 2014 -0700

----------------------------------------------------------------------
 tez-api/pom.xml                                 |  11 ++
 .../apache/tez/common/security/TokenCache.java  |  23 ++++
 .../tez/common/security/TestTokenCache.java     | 122 +++++++++++++++++++
 .../apache/tez/mapreduce/hadoop/MRHelpers.java  |  19 +++
 4 files changed, 175 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e32252cd/tez-api/pom.xml
----------------------------------------------------------------------
diff --git a/tez-api/pom.xml b/tez-api/pom.xml
index fe3c36b..75f5a62 100644
--- a/tez-api/pom.xml
+++ b/tez-api/pom.xml
@@ -53,6 +53,17 @@
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e32252cd/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
index 6c0b1fc..d270fd4 100644
--- a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
+++ b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java
@@ -137,4 +137,27 @@ public class TokenCache {
     }
     return (Token<JobTokenIdentifier>) token;
   }
+
+  /**
+   * Merge tokens from a configured binary file into provided Credentials object
+   * @param creds Credentials object to add new tokens to
+   * @param tokenFilePath Location of tokens' binary file
+   */
+  @InterfaceAudience.Private
+  public static void mergeBinaryTokens(Credentials creds,
+      Configuration conf, String tokenFilePath)
+      throws IOException {
+    if (tokenFilePath == null || tokenFilePath.isEmpty()) {
+      throw new RuntimeException("Invalid file path provided"
+          + ", tokenFilePath=" + tokenFilePath);
+    }
+    LOG.info("Merging additional tokens from binary file"
+        + ", binaryFileName=" + tokenFilePath);
+    Credentials binary = Credentials.readTokenStorageFile(
+        new Path("file:///" +  tokenFilePath), conf);
+
+    // supplement existing tokens with the tokens in the binary file
+    creds.mergeAll(binary);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e32252cd/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
new file mode 100644
index 0000000..4706e8a
--- /dev/null
+++ b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common.security;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class TestTokenCache {
+
+  private static Configuration conf;
+
+  private static String renewer;
+
+  @BeforeClass
+  public static void setup() throws Exception {
+    conf = new Configuration();
+    conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM");
+    renewer = Master.getMasterPrincipal(conf);
+  }
+
+  @Test
+  @SuppressWarnings("deprecation")
+  public void testBinaryCredentials() throws Exception {
+    String binaryTokenFile = null;
+    try {
+      Path TEST_ROOT_DIR = new Path("target");
+      binaryTokenFile = FileSystem.getLocal(conf).makeQualified(
+        new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath();
+
+      MockFileSystem fs1 = createFileSystemForServiceName("service1");
+      MockFileSystem fs2 = createFileSystemForServiceName("service2");
+      MockFileSystem fs3 = createFileSystemForServiceName("service3");
+
+      // get the tokens for fs1 & fs2 and write out to binary creds file
+      Credentials creds = new Credentials();
+      Token<?> token1 = fs1.getDelegationToken(renewer);
+      Token<?> token2 = fs2.getDelegationToken(renewer);
+      creds.addToken(token1.getService(), token1);
+      creds.addToken(token2.getService(), token2);
+      creds.writeTokenStorageFile(new Path(binaryTokenFile), conf);
+
+
+      Credentials newCreds = new Credentials();
+      TokenCache.mergeBinaryTokens(newCreds, conf, binaryTokenFile);
+
+      Assert.assertTrue(newCreds.getAllTokens().size() > 0);
+      checkTokens(creds, newCreds);
+    } finally {
+      if (binaryTokenFile != null) {
+        try {
+          FileSystem.getLocal(conf).delete(new Path(binaryTokenFile));
+        } catch (IOException e) {
+          // Ignore
+        }
+      }
+    }
+  }
+
+  private MockFileSystem createFileSystemForServiceName(final String service)
+      throws IOException {
+    MockFileSystem mockFs = new MockFileSystem();
+    when(mockFs.getCanonicalServiceName()).thenReturn(service);
+    when(mockFs.getDelegationToken(any(String.class))).thenAnswer(
+        new Answer<Token<?>>() {
+          int unique = 0;
+          @Override
+          public Token<?> answer(InvocationOnMock invocation) throws Throwable {
+            Token<?> token = new Token<TokenIdentifier>();
+            token.setService(new Text(service));
+            // use unique value so when we restore from token storage, we can
+            // tell if it's really the same token
+            token.setKind(new Text("token" + unique++));
+            return token;
+          }
+        });
+    return mockFs;
+  }
+
+  private void checkTokens(Credentials creds, Credentials newCreds) {
+    Assert.assertEquals(creds.getAllTokens().size(),
+        newCreds.getAllTokens().size());
+    for (Token<?> token : newCreds.getAllTokens()) {
+      Token<?> credsToken = creds.getToken(token.getService());
+      Assert.assertTrue(credsToken != null);
+      Assert.assertEquals(token, credsToken);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e32252cd/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index 3ed3702..d92a563 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobSubmissionFiles;
 import org.apache.hadoop.mapreduce.split.JobSplitWriter;
 import org.apache.hadoop.mapreduce.v2.proto.MRProtos;
+import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.ContainerLogAppender;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -63,6 +64,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.common.TezUtils;
 import org.apache.tez.common.TezYARNUtils;
+import org.apache.tez.common.security.TokenCache;
 import org.apache.tez.dag.api.InputDescriptor;
 import org.apache.tez.dag.api.OutputDescriptor;
 import org.apache.tez.dag.api.TezConfiguration;
@@ -1018,4 +1020,21 @@ public class MRHelpers {
 
     return Lists.newArrayList(iterable);
   }
+
+  /**
+   * Merge tokens from a configured binary file into provided Credentials object.
+   * Uses "mapreduce.job.credentials.binary" property to find location of token file.
+   * @param creds Credentials object to add new tokens to
+   * @param conf Configuration containing location of token file.
+   * @throws IOException
+   */
+  public static void mergeMRBinaryTokens(Credentials creds,
+      Configuration conf) throws IOException {
+    String tokenFilePath = conf.get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY);
+    if (tokenFilePath == null || tokenFilePath.isEmpty()) {
+      return;
+    }
+    TokenCache.mergeBinaryTokens(creds, conf, tokenFilePath);
+  }
+
 }


Mime
View raw message