ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sboi...@apache.org
Subject [06/35] ignite git commit: IGNITE-2525: YARN: Added Kerberos handling. This closes #494.
Date Fri, 26 Feb 2016 13:16:24 GMT
IGNITE-2525: YARN: Added Kerberos handling. This closes #494.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/62d69e0d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/62d69e0d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/62d69e0d

Branch: refs/heads/ignite-2407
Commit: 62d69e0da62b3dc9a5ba93bdf52194c6e1486e59
Parents: 592ece0
Author: iveselovskiy <iveselovskiy@gridgain.com>
Authored: Fri Feb 19 17:31:06 2016 +0300
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Fri Feb 19 17:31:06 2016 +0300

----------------------------------------------------------------------
 .../apache/ignite/yarn/ApplicationMaster.java   | 30 +++++++++++++++-----
 .../apache/ignite/yarn/IgniteYarnClient.java    | 25 ++++++++++++++++
 .../ignite/yarn/utils/IgniteYarnUtils.java      | 19 +++++++++++++
 3 files changed, 67 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/62d69e0d/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
index b9ab02d..609f29b 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
@@ -20,6 +20,7 @@ package org.apache.ignite.yarn;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -32,6 +33,8 @@ import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -67,10 +70,10 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
     private long schedulerTimeout = TimeUnit.SECONDS.toMillis(1);
 
     /** Yarn configuration. */
-    private YarnConfiguration conf;
+    private final YarnConfiguration conf;
 
     /** Cluster properties. */
-    private ClusterProperties props;
+    private final ClusterProperties props;
 
     /** Network manager. */
     private NMClient nmClient;
@@ -79,7 +82,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
     private AMRMClientAsync<AMRMClient.ContainerRequest> rmClient;
 
     /** Ignite path. */
-    private Path ignitePath;
+    private final Path ignitePath;
 
     /** Config path. */
     private Path cfgPath;
@@ -87,8 +90,11 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
     /** Hadoop file system. */
     private FileSystem fs;
 
+    /** Buffered tokens to be injected into newly allocated containers. */
+    private ByteBuffer allTokens;
+
     /** Running containers. */
-    private Map<ContainerId, IgniteContainer> containers = new ConcurrentHashMap<>();
+    private final Map<ContainerId, IgniteContainer> containers = new ConcurrentHashMap<>();
 
     /**
      * @param ignitePath Hdfs path to ignite.
@@ -107,6 +113,10 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
                 try {
                     ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
 
+                    if (UserGroupInformation.isSecurityEnabled())
+                        // Set the tokens to the newly allocated container:
+                        ctx.setTokens(allTokens.duplicate());
+
                     Map<String, String> env = new HashMap<>(System.getenv());
 
                     env.put("IGNITE_TCP_DISCOVERY_ADDRESSES", getAddress(c.getNodeId().getHost()));
@@ -192,10 +202,10 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
     /**
      * @return Address running nodes.
      */
-    private String getAddress(String address) {
+    private String getAddress(String addr) {
         if (containers.isEmpty()) {
-            if (address != null && !address.isEmpty())
-                return address + DEFAULT_PORT;
+            if (addr != null && !addr.isEmpty())
+                return addr + DEFAULT_PORT;
 
             return "";
         }
@@ -337,6 +347,12 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler
{
      * @throws IOException
      */
     public void init() throws IOException {
+        if (UserGroupInformation.isSecurityEnabled()) {
+            Credentials cred = UserGroupInformation.getCurrentUser().getCredentials();
+
+            allTokens = IgniteYarnUtils.createTokenBuffer(cred);
+        }
+
         fs = FileSystem.get(conf);
 
         nmClient = NMClient.createNMClient();

http://git-wip-us.apache.org/repos/asf/ignite/blob/62d69e0d/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
index 17a5616..2a9a53e 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.yarn;
 
 import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -25,6 +27,9 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -80,6 +85,7 @@ public class IgniteYarnClient {
         else
             ignite = new Path(props.ignitePath());
 
+        // Upload the jar file to HDFS.
         Path appJar = IgniteYarnUtils.copyLocalToHdfs(fs, pathAppMasterJar,
             props.igniteWorkDir() + File.separator + IgniteYarnUtils.JAR_NAME);
 
@@ -106,6 +112,25 @@ public class IgniteYarnClient {
 
         amContainer.setEnvironment(appMasterEnv);
 
+        // Setup security tokens
+        if (UserGroupInformation.isSecurityEnabled()) {
+            Credentials creds = new Credentials();
+
+            String tokRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
+
+            if (tokRenewer == null || tokRenewer.length() == 0)
+                throw new IOException("Master Kerberos principal for the RM is not set.");
+
+            log.info("Found RM principal: " + tokRenewer);
+
+            final Token<?> tokens[] = fs.addDelegationTokens(tokRenewer, creds);
+
+            if (tokens != null)
+                log.info("File system delegation tokens: " + Arrays.toString(tokens));
+
+            amContainer.setTokens(IgniteYarnUtils.createTokenBuffer(creds));
+        }
+
         // Set up resource type requirements for ApplicationMaster
         Resource capability = Records.newRecord(Resource.class);
         capability.setMemory(512);

http://git-wip-us.apache.org/repos/asf/ignite/blob/62d69e0d/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java
b/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java
index 6265e12..92507a7 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/utils/IgniteYarnUtils.java
@@ -17,9 +17,13 @@
 
 package org.apache.ignite.yarn.utils;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -83,4 +87,19 @@ public class IgniteYarnUtils {
 
         return dstPath;
     }
+
+    /**
+     * Creates a ByteBuffer with serialized {@link Credentials}.
+     *
+     * @param creds The credentials.
+     * @return The ByteBuffer with the credentials.
+     * @throws IOException
+     */
+    public static ByteBuffer createTokenBuffer(Credentials creds) throws IOException {
+        DataOutputBuffer dob = new DataOutputBuffer();
+
+        creds.writeTokenStorageToStream(dob);
+
+        return ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+    }
 }
\ No newline at end of file


Mime
View raw message