hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [2/8] hadoop git commit: HADOOP-12563. Updated utility (dtutil) to create/modify token files. Contributed by Matthew Paduano
Date Mon, 02 May 2016 21:57:12 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java
new file mode 100644
index 0000000..6067918
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.CommandShell;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestCommandShell {
+
+  public class Example extends CommandShell {
+    public static final String EXAMPLE = "example";
+    public static final String HELLO = "hello";
+    public static final String HELLO_MSG = "hello is running";
+    public static final String GOODBYE = "goodbye";
+    public static final String GOODBYE_MSG = "goodbye is running";
+
+    public String[] savedArgs = null;
+
+    @Override
+    protected int init(String[] args) throws Exception {
+      String command = args[0];
+      if (command.equals(HELLO)) {
+        setSubCommand(new Hello());
+      } else if (command.equals(GOODBYE)) {
+        setSubCommand(new Goodbye());
+      } else{
+        return 1;
+      }
+      savedArgs = args;
+      return 0;
+    }
+
+    public String getCommandUsage() {
+      return EXAMPLE;
+    }
+
+    public class Hello extends SubCommand {
+      public static final String HELLO_USAGE = EXAMPLE + " hello";
+      @Override
+      public boolean validate() {
+        return savedArgs.length == 1;
+      }
+      @Override
+      public void execute() throws Exception {
+        System.out.println(HELLO_MSG);
+      }
+      @Override
+      public String getUsage() {
+        return HELLO_USAGE;
+      }
+    }
+
+    public class Goodbye extends SubCommand {
+      public static final String GOODBYE_USAGE = EXAMPLE + " goodbye";
+      @Override
+      public void execute() throws Exception {
+        System.out.println(GOODBYE_MSG);
+      }
+      @Override
+      public String getUsage() {
+        return GOODBYE_USAGE;
+      }
+    }
+  }
+
+  private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+
+  private String outMsg(String message) {
+    return "OUT:\n" + outContent.toString() + "\n" + message;
+  }
+
+  @Before
+  public void setup() throws Exception {
+    System.setOut(new PrintStream(outContent));
+  }
+
+  @Test
+  public void testCommandShellExample() throws Exception {
+    Example ex = new Example();
+    ex.setConf(new Configuration());
+    int rc = 0;
+
+    outContent.reset();
+    String[] args1 = {"hello"};
+    rc = ex.run(args1);
+    assertEquals(outMsg("test exit code - normal hello"), 0, rc);
+    assertTrue(outMsg("test normal hello message"),
+               outContent.toString().contains(Example.HELLO_MSG));
+
+    outContent.reset();
+    String[] args2 = {"hello", "x"};
+    rc = ex.run(args2);
+    assertEquals(outMsg("test exit code - bad hello"), 1, rc);
+    assertTrue(outMsg("test bad hello message"),
+               outContent.toString().contains(Example.Hello.HELLO_USAGE));
+
+    outContent.reset();
+    String[] args3 = {"goodbye"};
+    rc = ex.run(args3);
+    assertEquals(outMsg("test exit code - normal goodbye"), 0, rc);
+    assertTrue(outMsg("test normal goodbye message"),
+               outContent.toString().contains(Example.GOODBYE_MSG));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
b/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
new file mode 100644
index 0000000..0f4a310
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
@@ -0,0 +1,14 @@
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+org.apache.hadoop.security.token.TestDtFetcher

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsDtFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsDtFetcher.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsDtFetcher.java
new file mode 100644
index 0000000..02aa4b9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsDtFetcher.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.DtFetcher;
+import org.apache.hadoop.security.token.Token;
+
+
+/**
+ *  DtFetcher is an interface which permits the abstraction and separation of
+ *  delegation token fetch implementaions across different packages and
+ *  compilation units.  Resolution of fetcher impl will be done at runtime.
+ */
+public class HdfsDtFetcher implements DtFetcher {
+  private static final Log LOG = LogFactory.getLog(HdfsDtFetcher.class);
+
+  private static final String SERVICE_NAME = HdfsConstants.HDFS_URI_SCHEME;
+
+  private static final String FETCH_FAILED = "Fetch of delegation token failed";
+
+  /**
+   * Returns the service name for HDFS, which is also a valid URL prefix.
+   */
+  public Text getServiceName() {
+    return new Text(SERVICE_NAME);
+  }
+
+  public boolean isTokenRequired() {
+    return UserGroupInformation.isSecurityEnabled();
+  }
+
+  /**
+   *  Returns Token object via FileSystem, null if bad argument.
+   *  @param conf - a Configuration object used with FileSystem.get()
+   *  @param creds - a Credentials object to which token(s) will be added
+   *  @param renewer  - the renewer to send with the token request
+   *  @param url  - the URL to which the request is sent
+   *  @return a Token, or null if fetch fails.
+   */
+  public Token<?> addDelegationTokens(Configuration conf, Credentials creds,
+                                  String renewer, String url) throws Exception {
+    if (!url.startsWith(getServiceName().toString())) {
+      url = getServiceName().toString() + "://" + url;
+    }
+    FileSystem fs = FileSystem.get(URI.create(url), conf);
+    Token<?> token = fs.getDelegationToken(renewer);
+    if (token == null) {
+      LOG.error(FETCH_FAILED);
+      throw new IOException(FETCH_FAILED);
+    }
+    creds.addToken(token.getService(), token);
+    return token;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SWebHdfsDtFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SWebHdfsDtFetcher.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SWebHdfsDtFetcher.java
new file mode 100644
index 0000000..46f9b00
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SWebHdfsDtFetcher.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.io.Text;
+
+/**
+ *  DtFetcher for SWebHdfsFileSystem using the base class HdfsDtFetcher impl.
+ */
+public class SWebHdfsDtFetcher extends HdfsDtFetcher {
+  private static final Log LOG = LogFactory.getLog(SWebHdfsDtFetcher.class);
+
+  private static final String SERVICE_NAME = WebHdfsConstants.SWEBHDFS_SCHEME;
+
+  @Override
+  public Text getServiceName() {
+    return new Text(SERVICE_NAME);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/WebHdfsDtFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/WebHdfsDtFetcher.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/WebHdfsDtFetcher.java
new file mode 100644
index 0000000..c2bb852
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/WebHdfsDtFetcher.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.io.Text;
+
+/**
+ *  DtFetcher for WebHdfsFileSystem using the base class HdfsDtFetcher impl.
+ */
+public class WebHdfsDtFetcher extends HdfsDtFetcher {
+  private static final Log LOG = LogFactory.getLog(WebHdfsDtFetcher.class);
+
+  private static final String SERVICE_NAME = WebHdfsConstants.WEBHDFS_SCHEME;
+
+  @Override
+  public Text getServiceName() {
+    return new Text(SERVICE_NAME);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
index 39821aa..f24b50e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
@@ -179,7 +179,8 @@ public class DelegationTokenFetcher {
     if (null != token) {
       Credentials cred = new Credentials();
       cred.addToken(token.getService(), token);
-      cred.writeTokenStorageFile(tokenFile, conf);
+      // dtutil is replacing this tool; preserve legacy functionality
+      cred.writeLegacyTokenStorageFile(tokenFile, conf);
 
       if (LOG.isDebugEnabled()) {
         LOG.debug("Fetched token " + fs.getUri() + " for " +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
new file mode 100644
index 0000000..a4217fb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.hdfs.HdfsDtFetcher
+org.apache.hadoop.hdfs.WebHdfsDtFetcher
+org.apache.hadoop.hdfs.SWebHdfsDtFetcher


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message