hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject [3/3] hadoop git commit: HDFS-10424. DatanodeLifelineProtocol not able to use under security cluster. Contributed by Chris Nauroth.
Date Fri, 20 May 2016 19:59:06 GMT
HDFS-10424. DatanodeLifelineProtocol not able to use under security cluster. Contributed by
Chris Nauroth.

(cherry picked from commit bcde1562d25c4f5595f4e3436dc3630315b1ceed)
(cherry picked from commit d38f2090fa998b716a5d7a8d2314927d6eb30e41)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/877af3ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/877af3ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/877af3ec

Branch: refs/heads/branch-2.8
Commit: 877af3ec7e0b5311ef2ac3899757d1180473ac46
Parents: dbde956
Author: Chris Nauroth <cnauroth@apache.org>
Authored: Fri May 20 12:50:23 2016 -0700
Committer: Chris Nauroth <cnauroth@apache.org>
Committed: Fri May 20 12:50:23 2016 -0700

----------------------------------------------------------------------
 .../hadoop/fs/CommonConfigurationKeys.java      |   3 +
 .../apache/hadoop/hdfs/HDFSPolicyProvider.java  |   6 +-
 .../qjournal/server/JournalNodeRpcServer.java   |   6 +-
 .../hdfs/server/namenode/NameNodeRpcServer.java |   5 +-
 .../hadoop/hdfs/TestHDFSPolicyProvider.java     | 121 +++++++++++++++++++
 5 files changed, 138 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/877af3ec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 63ea8d6..7f510bd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -176,6 +176,9 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic
{
   public static final String
   HADOOP_SECURITY_SERVICE_AUTHORIZATION_TRACING =
       "security.trace.protocol.acl";
+  public static final String
+      HADOOP_SECURITY_SERVICE_AUTHORIZATION_DATANODE_LIFELINE =
+          "security.datanode.lifeline.protocol.acl";
   public static final String 
   SECURITY_HA_SERVICE_PROTOCOL_ACL = "security.ha.service.protocol.acl";
   public static final String 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/877af3ec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java
index 5e53430..8c20553 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.ha.ZKFCProtocol;
 import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocol;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeLifelineProtocol;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
 import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
@@ -76,7 +77,10 @@ public class HDFSPolicyProvider extends PolicyProvider {
         GenericRefreshProtocol.class),
     new Service(
         CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_TRACING,
-        TraceAdminProtocol.class)
+        TraceAdminProtocol.class),
+    new Service(
+        CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DATANODE_LIFELINE,
+        DatanodeLifelineProtocol.class)
   };
   
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/877af3ec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
index 37dd4af..42e6a4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.URL;
 
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -45,9 +46,12 @@ import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RPC.Server;
 import org.apache.hadoop.net.NetUtils;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.BlockingService;
 
-class JournalNodeRpcServer implements QJournalProtocol {
+@InterfaceAudience.Private
+@VisibleForTesting
+public class JournalNodeRpcServer implements QJournalProtocol {
 
   private static final int HANDLER_COUNT = 5;
   private final JournalNode jn;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/877af3ec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index ed9b4e1..cfb470a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -44,6 +44,7 @@ import java.util.concurrent.Callable;
 import com.google.common.collect.Lists;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.CryptoProtocolVersion;
 import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
@@ -203,7 +204,9 @@ import com.google.protobuf.BlockingService;
  * This class is responsible for handling all of the RPC calls to the NameNode.
  * It is created, started, and stopped by {@link NameNode}.
  */
-class NameNodeRpcServer implements NamenodeProtocols {
+@InterfaceAudience.Private
+@VisibleForTesting
+public class NameNodeRpcServer implements NamenodeProtocols {
   
   private static final Logger LOG = NameNode.LOG;
   private static final Logger stateChangeLog = NameNode.stateChangeLog;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/877af3ec/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java
new file mode 100644
index 0000000..a7040f1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSPolicyProvider.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.commons.lang.ClassUtils;
+import org.apache.hadoop.hdfs.qjournal.server.JournalNodeRpcServer;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.security.authorize.Service;
+
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test suite covering HDFSPolicyProvider.  We expect that it contains a
+ * security policy definition for every RPC protocol used in HDFS.  The test
+ * suite works by scanning an RPC server's class to find the protocol interfaces
+ * it implements, and then comparing that to the protocol interfaces covered in
+ * HDFSPolicyProvider.  This is a parameterized test repeated for multiple HDFS
+ * RPC server classes.
+ */
+@RunWith(Parameterized.class)
+public class TestHDFSPolicyProvider {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestHDFSPolicyProvider.class);
+
+  private static List<Class<?>> policyProviderProtocols;
+
+  private static final Comparator<Class<?>> CLASS_NAME_COMPARATOR =
+      new Comparator<Class<?>>() {
+        @Override
+        public int compare(Class<?> lhs, Class<?> rhs) {
+          return lhs.getName().compareTo(rhs.getName());
+        }
+      };
+
+  @Rule
+  public TestName testName = new TestName();
+
+  private final Class<?> rpcServerClass;
+
+  @BeforeClass
+  public static void initialize() {
+    Service[] services = new HDFSPolicyProvider().getServices();
+    policyProviderProtocols = new ArrayList<>(services.length);
+    for (Service service : services) {
+      policyProviderProtocols.add(service.getProtocol());
+    }
+    Collections.sort(policyProviderProtocols, CLASS_NAME_COMPARATOR);
+  }
+
+  public TestHDFSPolicyProvider(Class<?> rpcServerClass) {
+    this.rpcServerClass = rpcServerClass;
+  }
+
+  @Parameters(name = "protocolsForServer-{0}")
+  public static List<Class<?>[]> data() {
+    return Arrays.asList(new Class<?>[][]{
+        {NameNodeRpcServer.class},
+        {DataNode.class},
+        {JournalNodeRpcServer.class}
+    });
+  }
+
+  @Test
+  public void testPolicyProviderForServer() {
+    List<?> ifaces = ClassUtils.getAllInterfaces(rpcServerClass);
+    List<Class<?>> serverProtocols = new ArrayList<>(ifaces.size());
+    for (Object obj : ifaces) {
+      Class<?> iface = (Class<?>)obj;
+      if (iface.getSimpleName().endsWith("Protocol")) {
+        serverProtocols.add(iface);
+      }
+    }
+    Collections.sort(serverProtocols, CLASS_NAME_COMPARATOR);
+    LOG.info("Running test {} for RPC server {}.  Found server protocols {} "
+        + "and policy provider protocols {}.", testName.getMethodName(),
+        rpcServerClass.getName(), serverProtocols, policyProviderProtocols);
+    assertFalse("Expected to find at least one protocol in server.",
+        serverProtocols.isEmpty());
+    assertTrue(
+        String.format("Expected all protocols for server %s to be defined in "
+            + "%s.  Server contains protocols %s.  Policy provider contains "
+            + "protocols %s.", rpcServerClass.getName(),
+            HDFSPolicyProvider.class.getName(), serverProtocols,
+            policyProviderProtocols),
+        policyProviderProtocols.containsAll(serverProtocols));
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message