Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id DE4B9200D2E for ; Tue, 31 Oct 2017 22:36:52 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id DCD211609EF; Tue, 31 Oct 2017 21:36:52 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D5AC01609E6 for ; Tue, 31 Oct 2017 22:36:51 +0100 (CET) Received: (qmail 84008 invoked by uid 500); 31 Oct 2017 21:36:51 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 83999 invoked by uid 99); 31 Oct 2017 21:36:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 31 Oct 2017 21:36:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DB3C3DFC25; Tue, 31 Oct 2017 21:36:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xyao@apache.org To: common-commits@hadoop.apache.org Message-Id: <07ee1d19733645099bfe652acdaea9cb@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-12697. Ozone services must stay disabled in secure setup for alpha. Contributed by Bharat Viswanadham. Date: Tue, 31 Oct 2017 21:36:50 +0000 (UTC) archived-at: Tue, 31 Oct 2017 21:36:53 -0000 Repository: hadoop Updated Branches: refs/heads/HDFS-7240 6739180c1 -> be6bf1b80 HDFS-12697. Ozone services must stay disabled in secure setup for alpha. Contributed by Bharat Viswanadham. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be6bf1b8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be6bf1b8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be6bf1b8 Branch: refs/heads/HDFS-7240 Commit: be6bf1b80f71a57a37fa9946177ef863930bedb5 Parents: 6739180 Author: Xiaoyu Yao Authored: Tue Oct 31 14:32:36 2017 -0700 Committer: Xiaoyu Yao Committed: Tue Oct 31 14:32:36 2017 -0700 ---------------------------------------------------------------------- .../hadoop-hdfs/src/main/bin/start-ozone.sh | 8 +++++ .../hadoop-hdfs/src/main/bin/stop-ozone.sh | 8 +++++ .../java/org/apache/hadoop/hdfs/DFSUtil.java | 20 ++++++++++++ .../hadoop/hdfs/server/datanode/DataNode.java | 6 ++-- .../hdfs/server/datanode/web/URLDispatcher.java | 33 ++++++++++++++------ .../hadoop/ozone/ksm/KeySpaceManager.java | 9 +++++- .../ozone/scm/StorageContainerManager.java | 10 ++++-- 7 files changed, 78 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/be6bf1b8/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh index 5291f57..ca6b6b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh @@ -44,6 +44,14 @@ else exit 1 fi +SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-) +SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-) + +if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then + echo "Ozone is not supported in a security enabled cluster." + exit 1 +fi + #--------------------------------------------------------- # Check if ozone is enabled OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-) http://git-wip-us.apache.org/repos/asf/hadoop/blob/be6bf1b8/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh index 216492b..15255c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh @@ -44,6 +44,14 @@ else exit 1 fi +SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-) +SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-) + +if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then + echo "Ozone is not supported in a security enabled cluster." + exit 1 +fi + #--------------------------------------------------------- # Check if ozone is enabled OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-) http://git-wip-us.apache.org/repos/asf/hadoop/blob/be6bf1b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 2f9781a..16fd401 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -36,6 +36,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED_DEFAULT; import java.io.ByteArrayInputStream; import java.io.DataInputStream; @@ -71,6 +73,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -1526,6 +1529,23 @@ public class DFSUtil { return password; } + public static boolean isOzoneEnabled(Configuration conf) { + String securityEnabled = conf.get(CommonConfigurationKeysPublic + .HADOOP_SECURITY_AUTHENTICATION, + "simple"); + boolean securityAuthorizationEnabled = conf.getBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, + false); + + if (securityEnabled.equals("kerberos") || securityAuthorizationEnabled) { + LOG.error("Ozone is not supported in a security enabled cluster. "); + return false; + } else { + return conf.getBoolean(OZONE_ENABLED, + OZONE_ENABLED_DEFAULT); + } + } + /** * Converts a Date into an ISO-8601 formatted datetime string. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/be6bf1b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 23c9c04..13fa4a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -46,8 +46,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_DEF import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_KEY; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED_DEFAULT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; import static org.apache.hadoop.util.ExitUtil.terminate; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -475,8 +473,8 @@ public class DataNode extends ReconfigurableBase this.pipelineSupportECN = conf.getBoolean( DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED, DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED_DEFAULT); - this.ozoneEnabled = conf.getBoolean(OZONE_ENABLED, - OZONE_ENABLED_DEFAULT); + + this.ozoneEnabled = DFSUtil.isOzoneEnabled(conf); confVersion = "core-" + conf.get("hadoop.common.configuration.version", "UNSPECIFIED") + http://git-wip-us.apache.org/repos/asf/hadoop/blob/be6bf1b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java index 5114298..dd958d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java @@ -68,33 +68,48 @@ class URLDispatcher extends SimpleChannelInboundHandler { p.replace(this, RequestDispatchObjectStoreChannelHandler.class.getSimpleName(), h); h.channelRead0(ctx, req); - } else { + } else if (!isObjectStoreRequestHeaders(req)){ SimpleHttpProxyHandler h = new SimpleHttpProxyHandler(proxyHost); p.replace(this, SimpleHttpProxyHandler.class.getSimpleName(), h); h.channelRead0(ctx, req); } } + /* - * Returns true if the request is to be handled by the object store. + * Returns true if the request has ozone headers * * @param req HTTP request - * @return true if the request is to be handled by the object store + * @return true if request has ozone header, else false */ - private boolean isObjectStoreRequest(HttpRequest req) { - if (this.objectStoreJerseyContainer == null) { - LOG.debug("ozone : dispatching call to webHDFS"); - return false; - } + + private boolean isObjectStoreRequestHeaders(HttpRequest req) { for (String version : req.headers().getAll(Header.OZONE_VERSION_HEADER)) { if (version != null) { - LOG.debug("ozone : dispatching call to Ozone"); + LOG.debug("ozone : dispatching call to Ozone, when security is not " + + "enabled"); return true; } } return false; } + + /* + * Returns true if the request is to be handled by the object store. + * + * @param req HTTP request + * @return true if the request is to be handled by the object store + */ + private boolean isObjectStoreRequest(HttpRequest req) { + if (this.objectStoreJerseyContainer == null) { + LOG.debug("ozone : ozone is disabled or when security is enabled, ozone" + + " is not supported"); + return false; + } + return isObjectStoreRequestHeaders(req); + } + /** * Returns true if the request is to be handled by WebHDFS. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/be6bf1b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java index 22ad4b8..a48258e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java @@ -57,6 +57,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; import static org.apache.hadoop.ozone.ksm.KSMConfigKeys .OZONE_KSM_ADDRESS_KEY; import static org.apache.hadoop.ozone.ksm.KSMConfigKeys @@ -186,7 +187,13 @@ public class KeySpaceManager extends ServiceRuntimeInfoImpl public static void main(String[] argv) throws IOException { StringUtils.startupShutdownMessage(KeySpaceManager.class, argv, LOG); try { - KeySpaceManager ksm = new KeySpaceManager(new OzoneConfiguration()); + OzoneConfiguration conf = new OzoneConfiguration(); + if (!DFSUtil.isOzoneEnabled(conf)) { + System.out.println("KSM cannot be started in secure mode or when " + + OZONE_ENABLED + " is set to false"); + System.exit(1); + } + KeySpaceManager ksm = new KeySpaceManager(conf); ksm.start(); ksm.join(); } catch (Throwable t) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/be6bf1b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java index 84e4386..d341e2c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java @@ -104,6 +104,7 @@ import java.util.UUID; import java.util.Collections; import java.util.stream.Collectors; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; import static org.apache.hadoop.ozone.protocol.proto .ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result; import static org.apache.hadoop.scm.ScmConfigKeys @@ -321,8 +322,13 @@ public class StorageContainerManager extends ServiceRuntimeInfoImpl StringUtils.startupShutdownMessage(StorageContainerManager.class, argv, LOG); try { - StorageContainerManager scm = new StorageContainerManager( - new OzoneConfiguration()); + OzoneConfiguration conf = new OzoneConfiguration(); + if (!DFSUtil.isOzoneEnabled(conf)) { + System.out.println("SCM cannot be started in secure mode or when " + + OZONE_ENABLED + " is set to false"); + System.exit(1); + } + StorageContainerManager scm = new StorageContainerManager(conf); scm.start(); scm.join(); } catch (Throwable t) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org