hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jzh...@apache.org
Subject hadoop git commit: HDFS-11418. HttpFS should support old SSL clients. Contributed by John Zhuge.
Date Thu, 02 Mar 2017 03:59:30 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 133857ba0 -> cacaa299c


HDFS-11418. HttpFS should support old SSL clients. Contributed by John Zhuge.

Change-Id: I05b1228f7f168e9b5904365ba9e81be3f29dce74


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cacaa299
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cacaa299
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cacaa299

Branch: refs/heads/branch-2
Commit: cacaa299cf8b5e90d6993e4bbd718ccf7b21b6e5
Parents: 133857b
Author: John Zhuge <jzhuge@cloudera.com>
Authored: Wed Feb 15 11:08:57 2017 -0800
Committer: John Zhuge <john.zhuge@gmail.com>
Committed: Wed Mar 1 19:45:25 2017 -0800

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml  |  3 +
 .../src/main/conf/httpfs-env.sh                 |  4 +
 .../src/main/libexec/httpfs-config.sh           | 35 +++++++-
 .../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh  | 47 ++++++++---
 .../src/main/tomcat/catalina-default.properties | 87 ++++++++++++++++++++
 .../src/main/tomcat/ssl-server.xml              |  1 +
 .../src/site/markdown/ServerSetup.md.vm         |  6 +-
 7 files changed, 168 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cacaa299/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index df1e5ed..a064ed2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -526,6 +526,9 @@
                     <delete dir="${project.build.directory}/tomcat.exp"/>
                     <delete dir="${httpfs.tomcat.dist.dir}/webapps"/>
                     <mkdir dir="${httpfs.tomcat.dist.dir}/webapps"/>
+                    <delete file="${httpfs.tomcat.dist.dir}/conf/catalina-default.properties"/>
+                    <copy file="${basedir}/src/main/tomcat/catalina-default.properties"
+                          toDir="${httpfs.tomcat.dist.dir}/conf"/>
                     <delete file="${httpfs.tomcat.dist.dir}/conf/server.xml"/>
                     <copy file="${basedir}/src/main/tomcat/server.xml"
                           toDir="${httpfs.tomcat.dist.dir}/conf"/>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cacaa299/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
index f19f706..48be812 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
@@ -44,6 +44,10 @@
 #
 # export HTTPFS_SSL_ENABLED=false
 
+# The comma separated list of encryption ciphers for SSL
+#
+# export HTTPFS_SSL_CIPHERS=
+
 # The maximum size of Tomcat HTTP header
 #
 # export HTTPFS_MAX_HTTP_HEADER_SIZE=65536

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cacaa299/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
index 5ecb61a..7e47cb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
@@ -56,7 +56,10 @@ print "Setting HTTPFS_HOME:          ${HTTPFS_HOME}"
 if [ -e "${HTTPFS_HOME}/bin/httpfs-env.sh" ]; then
   print "Sourcing:                    ${HTTPFS_HOME}/bin/httpfs-env.sh"
   source ${HTTPFS_HOME}/bin/httpfs-env.sh
-  grep "^ *export " ${HTTPFS_HOME}/bin/httpfs-env.sh | sed 's/ *export/  setting/'
+  if [ "${HTTPFS_SILENT}" != "true" ]; then
+    grep "^ *export " "${HTTPFS_HOME}/bin/httpfs-env.sh" |
+      sed 's/ *export/  setting/'
+  fi
 fi
 
 # verify that the sourced env file didn't change HTTPFS_HOME
@@ -81,7 +84,10 @@ httpfs_config=${HTTPFS_CONFIG}
 if [ -e "${HTTPFS_CONFIG}/httpfs-env.sh" ]; then
   print "Sourcing:                    ${HTTPFS_CONFIG}/httpfs-env.sh"
   source ${HTTPFS_CONFIG}/httpfs-env.sh
-  grep "^ *export " ${HTTPFS_CONFIG}/httpfs-env.sh | sed 's/ *export/  setting/'
+  if [ "${HTTPFS_SILENT}" != "true" ]; then
+    grep "^ *export " "${HTTPFS_CONFIG}/httpfs-env.sh" |
+      sed 's/ *export/  setting/'
+  fi
 fi
 
 # verify that the sourced env file didn't change HTTPFS_HOME
@@ -150,6 +156,31 @@ else
   print "Using   HTTPFS_SSL_ENABLED: ${HTTPFS_SSL_ENABLED}"
 fi
 
+if [ "${HTTPFS_SSL_CIPHERS}" = "" ]; then
+  export HTTPFS_SSL_CIPHERS="TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"
+  HTTPFS_SSL_CIPHERS+=",TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"
+  HTTPFS_SSL_CIPHERS+=",TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384"
+  HTTPFS_SSL_CIPHERS+=",TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256"
+  HTTPFS_SSL_CIPHERS+=",TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384"
+  HTTPFS_SSL_CIPHERS+=",TLS_ECDH_RSA_WITH_AES_256_CBC_SHA"
+  HTTPFS_SSL_CIPHERS+=",TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256"
+  HTTPFS_SSL_CIPHERS+=",TLS_ECDH_RSA_WITH_AES_128_CBC_SHA"
+  HTTPFS_SSL_CIPHERS+=",TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA"
+  HTTPFS_SSL_CIPHERS+=",TLS_RSA_WITH_AES_256_CBC_SHA256"
+  HTTPFS_SSL_CIPHERS+=",TLS_RSA_WITH_AES_256_CBC_SHA"
+  HTTPFS_SSL_CIPHERS+=",TLS_RSA_WITH_AES_128_CBC_SHA256"
+  HTTPFS_SSL_CIPHERS+=",TLS_RSA_WITH_AES_128_CBC_SHA"
+  HTTPFS_SSL_CIPHERS+=",TLS_RSA_WITH_3DES_EDE_CBC_SHA"
+  HTTPFS_SSL_CIPHERS+=",TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"
+  HTTPFS_SSL_CIPHERS+=",TLS_DHE_RSA_WITH_AES_256_CBC_SHA"
+  HTTPFS_SSL_CIPHERS+=",TLS_DHE_RSA_WITH_AES_128_CBC_SHA256"
+  HTTPFS_SSL_CIPHERS+=",TLS_DHE_RSA_WITH_AES_128_CBC_SHA"
+  HTTPFS_SSL_CIPHERS+=",TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA"
+  print "Setting HTTPFS_SSL_CIPHERS: ${HTTPFS_SSL_CIPHERS}"
+else
+  print "Using   HTTPFS_SSL_CIPHERS: ${HTTPFS_SSL_CIPHERS}"
+fi
+
 if [ "${HTTPFS_SSL_KEYSTORE_FILE}" = "" ]; then
   export HTTPFS_SSL_KEYSTORE_FILE=${HOME}/.keystore
   print "Setting HTTPFS_SSL_KEYSTORE_FILE:     ${HTTPFS_SSL_KEYSTORE_FILE}"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cacaa299/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
index 203024e..e6b1f35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
@@ -29,6 +29,8 @@ done
 BASEDIR=`dirname ${PRG}`
 BASEDIR=`cd ${BASEDIR}/..;pwd`
 
+HTTPFS_SILENT=${HTTPFS_SILENT:-true}
+
 source ${HADOOP_LIBEXEC_DIR:-${BASEDIR}/libexec}/httpfs-config.sh
 
 # The Java System property 'httpfs.http.port' it is not used by HttpFS,
@@ -37,21 +39,42 @@ source ${HADOOP_LIBEXEC_DIR:-${BASEDIR}/libexec}/httpfs-config.sh
 print "Using   CATALINA_OPTS:       ${CATALINA_OPTS}"
 
 catalina_opts="-Dproc_httpfs";
-catalina_opts="${catalina_opts} -Dhttpfs.home.dir=${HTTPFS_HOME}";
-catalina_opts="${catalina_opts} -Dhttpfs.config.dir=${HTTPFS_CONFIG}";
-catalina_opts="${catalina_opts} -Dhttpfs.log.dir=${HTTPFS_LOG}";
-catalina_opts="${catalina_opts} -Dhttpfs.temp.dir=${HTTPFS_TEMP}";
-catalina_opts="${catalina_opts} -Dhttpfs.admin.port=${HTTPFS_ADMIN_PORT}";
-catalina_opts="${catalina_opts} -Dhttpfs.http.port=${HTTPFS_HTTP_PORT}";
-catalina_opts="${catalina_opts} -Dhttpfs.http.hostname=${HTTPFS_HTTP_HOSTNAME}";
-catalina_opts="${catalina_opts} -Dhttpfs.ssl.enabled=${HTTPFS_SSL_ENABLED}";
-catalina_opts="${catalina_opts} -Dhttpfs.ssl.keystore.file=${HTTPFS_SSL_KEYSTORE_FILE}";
-catalina_opts="${catalina_opts} -Dhttpfs.ssl.keystore.pass=${HTTPFS_SSL_KEYSTORE_PASS}";
+catalina_opts="${catalina_opts} -Dhttpfs.log.dir=${HTTPFS_LOG}"
 
 print "Adding to CATALINA_OPTS:     ${catalina_opts}"
 
 export CATALINA_OPTS="${CATALINA_OPTS} ${catalina_opts}"
 
+catalina_init_properties() {
+  cp "${CATALINA_BASE}/conf/catalina-default.properties" \
+    "${CATALINA_BASE}/conf/catalina.properties"
+}
+
+catalina_set_property() {
+  local key=$1
+  local value=$2
+  [[ -z "${value}" ]] && return
+  local disp_value="${3:-${value}}"
+  print "Setting catalina property ${key} to ${disp_value}"
+  echo "${key}=${value}" >> "${CATALINA_BASE}/conf/catalina.properties"
+}
+
+if [[ "${1}" = "start" || "${1}" = "run" ]]; then
+  catalina_init_properties
+  catalina_set_property "httpfs.home.dir" "${HTTPFS_HOME}"
+  catalina_set_property "httpfs.config.dir" "${HTTPFS_CONFIG}"
+  catalina_set_property "httpfs.temp.dir" "${HTTPFS_TEMP}"
+  catalina_set_property "httpfs.admin.port" "${HTTPFS_ADMIN_PORT}"
+  catalina_set_property "httpfs.http.port" "${HTTPFS_HTTP_PORT}"
+  catalina_set_property "httpfs.http.hostname" "${HTTPFS_HTTP_HOSTNAME}"
+  catalina_set_property "httpfs.ssl.enabled" "${HTTPFS_SSL_ENABLED}"
+  catalina_set_property "httpfs.ssl.ciphers" "${HTTPFS_SSL_CIPHERS}"
+  catalina_set_property "httpfs.ssl.keystore.file" \
+    "${HTTPFS_SSL_KEYSTORE_FILE}"
+  catalina_set_property "httpfs.ssl.keystore.pass" \
+    "${HTTPFS_SSL_KEYSTORE_PASS}" "<redacted>"
+fi
+
 # A bug in catalina.sh script does not use CATALINA_OPTS for stopping the server
 #
 if [ "${1}" = "stop" ]; then
@@ -59,8 +82,8 @@ if [ "${1}" = "stop" ]; then
 fi
 
 if [ "${HTTPFS_SILENT}" != "true" ]; then
-  exec ${HTTPFS_CATALINA_HOME}/bin/catalina.sh "$@"
+  exec "${HTTPFS_CATALINA_HOME}/bin/catalina.sh" "$@"
 else
-  exec ${HTTPFS_CATALINA_HOME}/bin/catalina.sh "$@" > /dev/null
+  exec "${HTTPFS_CATALINA_HOME}/bin/catalina.sh" "$@" > /dev/null
 fi
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cacaa299/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/catalina-default.properties
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/catalina-default.properties
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/catalina-default.properties
new file mode 100644
index 0000000..463b3df
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/catalina-default.properties
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# List of comma-separated packages that start with or equal this string
+# will cause a security exception to be thrown when
+# passed to checkPackageAccess unless the
+# corresponding RuntimePermission ("accessClassInPackage."+package) has
+# been granted.
+package.access=sun.,org.apache.catalina.,org.apache.coyote.,org.apache.jasper.,\
+org.apache.naming.resources.,org.apache.tomcat.,sun.beans.
+#
+# List of comma-separated packages that start with or equal this string
+# will cause a security exception to be thrown when
+# passed to checkPackageDefinition unless the
+# corresponding RuntimePermission ("defineClassInPackage."+package) has
+# been granted.
+#
+# by default, no packages are restricted for definition, and none of
+# the class loaders supplied with the JDK call checkPackageDefinition.
+#
+package.definition=sun.,java.,org.apache.catalina.,org.apache.coyote.,\
+org.apache.jasper.,org.apache.naming.,org.apache.tomcat.
+
+#
+#
+# List of comma-separated paths defining the contents of the "common"
+# classloader. Prefixes should be used to define what is the repository type.
+# Path may be relative to the CATALINA_HOME or CATALINA_BASE path or absolute.
+# If left as blank,the JVM system loader will be used as Catalina's "common"
+# loader.
+# Examples:
+#     "foo": Add this folder as a class repository
+#     "foo/*.jar": Add all the JARs of the specified folder as class
+#                  repositories
+#     "foo/bar.jar": Add bar.jar as a class repository
+common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar
+
+#
+# List of comma-separated paths defining the contents of the "server"
+# classloader. Prefixes should be used to define what is the repository type.
+# Path may be relative to the CATALINA_HOME or CATALINA_BASE path or absolute.
+# If left as blank, the "common" loader will be used as Catalina's "server"
+# loader.
+# Examples:
+#     "foo": Add this folder as a class repository
+#     "foo/*.jar": Add all the JARs of the specified folder as class
+#                  repositories
+#     "foo/bar.jar": Add bar.jar as a class repository
+server.loader=
+
+#
+# List of comma-separated paths defining the contents of the "shared"
+# classloader. Prefixes should be used to define what is the repository type.
+# Path may be relative to the CATALINA_BASE path or absolute. If left as blank,
+# the "common" loader will be used as Catalina's "shared" loader.
+# Examples:
+#     "foo": Add this folder as a class repository
+#     "foo/*.jar": Add all the JARs of the specified folder as class
+#                  repositories
+#     "foo/bar.jar": Add bar.jar as a class repository
+# Please note that for single jars, e.g. bar.jar, you need the URL form
+# starting with file:.
+shared.loader=
+
+#
+# String cache configuration.
+tomcat.util.buf.StringCache.byte.enabled=true
+#tomcat.util.buf.StringCache.char.enabled=true
+#tomcat.util.buf.StringCache.trainThreshold=500000
+#tomcat.util.buf.StringCache.cacheSize=5000
+
+#
+# Copy catalina-default.properties to catalina.properties
+# and then append the custom properties.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cacaa299/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
index 92bee15..021accf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
@@ -72,6 +72,7 @@
                maxThreads="150" scheme="https" secure="true"
                maxHttpHeaderSize="${httpfs.max.http.header.size}"
                clientAuth="false" sslEnabledProtocols="TLSv1,TLSv1.1,TLSv1.2,SSLv2Hello"
+               ciphers="${httpfs.ssl.ciphers}"
                keystoreFile="${httpfs.ssl.keystore.file}"
                keystorePass="${httpfs.ssl.keystore.pass}"/>
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cacaa299/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
index 4b66732..671ef2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
@@ -120,4 +120,8 @@ Start HttpFS. It should work over HTTPS.
 
 Using the Hadoop `FileSystem` API or the Hadoop FS shell, use the `swebhdfs://` scheme. Make
sure the JVM is picking up the truststore containing the public key of the SSL certificate
if using a self-signed certificate.
 
-NOTE: Some old SSL clients may use weak ciphers that are not supported by the HttpFS server.
It is recommended to upgrade the SSL client.
+In order to support some old SSL clients, the default encryption ciphers
+include a few relatively weaker ciphers. Set environment variable
+`HTTPFS_SSL_CIPHERS` or property `httpfs.ssl.ciphers` to override. The value
+is a comma separated list of ciphers documented in this
+[Tomcat Wiki](https://wiki.apache.org/tomcat/Security/Ciphers).
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message