Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 5FCFB200AE1 for ; Mon, 6 Jun 2016 20:50:55 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 5E3A5160A24; Mon, 6 Jun 2016 18:50:55 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 7FD9D160A1E for ; Mon, 6 Jun 2016 20:50:54 +0200 (CEST) Received: (qmail 19545 invoked by uid 500); 6 Jun 2016 18:50:53 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 19536 invoked by uid 99); 6 Jun 2016 18:50:53 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 06 Jun 2016 18:50:53 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7E6BFDFDEC; Mon, 6 Jun 2016 18:50:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: rzang@apache.org To: commits@ambari.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-17039 - Takes long time to start or fail to start service after enabling SSL due to "dfs.https.enable" (rzang) Date: Mon, 6 Jun 2016 18:50:53 +0000 (UTC) archived-at: Mon, 06 Jun 2016 18:50:55 -0000 Repository: ambari Updated Branches: refs/heads/trunk d6ae8e3d4 -> 66fd5920f AMBARI-17039 - Takes long time to start or fail to start service after enabling SSL due to "dfs.https.enable" (rzang) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/66fd5920 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/66fd5920 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/66fd5920 Branch: refs/heads/trunk Commit: 66fd5920f47f756b469244b67b49fd0e7372a54f Parents: d6ae8e3 Author: Richard Zang Authored: Mon Jun 6 11:48:37 2016 -0700 Committer: Richard Zang Committed: Mon Jun 6 11:48:37 2016 -0700 ---------------------------------------------------------------------- .../libraries/functions/hdfs_utils.py | 33 ++++++++++++++++++++ .../libraries/functions/namenode_ha_utils.py | 4 ++- .../libraries/providers/hdfs_resource.py | 10 ++---- .../2.1.0.2.0/package/scripts/params_linux.py | 6 ++-- 4 files changed, 43 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/66fd5920/ambari-common/src/main/python/resource_management/libraries/functions/hdfs_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hdfs_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/hdfs_utils.py new file mode 100644 index 0000000..06bb65f --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/hdfs_utils.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + + +""" +Check both dfs.http.policy and deprecated dfs.https.enable +""" +def is_https_enabled_in_hdfs(dfs_http_policy, dfs_https_enable): + https_enabled = False + if not is_empty(dfs_http_policy): + https_enabled = dfs_http_policy.lower() == "https_only" + elif not is_empty(dfs_https_enable): + https_enabled = dfs_https_enable + return https_enabled \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/66fd5920/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py index 919ccb5..ca36e62 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py @@ -24,6 +24,8 @@ from resource_management.core.base import Fail from resource_management.core import shell from resource_management.core.logger import Logger from resource_management.libraries.functions.decorator import retry +from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs + __all__ = ["get_namenode_states", "get_active_namenode", "get_property_for_active_namenode", "get_nameservice"] @@ -75,7 +77,7 @@ def get_namenode_states_noretries(hdfs_site, security_enabled, run_user): # ie dfs.namenode.http-address.hacluster.nn1 nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',') for nn_unique_id in nn_unique_ids: - is_https_enabled = is_empty(hdfs_site['dfs.http.policy']) and hdfs_site['dfs.http.policy'].upper() == "HTTPS_ONLY" + is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) rpc_key = NAMENODE_RPC_FRAGMENT.format(name_service,nn_unique_id) if not is_https_enabled: http://git-wip-us.apache.org/repos/asf/ambari/blob/66fd5920/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py index de46c1c..12d9ee4 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py @@ -34,6 +34,8 @@ from resource_management.libraries.functions import format from resource_management.libraries.functions.get_user_call_output import get_user_call_output from resource_management.libraries.functions import is_empty from resource_management.libraries.functions import namenode_ha_utils +from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs + import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. import subprocess @@ -133,13 +135,7 @@ class WebHDFSUtil: security_enabled, run_user) http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address', security_enabled, run_user) - - # check for dfs.http.policy and after that for deprecated(for newer stacks) dfs.https.enable - self.is_https_enabled = False - if not is_empty(hdfs_site['dfs.http.policy']): - self.is_https_enabled = hdfs_site['dfs.http.policy'].lower() == "https_only" - elif not is_empty(hdfs_site['dfs.https.enable']): - self.is_https_enabled = hdfs_site['dfs.https.enable'] + self.is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) address = https_nn_address if self.is_https_enabled else http_nn_address protocol = "https" if self.is_https_enabled else "http" http://git-wip-us.apache.org/repos/asf/ambari/blob/66fd5920/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index b634ce9..9af87d4 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -43,6 +43,8 @@ from resource_management.libraries.functions.format_jvm_option import format_jvm from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.functions.expect import expect +from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs + config = Script.get_config() @@ -420,8 +422,8 @@ policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_ jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' -is_https_enabled = config['configurations']['hdfs-site']['dfs.https.enable'] if \ - not is_empty(config['configurations']['hdfs-site']['dfs.https.enable']) else False +is_https_enabled = is_https_enabled_in_hdfs(config['configurations']['hdfs-site']['dfs.http.policy'], + config['configurations']['hdfs-site']['dfs.https.enable']) if has_ranger_admin: enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes')