Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 8EFB4200D14 for ; Tue, 3 Oct 2017 14:45:46 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 8D745160BDB; Tue, 3 Oct 2017 12:45:46 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 41DED1609DD for ; Tue, 3 Oct 2017 14:45:44 +0200 (CEST) Received: (qmail 9884 invoked by uid 500); 3 Oct 2017 12:45:43 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 9866 invoked by uid 99); 3 Oct 2017 12:45:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 03 Oct 2017 12:45:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7B42CE0395; Tue, 3 Oct 2017 12:45:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dsen@apache.org To: commits@ambari.apache.org Date: Tue, 03 Oct 2017 12:45:43 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [3/5] ambari git commit: AMBARI-22095 Make hooks stack agnostic (dsen) archived-at: Tue, 03 Oct 2017 12:45:46 -0000 http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py deleted file mode 100644 index 20992e2..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py +++ /dev/null @@ -1,254 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import collections -import re -import os -import ast - -import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. - -from resource_management.libraries.script import Script -from resource_management.libraries.functions import default -from resource_management.libraries.functions import format -from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import format_jvm_option -from resource_management.libraries.functions.is_empty import is_empty -from resource_management.libraries.functions.version import format_stack_version -from resource_management.libraries.functions.expect import expect -from resource_management.libraries.functions import StackFeature -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.stack_features import get_stack_feature_version -from resource_management.libraries.functions.get_architecture import get_architecture -from ambari_commons.constants import AMBARI_SUDO_BINARY - - -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() - -stack_root = Script.get_stack_root() - -architecture = get_architecture() - -dfs_type = default("/commandParams/dfs_type", "") - -artifact_dir = format("{tmp_dir}/AMBARI-artifacts/") -jdk_name = default("/hostLevelParams/jdk_name", None) -java_home = config['hostLevelParams']['java_home'] -java_version = expect("/hostLevelParams/java_version", int) -jdk_location = config['hostLevelParams']['jdk_location'] - -hadoop_custom_extensions_enabled = default("/configurations/core-site/hadoop.custom-extensions.enabled", False) - -sudo = AMBARI_SUDO_BINARY - -ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] - -stack_version_unformatted = config['hostLevelParams']['stack_version'] -stack_version_formatted = format_stack_version(stack_version_unformatted) - -upgrade_type = Script.get_upgrade_type(default("/commandParams/upgrade_type", "")) -version = default("/commandParams/version", None) -# Handle upgrade and downgrade -if (upgrade_type is not None) and version: - stack_version_formatted = format_stack_version(version) -ambari_java_home = default("/commandParams/ambari_java_home", None) -ambari_jdk_name = default("/commandParams/ambari_jdk_name", None) - -security_enabled = config['configurations']['cluster-env']['security_enabled'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] - -# Some datanode settings -dfs_dn_addr = default('/configurations/hdfs-site/dfs.datanode.address', None) -dfs_dn_http_addr = default('/configurations/hdfs-site/dfs.datanode.http.address', None) -dfs_dn_https_addr = default('/configurations/hdfs-site/dfs.datanode.https.address', None) -dfs_http_policy = default('/configurations/hdfs-site/dfs.http.policy', None) -secure_dn_ports_are_in_use = False - -def get_port(address): - """ - Extracts port from the address like 0.0.0.0:1019 - """ - if address is None: - return None - m = re.search(r'(?:http(?:s)?://)?([\w\d.]*):(\d{1,5})', address) - if m is not None: - return int(m.group(2)) - else: - return None - -def is_secure_port(port): - """ - Returns True if port is root-owned at *nix systems - """ - if port is not None: - return port < 1024 - else: - return False - -# upgrades would cause these directories to have a version instead of "current" -# which would cause a lot of problems when writing out hadoop-env.sh; instead -# force the use of "current" in the hook -hdfs_user_nofile_limit = default("/configurations/hadoop-env/hdfs_user_nofile_limit", "128000") -hadoop_home = stack_select.get_hadoop_dir("home") -hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") -hadoop_lib_home = stack_select.get_hadoop_dir("lib") - -hadoop_dir = "/etc/hadoop" -hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir") -datanode_max_locked_memory = config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory'] -is_datanode_max_locked_memory_set = not is_empty(config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory']) - -mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*" - -if not security_enabled: - hadoop_secure_dn_user = '""' -else: - dfs_dn_port = get_port(dfs_dn_addr) - dfs_dn_http_port = get_port(dfs_dn_http_addr) - dfs_dn_https_port = get_port(dfs_dn_https_addr) - # We try to avoid inability to start datanode as a plain user due to usage of root-owned ports - if dfs_http_policy == "HTTPS_ONLY": - secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_https_port) - elif dfs_http_policy == "HTTP_AND_HTTPS": - secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port) or is_secure_port(dfs_dn_https_port) - else: # params.dfs_http_policy == "HTTP_ONLY" or not defined: - secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port) - if secure_dn_ports_are_in_use: - hadoop_secure_dn_user = hdfs_user - else: - hadoop_secure_dn_user = '""' - -#hadoop params -hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] -hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] -hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger'] - -jsvc_path = "/usr/lib/bigtop-utils" - -hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] -namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] -namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize'] -namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize'] -namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m") -namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m") - -jtnode_opt_newsize = "200m" -jtnode_opt_maxnewsize = "200m" -jtnode_heapsize = "1024m" -ttnode_heapsize = "1024m" - -dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize'] -nfsgateway_heapsize = config['configurations']['hadoop-env']['nfsgateway_heapsize'] -mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce") -mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce") -hadoop_env_sh_template = config['configurations']['hadoop-env']['content'] - -#users and groups -hbase_user = config['configurations']['hbase-env']['hbase_user'] -smoke_user = config['configurations']['cluster-env']['smokeuser'] -gmetad_user = config['configurations']['ganglia-env']["gmetad_user"] -gmond_user = config['configurations']['ganglia-env']["gmond_user"] -tez_user = config['configurations']['tez-env']["tez_user"] -oozie_user = config['configurations']['oozie-env']["oozie_user"] -falcon_user = config['configurations']['falcon-env']["falcon_user"] -ranger_user = config['configurations']['ranger-env']["ranger_user"] -zeppelin_user = config['configurations']['zeppelin-env']["zeppelin_user"] -zeppelin_group = config['configurations']['zeppelin-env']["zeppelin_group"] - -user_group = config['configurations']['cluster-env']['user_group'] - -ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) -namenode_host = default("/clusterHostInfo/namenode_host", []) -hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", []) -oozie_servers = default("/clusterHostInfo/oozie_server", []) -falcon_server_hosts = default("/clusterHostInfo/falcon_server_hosts", []) -ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) -zeppelin_master_hosts = default("/clusterHostInfo/zeppelin_master_hosts", []) - -# get the correct version to use for checking stack features -version_for_stack_feature_checks = get_stack_feature_version(config) - - -has_namenode = not len(namenode_host) == 0 -has_ganglia_server = not len(ganglia_server_hosts) == 0 -has_tez = 'tez-site' in config['configurations'] -has_hbase_masters = not len(hbase_master_hosts) == 0 -has_oozie_server = not len(oozie_servers) == 0 -has_falcon_server_hosts = not len(falcon_server_hosts) == 0 -has_ranger_admin = not len(ranger_admin_hosts) == 0 -has_zeppelin_master = not len(zeppelin_master_hosts) == 0 -stack_supports_zk_security = check_stack_feature(StackFeature.SECURE_ZOOKEEPER, version_for_stack_feature_checks) - -# HDFS High Availability properties -dfs_ha_enabled = False -dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.internal.nameservices', None) -if dfs_ha_nameservices is None: - dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.nameservices', None) -dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None) -if dfs_ha_namenode_ids: - dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",") - dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list) - if dfs_ha_namenode_ids_array_len > 1: - dfs_ha_enabled = True - -if has_namenode or dfs_type == 'HCFS': - hadoop_conf_dir = conf_select.get_hadoop_conf_dir() - hadoop_conf_secure_dir = os.path.join(hadoop_conf_dir, "secure") - -hbase_tmp_dir = "/tmp/hbase-hbase" - -proxyuser_group = default("/configurations/hadoop-env/proxyuser_group","users") -ranger_group = config['configurations']['ranger-env']['ranger_group'] -dfs_cluster_administrators_group = config['configurations']['hdfs-site']["dfs.cluster.administrators"] - -sysprep_skip_create_users_and_groups = default("/configurations/cluster-env/sysprep_skip_create_users_and_groups", False) -ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False) -fetch_nonlocal_groups = config['configurations']['cluster-env']["fetch_nonlocal_groups"] - -smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}") -if has_hbase_masters: - hbase_user_dirs = format("/home/{hbase_user},/tmp/{hbase_user},/usr/bin/{hbase_user},/var/log/{hbase_user},{hbase_tmp_dir}") -#repo params -repo_info = config['hostLevelParams']['repo_info'] -service_repo_info = default("/hostLevelParams/service_repo_info",None) - -user_to_groups_dict = {} - -#Append new user-group mapping to the dict -try: - user_group_map = ast.literal_eval(config['hostLevelParams']['user_groups']) - for key in user_group_map.iterkeys(): - user_to_groups_dict[key] = user_group_map[key] -except ValueError: - print('User Group mapping (user_group) is missing in the hostLevelParams') - -user_to_gid_dict = collections.defaultdict(lambda:user_group) - -user_list = json.loads(config['hostLevelParams']['user_list']) -group_list = json.loads(config['hostLevelParams']['group_list']) -host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) - -tez_am_view_acls = config['configurations']['tez-site']["tez.am.view-acls"] -override_uid = str(default("/configurations/cluster-env/override_uid", "true")).lower() - -# if NN HA on secure clutser, access Zookeper securely -if stack_supports_zk_security and dfs_ha_enabled and security_enabled: - hadoop_zkfc_opts=format("-Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config={hadoop_conf_secure_dir}/hdfs_jaas.conf -Dzookeeper.sasl.clientconfig=Client") http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py deleted file mode 100644 index 27679e0..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py +++ /dev/null @@ -1,273 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import os -import re -import getpass -import tempfile -from copy import copy -from resource_management.libraries.functions.version import compare_versions -from resource_management import * -from resource_management.core import shell - -def setup_users(): - """ - Creates users before cluster installation - """ - import params - - should_create_users_and_groups = False - if params.host_sys_prepped: - should_create_users_and_groups = not params.sysprep_skip_create_users_and_groups - else: - should_create_users_and_groups = not params.ignore_groupsusers_create - - if should_create_users_and_groups: - for group in params.group_list: - Group(group, - ) - - for user in params.user_list: - User(user, - uid = get_uid(user) if params.override_uid == "true" else None, - gid = params.user_to_gid_dict[user], - groups = params.user_to_groups_dict[user], - fetch_nonlocal_groups = params.fetch_nonlocal_groups, - ) - - if params.override_uid == "true": - set_uid(params.smoke_user, params.smoke_user_dirs) - else: - Logger.info('Skipping setting uid for smoke user as host is sys prepped') - else: - Logger.info('Skipping creation of User and Group as host is sys prepped or ignore_groupsusers_create flag is on') - pass - - - if params.has_hbase_masters: - Directory (params.hbase_tmp_dir, - owner = params.hbase_user, - mode=0775, - create_parents = True, - cd_access="a", - ) - - if params.override_uid == "true": - set_uid(params.hbase_user, params.hbase_user_dirs) - else: - Logger.info('Skipping setting uid for hbase user as host is sys prepped') - - if should_create_users_and_groups: - if params.has_namenode: - create_dfs_cluster_admins() - if params.has_tez and params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.3') >= 0: - create_tez_am_view_acls() - else: - Logger.info('Skipping setting dfs cluster admin and tez view acls as host is sys prepped') - -def create_dfs_cluster_admins(): - """ - dfs.cluster.administrators support format - """ - import params - - groups_list = create_users_and_groups(params.dfs_cluster_administrators_group) - - User(params.hdfs_user, - groups = params.user_to_groups_dict[params.hdfs_user] + groups_list, - fetch_nonlocal_groups = params.fetch_nonlocal_groups - ) - -def create_tez_am_view_acls(): - - """ - tez.am.view-acls support format - """ - import params - - if not params.tez_am_view_acls.startswith("*"): - create_users_and_groups(params.tez_am_view_acls) - -def create_users_and_groups(user_and_groups): - - import params - - parts = re.split('\s+', user_and_groups) - if len(parts) == 1: - parts.append("") - - users_list = parts[0].strip(",").split(",") if parts[0] else [] - groups_list = parts[1].strip(",").split(",") if parts[1] else [] - - # skip creating groups and users if * is provided as value. - users_list = filter(lambda x: x != '*' , users_list) - groups_list = filter(lambda x: x != '*' , groups_list) - - if users_list: - User(users_list, - fetch_nonlocal_groups = params.fetch_nonlocal_groups - ) - - if groups_list: - Group(copy(groups_list), - ) - return groups_list - -def set_uid(user, user_dirs): - """ - user_dirs - comma separated directories - """ - import params - - File(format("{tmp_dir}/changeUid.sh"), - content=StaticFile("changeToSecureUid.sh"), - mode=0555) - ignore_groupsusers_create_str = str(params.ignore_groupsusers_create).lower() - uid = get_uid(user, return_existing=True) - Execute(format("{tmp_dir}/changeUid.sh {user} {user_dirs} {new_uid}", new_uid=0 if uid is None else uid), - not_if = format("(test $(id -u {user}) -gt 1000) || ({ignore_groupsusers_create_str})")) - -def get_uid(user, return_existing=False): - """ - Tries to get UID for username. It will try to find UID in custom properties in *cluster_env* and, if *return_existing=True*, - it will try to return UID of existing *user*. - - :param user: username to get UID for - :param return_existing: return UID for existing user - :return: - """ - import params - user_str = str(user) + "_uid" - service_env = [ serviceEnv for serviceEnv in params.config['configurations'] if user_str in params.config['configurations'][serviceEnv]] - - if service_env and params.config['configurations'][service_env[0]][user_str]: - service_env_str = str(service_env[0]) - uid = params.config['configurations'][service_env_str][user_str] - if len(service_env) > 1: - Logger.warning("Multiple values found for %s, using %s" % (user_str, uid)) - return uid - else: - if return_existing: - # pick up existing UID or try to find available UID in /etc/passwd, see changeToSecureUid.sh for more info - if user == params.smoke_user: - return None - File(format("{tmp_dir}/changeUid.sh"), - content=StaticFile("changeToSecureUid.sh"), - mode=0555) - code, newUid = shell.call(format("{tmp_dir}/changeUid.sh {user}")) - return int(newUid) - else: - # do not return UID for existing user, used in User resource call to let OS to choose UID for us - return None - -def setup_hadoop_env(): - import params - stackversion = params.stack_version_unformatted - Logger.info("FS Type: {0}".format(params.dfs_type)) - if params.has_namenode or stackversion.find('Gluster') >= 0 or params.dfs_type == 'HCFS': - if params.security_enabled: - tc_owner = "root" - else: - tc_owner = params.hdfs_user - - # create /etc/hadoop - Directory(params.hadoop_dir, mode=0755) - - # write out hadoop-env.sh, but only if the directory exists - if os.path.exists(params.hadoop_conf_dir): - File(os.path.join(params.hadoop_conf_dir, 'hadoop-env.sh'), owner=tc_owner, - group=params.user_group, - content=InlineTemplate(params.hadoop_env_sh_template)) - - # Create tmp dir for java.io.tmpdir - # Handle a situation when /tmp is set to noexec - Directory(params.hadoop_java_io_tmpdir, - owner=params.hdfs_user, - group=params.user_group, - mode=01777 - ) - -def setup_java(): - """ - Install jdk using specific params. - Install ambari jdk as well if the stack and ambari jdk are different. - """ - import params - __setup_java(custom_java_home=params.java_home, custom_jdk_name=params.jdk_name) - if params.ambari_java_home and params.ambari_java_home != params.java_home: - __setup_java(custom_java_home=params.ambari_java_home, custom_jdk_name=params.ambari_jdk_name) - -def __setup_java(custom_java_home, custom_jdk_name): - """ - Installs jdk using specific params, that comes from ambari-server - """ - import params - java_exec = format("{custom_java_home}/bin/java") - - if not os.path.isfile(java_exec): - if not params.jdk_name: # if custom jdk is used. - raise Fail(format("Unable to access {java_exec}. Confirm you have copied jdk to this host.")) - - jdk_curl_target = format("{tmp_dir}/{custom_jdk_name}") - java_dir = os.path.dirname(params.java_home) - - Directory(params.artifact_dir, - create_parents = True, - ) - - File(jdk_curl_target, - content = DownloadSource(format("{jdk_location}/{custom_jdk_name}")), - not_if = format("test -f {jdk_curl_target}") - ) - - File(jdk_curl_target, - mode = 0755, - ) - - tmp_java_dir = tempfile.mkdtemp(prefix="jdk_tmp_", dir=params.tmp_dir) - - try: - if params.jdk_name.endswith(".bin"): - chmod_cmd = ("chmod", "+x", jdk_curl_target) - install_cmd = format("cd {tmp_java_dir} && echo A | {jdk_curl_target} -noregister && {sudo} cp -rp {tmp_java_dir}/* {java_dir}") - elif params.jdk_name.endswith(".gz"): - chmod_cmd = ("chmod","a+x", java_dir) - install_cmd = format("cd {tmp_java_dir} && tar -xf {jdk_curl_target} && {sudo} cp -rp {tmp_java_dir}/* {java_dir}") - - Directory(java_dir - ) - - Execute(chmod_cmd, - sudo = True, - ) - - Execute(install_cmd, - ) - - finally: - Directory(tmp_java_dir, action="delete") - - File(format("{custom_java_home}/bin/java"), - mode=0755, - cd_access="a", - ) - Execute(('chmod', '-R', '755', params.java_home), - sudo = True, - ) - http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/hook.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/hook.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/hook.py deleted file mode 100644 index ce17776..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/hook.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -from resource_management import * -from shared_initialization import * -from repo_initialization import * - -class BeforeInstallHook(Hook): - - def hook(self, env): - import params - - self.run_custom_hook('before-ANY') - env.set_params(params) - - install_repos() - install_packages() - -if __name__ == "__main__": - BeforeInstallHook().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py deleted file mode 100644 index 50c5a40..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py +++ /dev/null @@ -1,115 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from ambari_commons.constants import AMBARI_SUDO_BINARY -from resource_management.libraries.functions.version import format_stack_version, compare_versions -from resource_management.core.system import System -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions import default, format -from resource_management.libraries.functions.expect import expect - -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() -sudo = AMBARI_SUDO_BINARY - -stack_version_unformatted = config['hostLevelParams']['stack_version'] -agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability'] -agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int) -stack_version_formatted = format_stack_version(stack_version_unformatted) - -#users and groups -hbase_user = config['configurations']['hbase-env']['hbase_user'] -smoke_user = config['configurations']['cluster-env']['smokeuser'] -gmetad_user = config['configurations']['ganglia-env']["gmetad_user"] -gmond_user = config['configurations']['ganglia-env']["gmond_user"] -tez_user = config['configurations']['tez-env']["tez_user"] - -user_group = config['configurations']['cluster-env']['user_group'] -proxyuser_group = default("/configurations/hadoop-env/proxyuser_group","users") - -hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] - -# repo templates -repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] -repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] - -#hosts -hostname = config["hostname"] -ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] -rm_host = default("/clusterHostInfo/rm_host", []) -slave_hosts = default("/clusterHostInfo/slave_hosts", []) -oozie_servers = default("/clusterHostInfo/oozie_server", []) -hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", []) -hive_server_host = default("/clusterHostInfo/hive_server_host", []) -hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", []) -hs_host = default("/clusterHostInfo/hs_host", []) -jtnode_host = default("/clusterHostInfo/jtnode_host", []) -namenode_host = default("/clusterHostInfo/namenode_host", []) -zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) -ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) -storm_server_hosts = default("/clusterHostInfo/nimbus_hosts", []) -falcon_host = default('/clusterHostInfo/falcon_server_hosts', []) - -has_sqoop_client = 'sqoop-env' in config['configurations'] -has_namenode = not len(namenode_host) == 0 -has_hs = not len(hs_host) == 0 -has_resourcemanager = not len(rm_host) == 0 -has_slaves = not len(slave_hosts) == 0 -has_oozie_server = not len(oozie_servers) == 0 -has_hcat_server_host = not len(hcat_server_hosts) == 0 -has_hive_server_host = not len(hive_server_host) == 0 -has_hbase_masters = not len(hbase_master_hosts) == 0 -has_zk_host = not len(zk_hosts) == 0 -has_ganglia_server = not len(ganglia_server_hosts) == 0 -has_storm_server = not len(storm_server_hosts) == 0 -has_falcon_server = not len(falcon_host) == 0 -has_tez = 'tez-site' in config['configurations'] - -is_namenode_master = hostname in namenode_host -is_jtnode_master = hostname in jtnode_host -is_rmnode_master = hostname in rm_host -is_hsnode_master = hostname in hs_host -is_hbase_master = hostname in hbase_master_hosts -is_slave = hostname in slave_hosts -if has_ganglia_server: - ganglia_server_host = ganglia_server_hosts[0] - -hbase_tmp_dir = "/tmp/hbase-hbase" - -#security params -security_enabled = config['configurations']['cluster-env']['security_enabled'] - -#java params -java_home = config['hostLevelParams']['java_home'] -artifact_dir = format("{tmp_dir}/AMBARI-artifacts/") -jdk_name = default("/hostLevelParams/jdk_name", None) # None when jdk is already installed by user -jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user -jce_location = config['hostLevelParams']['jdk_location'] -jdk_location = config['hostLevelParams']['jdk_location'] -ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False) -host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) - -smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}") -if has_hbase_masters: - hbase_user_dirs = format("/home/{hbase_user},/tmp/{hbase_user},/usr/bin/{hbase_user},/var/log/{hbase_user},{hbase_tmp_dir}") -#repo params -repo_info = config['hostLevelParams']['repo_info'] -service_repo_info = default("/hostLevelParams/service_repo_info",None) - -repo_file = default("/repositoryFile", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py deleted file mode 100644 index 9f2b344..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from ambari_commons.os_check import OSCheck -from resource_management.libraries.resources.repository import Repository -from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository, UBUNTU_REPO_COMPONENTS_POSTFIX -from resource_management.core.logger import Logger -import ambari_simplejson as json - - -def _alter_repo(action, repo_string, repo_template): - """ - @param action: "delete" or "create" - @param repo_string: e.g. "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]" - """ - repo_dicts = json.loads(repo_string) - - if not isinstance(repo_dicts, list): - repo_dicts = [repo_dicts] - - if 0 == len(repo_dicts): - Logger.info("Repository list is empty. Ambari may not be managing the repositories.") - else: - Logger.info("Initializing {0} repositories".format(str(len(repo_dicts)))) - - for repo in repo_dicts: - if not 'baseUrl' in repo: - repo['baseUrl'] = None - if not 'mirrorsList' in repo: - repo['mirrorsList'] = None - - ubuntu_components = [ repo['distribution'] if 'distribution' in repo and repo['distribution'] else repo['repoName'] ] \ - + [repo['components'].replace(",", " ") if 'components' in repo and repo['components'] else UBUNTU_REPO_COMPONENTS_POSTFIX] - - Repository(repo['repoId'], - action = action, - base_url = repo['baseUrl'], - mirror_list = repo['mirrorsList'], - repo_file_name = repo['repoName'], - repo_template = repo_template, - components = ubuntu_components) # ubuntu specific - - -def install_repos(): - import params - if params.host_sys_prepped: - return - - template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else params.repo_ubuntu - - # use this newer way of specifying repositories, if available - if params.repo_file is not None: - create_repo_files(template, CommandRepository(params.repo_file)) - return - - _alter_repo("create", params.repo_info, template) - - if params.service_repo_info: - _alter_repo("create", params.service_repo_info, template) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py deleted file mode 100644 index 1609050..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import os - -from resource_management.libraries.functions import stack_tools -from resource_management.libraries.functions.version import compare_versions -from resource_management.core.resources.packaging import Package - -def install_packages(): - import params - if params.host_sys_prepped: - return - - packages = ['unzip', 'curl'] - if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0: - stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME) - packages.append(stack_selector_package) - Package(packages, - retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, - retry_count=params.agent_stack_retry_count) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-RESTART/scripts/hook.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-RESTART/scripts/hook.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-RESTART/scripts/hook.py deleted file mode 100644 index 14b9d99..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-RESTART/scripts/hook.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from resource_management import * - -class BeforeRestartHook(Hook): - - def hook(self, env): - self.run_custom_hook('before-START') - -if __name__ == "__main__": - BeforeRestartHook().execute() - http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh deleted file mode 100644 index 68aa96d..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - -export hdfs_user=$1 -shift -export conf_dir=$1 -shift -export bin_dir=$1 -shift -export mark_dir=$1 -shift -export name_dirs=$* - -export EXIT_CODE=0 -export command="namenode -format" -export list_of_non_empty_dirs="" - -mark_file=/var/run/hadoop/hdfs/namenode-formatted -if [[ -f ${mark_file} ]] ; then - /var/lib/ambari-agent/ambari-sudo.sh rm -f ${mark_file} - /var/lib/ambari-agent/ambari-sudo.sh mkdir -p ${mark_dir} -fi - -if [[ ! -d $mark_dir ]] ; then - for dir in `echo $name_dirs | tr ',' ' '` ; do - echo "NameNode Dirname = $dir" - cmd="ls $dir | wc -l | grep -q ^0$" - eval $cmd - if [[ $? -ne 0 ]] ; then - (( EXIT_CODE = $EXIT_CODE + 1 )) - list_of_non_empty_dirs="$list_of_non_empty_dirs $dir" - fi - done - - if [[ $EXIT_CODE == 0 ]] ; then - /var/lib/ambari-agent/ambari-sudo.sh su ${hdfs_user} - -s /bin/bash -c "export PATH=$PATH:$bin_dir ; yes Y | hdfs --config ${conf_dir} ${command}" - (( EXIT_CODE = $EXIT_CODE | $? )) - else - echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}" - fi -else - echo "${mark_dir} exists. Namenode DFS already formatted" -fi - -exit $EXIT_CODE - http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/fast-hdfs-resource.jar ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/fast-hdfs-resource.jar b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/fast-hdfs-resource.jar deleted file mode 100644 index 6c993bf..0000000 Binary files a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/fast-hdfs-resource.jar and /dev/null differ http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/task-log4j.properties ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/task-log4j.properties b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/task-log4j.properties deleted file mode 100644 index 7e12962..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/task-log4j.properties +++ /dev/null @@ -1,134 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# - - -# Define some default values that can be overridden by system properties -hadoop.root.logger=INFO,console -hadoop.log.dir=. -hadoop.log.file=hadoop.log - -# -# Job Summary Appender -# -# Use following logger to send summary to separate file defined by -# hadoop.mapreduce.jobsummary.log.file rolled daily: -# hadoop.mapreduce.jobsummary.logger=INFO,JSA -# -hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} -hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log - -# Define the root logger to the system property "hadoop.root.logger". -log4j.rootLogger=${hadoop.root.logger}, EventCounter - -# Logging Threshold -log4j.threshhold=ALL - -# -# Daily Rolling File Appender -# - -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -# Debugging Pattern format -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - - -# -# console -# Add "console" to rootlogger above if you want to use this -# - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n - -# -# TaskLog Appender -# - -#Default values -hadoop.tasklog.taskid=null -hadoop.tasklog.iscleanup=false -hadoop.tasklog.noKeepSplits=4 -hadoop.tasklog.totalLogFileSize=100 -hadoop.tasklog.purgeLogSplits=true -hadoop.tasklog.logsRetainHours=12 - -log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender -log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} -log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} -log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} - -log4j.appender.TLA.layout=org.apache.log4j.PatternLayout -log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n - -# -# Rolling File Appender -# - -#log4j.appender.RFA=org.apache.log4j.RollingFileAppender -#log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} - -# Logfile size and and 30-day backups -#log4j.appender.RFA.MaxFileSize=1MB -#log4j.appender.RFA.MaxBackupIndex=30 - -#log4j.appender.RFA.layout=org.apache.log4j.PatternLayout -#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n -#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - - -# Custom Logging levels - -hadoop.metrics.log.level=INFO -#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG -#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG -#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG -log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level} - -# Jets3t library -log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR - -# -# Null Appender -# Trap security logger on the hadoop client side -# -log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender - -# -# Event Counter Appender -# Sends counts of logging messages at different severity levels to Hadoop Metrics. -# -log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter - -# Removes "deprecated" messages -log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/topology_script.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/topology_script.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/topology_script.py deleted file mode 100644 index 0f7a55c..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/topology_script.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -''' -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' - -import sys, os -from string import join -import ConfigParser - - -DEFAULT_RACK = "/default-rack" -DATA_FILE_NAME = os.path.dirname(os.path.abspath(__file__)) + "/topology_mappings.data" -SECTION_NAME = "network_topology" - -class TopologyScript(): - - def load_rack_map(self): - try: - #RACK_MAP contains both host name vs rack and ip vs rack mappings - mappings = ConfigParser.ConfigParser() - mappings.read(DATA_FILE_NAME) - return dict(mappings.items(SECTION_NAME)) - except ConfigParser.NoSectionError: - return {} - - def get_racks(self, rack_map, args): - if len(args) == 1: - return DEFAULT_RACK - else: - return join([self.lookup_by_hostname_or_ip(input_argument, rack_map) for input_argument in args[1:]],) - - def lookup_by_hostname_or_ip(self, hostname_or_ip, rack_map): - #try looking up by hostname - rack = rack_map.get(hostname_or_ip) - if rack is not None: - return rack - #try looking up by ip - rack = rack_map.get(self.extract_ip(hostname_or_ip)) - #try by localhost since hadoop could be passing in 127.0.0.1 which might not be mapped - return rack if rack is not None else rack_map.get("localhost.localdomain", DEFAULT_RACK) - - #strips out port and slashes in case hadoop passes in something like 127.0.0.1/127.0.0.1:50010 - def extract_ip(self, container_string): - return container_string.split("/")[0].split(":")[0] - - def execute(self, args): - rack_map = self.load_rack_map() - rack = self.get_racks(rack_map, args) - print rack - -if __name__ == "__main__": - TopologyScript().execute(sys.argv) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py deleted file mode 100644 index 04299ba..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import os - -from resource_management.core.resources import Directory -from resource_management.core.resources import Execute -from resource_management.libraries.functions import default -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions import format - - -DEFAULT_HADOOP_HDFS_EXTENSION_DIR = "/hdp/ext/{0}/hadoop" -DEFAULT_HADOOP_HIVE_EXTENSION_DIR = "/hdp/ext/{0}/hive" -DEFAULT_HADOOP_HBASE_EXTENSION_DIR = "/hdp/ext/{0}/hbase" - -def setup_extensions(): - """ - The goal of this method is to distribute extensions (for example jar files) from - HDFS (/hdp/ext/{major_stack_version}/{service_name}) to all nodes which contain related - components of service (YARN, HIVE or HBASE). Extensions should be added to HDFS by - user manually. - """ - - import params - - # Hadoop Custom extensions - hadoop_custom_extensions_enabled = default("/configurations/core-site/hadoop.custom-extensions.enabled", False) - hadoop_custom_extensions_services = default("/configurations/core-site/hadoop.custom-extensions.services", "") - hadoop_custom_extensions_owner = default("/configurations/core-site/hadoop.custom-extensions.owner", params.hdfs_user) - hadoop_custom_extensions_hdfs_dir = get_config_formatted_value(default("/configurations/core-site/hadoop.custom-extensions.root", - DEFAULT_HADOOP_HDFS_EXTENSION_DIR.format(params.major_stack_version))) - hadoop_custom_extensions_services = [ service.strip().upper() for service in hadoop_custom_extensions_services.split(",") ] - hadoop_custom_extensions_services.append("YARN") - - hadoop_custom_extensions_local_dir = "{0}/current/ext/hadoop".format(Script.get_stack_root()) - - if params.current_service in hadoop_custom_extensions_services: - clean_extensions(hadoop_custom_extensions_local_dir) - if hadoop_custom_extensions_enabled: - download_extensions(hadoop_custom_extensions_owner, params.user_group, - hadoop_custom_extensions_hdfs_dir, - hadoop_custom_extensions_local_dir) - - setup_extensions_hive() - - hbase_custom_extensions_services = [] - hbase_custom_extensions_services.append("HBASE") - if params.current_service in hbase_custom_extensions_services: - setup_hbase_extensions() - - -def setup_hbase_extensions(): - import params - - # HBase Custom extensions - hbase_custom_extensions_enabled = default("/configurations/hbase-site/hbase.custom-extensions.enabled", False) - hbase_custom_extensions_owner = default("/configurations/hbase-site/hbase.custom-extensions.owner", params.hdfs_user) - hbase_custom_extensions_hdfs_dir = get_config_formatted_value(default("/configurations/hbase-site/hbase.custom-extensions.root", - DEFAULT_HADOOP_HBASE_EXTENSION_DIR.format(params.major_stack_version))) - hbase_custom_extensions_local_dir = "{0}/current/ext/hbase".format(Script.get_stack_root()) - - impacted_components = ['HBASE_MASTER', 'HBASE_REGIONSERVER', 'PHOENIX_QUERY_SERVER']; - role = params.config.get('role','') - - if role in impacted_components: - clean_extensions(hbase_custom_extensions_local_dir) - if hbase_custom_extensions_enabled: - download_extensions(hbase_custom_extensions_owner, params.user_group, - hbase_custom_extensions_hdfs_dir, - hbase_custom_extensions_local_dir) - - -def setup_extensions_hive(): - import params - - hive_custom_extensions_enabled = default("/configurations/hive-site/hive.custom-extensions.enabled", False) - hive_custom_extensions_owner = default("/configurations/hive-site/hive.custom-extensions.owner", params.hdfs_user) - hive_custom_extensions_hdfs_dir = DEFAULT_HADOOP_HIVE_EXTENSION_DIR.format(params.major_stack_version) - - hive_custom_extensions_local_dir = "{0}/current/ext/hive".format(Script.get_stack_root()) - - impacted_components = ['HIVE_SERVER', 'HIVE_CLIENT']; - role = params.config.get('role','') - - # Run copying for HIVE_SERVER and HIVE_CLIENT - if params.current_service == 'HIVE' and role in impacted_components: - clean_extensions(hive_custom_extensions_local_dir) - if hive_custom_extensions_enabled: - download_extensions(hive_custom_extensions_owner, params.user_group, - hive_custom_extensions_hdfs_dir, - hive_custom_extensions_local_dir) - -def download_extensions(owner_user, owner_group, hdfs_source_dir, local_target_dir): - """ - :param owner_user: user owner of the HDFS directory - :param owner_group: group owner of the HDFS directory - :param hdfs_source_dir: the HDFS directory from where the files are being pull - :param local_target_dir: the location of where to download the files - :return: Will return True if successful, otherwise, False. - """ - import params - - if not os.path.isdir(local_target_dir): - extensions_tmp_dir=format("{tmp_dir}/custom_extensions") - Directory(local_target_dir, - owner="root", - mode=0755, - group="root", - create_parents=True) - - params.HdfsResource(hdfs_source_dir, - type="directory", - action="create_on_execute", - owner=owner_user, - group=owner_group, - mode=0755) - - Directory(extensions_tmp_dir, - owner=params.hdfs_user, - mode=0755, - create_parents=True) - - # copy from hdfs to /tmp - params.HdfsResource(extensions_tmp_dir, - type="directory", - action="download_on_execute", - source=hdfs_source_dir, - user=params.hdfs_user, - mode=0644, - replace_existing_files=True) - - # Execute command is not quoting correctly. - cmd = format("{sudo} mv {extensions_tmp_dir}/* {local_target_dir}") - only_if_cmd = "ls -d {extensions_tmp_dir}/*".format(extensions_tmp_dir=extensions_tmp_dir) - Execute(cmd, only_if=only_if_cmd) - - only_if_local = 'ls -d "{local_target_dir}"'.format(local_target_dir=local_target_dir) - Execute(("chown", "-R", "root:root", local_target_dir), - sudo=True, - only_if=only_if_local) - - params.HdfsResource(None,action="execute") - return True - -def clean_extensions(local_dir): - """ - :param local_dir: The local directory where the extensions are stored. - :return: Will return True if successful, otherwise, False. - """ - if os.path.isdir(local_dir): - Directory(local_dir, - action="delete") - return True - -def get_config_formatted_value(property_value): - return format(property_value.replace("{{", "{").replace("}}", "}")) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/hook.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/hook.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/hook.py deleted file mode 100644 index 4cb276a..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/hook.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -from resource_management import * -from rack_awareness import create_topology_script_and_mapping -from shared_initialization import setup_hadoop, setup_configs, create_javahome_symlink, setup_unlimited_key_jce_policy -from custom_extensions import setup_extensions - -class BeforeStartHook(Hook): - - def hook(self, env): - import params - - self.run_custom_hook('before-ANY') - env.set_params(params) - - setup_hadoop() - setup_configs() - create_javahome_symlink() - create_topology_script_and_mapping() - setup_unlimited_key_jce_policy() - if params.stack_supports_hadoop_custom_extensions: - setup_extensions() - -if __name__ == "__main__": - BeforeStartHook().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py deleted file mode 100644 index 6c26e01..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py +++ /dev/null @@ -1,380 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import os - -from resource_management.libraries.functions import conf_select -from resource_management.libraries.functions import stack_select -from resource_management.libraries.functions import default -from resource_management.libraries.functions import format_jvm_option -from resource_management.libraries.functions import format -from resource_management.libraries.functions.version import format_stack_version, compare_versions, get_major_version -from ambari_commons.os_check import OSCheck -from resource_management.libraries.script.script import Script -from resource_management.libraries.functions import get_kinit_path -from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources -from resource_management.libraries.resources.hdfs_resource import HdfsResource -from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.stack_features import get_stack_feature_version -from resource_management.libraries.functions import StackFeature -from ambari_commons.constants import AMBARI_SUDO_BINARY - -config = Script.get_config() -tmp_dir = Script.get_tmp_dir() -artifact_dir = tmp_dir + "/AMBARI-artifacts" - -version_for_stack_feature_checks = get_stack_feature_version(config) -stack_supports_hadoop_custom_extensions = check_stack_feature(StackFeature.HADOOP_CUSTOM_EXTENSIONS, version_for_stack_feature_checks) - -sudo = AMBARI_SUDO_BINARY - -# Global flag enabling or disabling the sysprep feature -host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) - -# Whether to skip copying fast-hdfs-resource.jar to /var/lib/ambari-agent/lib/ -# This is required if tarballs are going to be copied to HDFS, so set to False -sysprep_skip_copy_fast_jar_hdfs = host_sys_prepped and default("/configurations/cluster-env/sysprep_skip_copy_fast_jar_hdfs", False) - -# Whether to skip setting up the unlimited key JCE policy -sysprep_skip_setup_jce = host_sys_prepped and default("/configurations/cluster-env/sysprep_skip_setup_jce", False) - -stack_version_unformatted = config['hostLevelParams']['stack_version'] -stack_version_formatted = format_stack_version(stack_version_unformatted) -major_stack_version = get_major_version(stack_version_formatted) - -dfs_type = default("/commandParams/dfs_type", "") -hadoop_conf_dir = "/etc/hadoop/conf" -component_list = default("/localComponents", []) - -hdfs_tmp_dir = default("/configurations/hadoop-env/hdfs_tmp_dir", "/tmp") - -hadoop_metrics2_properties_content = None -if 'hadoop-metrics2.properties' in config['configurations']: - hadoop_metrics2_properties_content = config['configurations']['hadoop-metrics2.properties']['content'] - -hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") -hadoop_lib_home = stack_select.get_hadoop_dir("lib") -hadoop_bin = stack_select.get_hadoop_dir("sbin") - -mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*" -hadoop_home = stack_select.get_hadoop_dir("home") -create_lib_snappy_symlinks = False - -current_service = config['serviceName'] - -#security params -security_enabled = config['configurations']['cluster-env']['security_enabled'] - -ambari_server_resources_url = default("/hostLevelParams/jdk_location", None) -if ambari_server_resources_url is not None and ambari_server_resources_url.endswith('/'): - ambari_server_resources_url = ambari_server_resources_url[:-1] - -# Unlimited key JCE policy params -jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user -unlimited_key_jce_required = default("/hostLevelParams/unlimited_key_jce_required", False) -jdk_name = default("/hostLevelParams/jdk_name", None) -java_home = default("/hostLevelParams/java_home", None) -java_exec = "{0}/bin/java".format(java_home) if java_home is not None else "/bin/java" - -#users and groups -has_hadoop_env = 'hadoop-env' in config['configurations'] -mapred_user = config['configurations']['mapred-env']['mapred_user'] -hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -yarn_user = config['configurations']['yarn-env']['yarn_user'] - -user_group = config['configurations']['cluster-env']['user_group'] - -#hosts -hostname = config["hostname"] -ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] -rm_host = default("/clusterHostInfo/rm_host", []) -slave_hosts = default("/clusterHostInfo/slave_hosts", []) -oozie_servers = default("/clusterHostInfo/oozie_server", []) -hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", []) -hive_server_host = default("/clusterHostInfo/hive_server_host", []) -hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", []) -hs_host = default("/clusterHostInfo/hs_host", []) -jtnode_host = default("/clusterHostInfo/jtnode_host", []) -namenode_host = default("/clusterHostInfo/namenode_host", []) -zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) -ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) -cluster_name = config["clusterName"] -set_instanceId = "false" -if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_external_hosts' in config['configurations']['cluster-env']: - ams_collector_hosts = config['configurations']['cluster-env']['metrics_collector_external_hosts'] - set_instanceId = "true" -else: - ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", [])) - -has_namenode = not len(namenode_host) == 0 -has_resourcemanager = not len(rm_host) == 0 -has_slaves = not len(slave_hosts) == 0 -has_oozie_server = not len(oozie_servers) == 0 -has_hcat_server_host = not len(hcat_server_hosts) == 0 -has_hive_server_host = not len(hive_server_host) == 0 -has_hbase_masters = not len(hbase_master_hosts) == 0 -has_zk_host = not len(zk_hosts) == 0 -has_ganglia_server = not len(ganglia_server_hosts) == 0 -has_metric_collector = not len(ams_collector_hosts) == 0 - -is_namenode_master = hostname in namenode_host -is_jtnode_master = hostname in jtnode_host -is_rmnode_master = hostname in rm_host -is_hsnode_master = hostname in hs_host -is_hbase_master = hostname in hbase_master_hosts -is_slave = hostname in slave_hosts - -if has_ganglia_server: - ganglia_server_host = ganglia_server_hosts[0] - -metric_collector_port = None -if has_metric_collector: - if 'cluster-env' in config['configurations'] and \ - 'metrics_collector_external_port' in config['configurations']['cluster-env']: - metric_collector_port = config['configurations']['cluster-env']['metrics_collector_external_port'] - else: - metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188") - if metric_collector_web_address.find(':') != -1: - metric_collector_port = metric_collector_web_address.split(':')[1] - else: - metric_collector_port = '6188' - if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY": - metric_collector_protocol = 'https' - else: - metric_collector_protocol = 'http' - metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "") - metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "") - metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "") - - pass -metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60) -metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10) - -host_in_memory_aggregation = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation", True) -host_in_memory_aggregation_port = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.port", 61888) - -# Cluster Zookeeper quorum -zookeeper_quorum = None -if has_zk_host: - if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']: - zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort'] - else: - zookeeper_clientPort = '2181' - zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts']) - # last port config - zookeeper_quorum += ':' + zookeeper_clientPort - -#hadoop params - -if has_namenode or dfs_type == 'HCFS': - hadoop_tmp_dir = format("/tmp/hadoop-{hdfs_user}") - hadoop_conf_dir = conf_select.get_hadoop_conf_dir() - task_log4j_properties_location = os.path.join(hadoop_conf_dir, "task-log4j.properties") - -hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] -hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] -hbase_tmp_dir = "/tmp/hbase-hbase" -#db params -server_db_name = config['hostLevelParams']['db_name'] -db_driver_filename = config['hostLevelParams']['db_driver_filename'] -oracle_driver_url = config['hostLevelParams']['oracle_jdbc_url'] -mysql_driver_url = config['hostLevelParams']['mysql_jdbc_url'] -oracle_driver_symlink_url = format("{ambari_server_resources_url}/oracle-jdbc-driver.jar") -mysql_driver_symlink_url = format("{ambari_server_resources_url}/mysql-jdbc-driver.jar") - -ambari_db_rca_url = config['hostLevelParams']['ambari_db_rca_url'][0] -ambari_db_rca_driver = config['hostLevelParams']['ambari_db_rca_driver'][0] -ambari_db_rca_username = config['hostLevelParams']['ambari_db_rca_username'][0] -ambari_db_rca_password = config['hostLevelParams']['ambari_db_rca_password'][0] - -if has_namenode and 'rca_enabled' in config['configurations']['hadoop-env']: - rca_enabled = config['configurations']['hadoop-env']['rca_enabled'] -else: - rca_enabled = False -rca_disabled_prefix = "###" -if rca_enabled == True: - rca_prefix = "" -else: - rca_prefix = rca_disabled_prefix - -#hadoop-env.sh - -jsvc_path = "/usr/lib/bigtop-utils" - -hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] -namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] -namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize'] -namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize'] -namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m") -namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m") - -jtnode_opt_newsize = "200m" -jtnode_opt_maxnewsize = "200m" -jtnode_heapsize = "1024m" -ttnode_heapsize = "1024m" - -dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize'] -mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce") -mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce") - -#log4j.properties - -yarn_log_dir_prefix = default("/configurations/yarn-env/yarn_log_dir_prefix","/var/log/hadoop-yarn") - -dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None) - -# Hdfs log4j settings -hadoop_log_max_backup_size = default('configurations/hdfs-log4j/hadoop_log_max_backup_size', 256) -hadoop_log_number_of_backup_files = default('configurations/hdfs-log4j/hadoop_log_number_of_backup_files', 10) -hadoop_security_log_max_backup_size = default('configurations/hdfs-log4j/hadoop_security_log_max_backup_size', 256) -hadoop_security_log_number_of_backup_files = default('configurations/hdfs-log4j/hadoop_security_log_number_of_backup_files', 20) - -# Yarn log4j settings -yarn_rm_summary_log_max_backup_size = default('configurations/yarn-log4j/yarn_rm_summary_log_max_backup_size', 256) -yarn_rm_summary_log_number_of_backup_files = default('configurations/yarn-log4j/yarn_rm_summary_log_number_of_backup_files', 20) - -#log4j.properties -if (('hdfs-log4j' in config['configurations']) and ('content' in config['configurations']['hdfs-log4j'])): - log4j_props = config['configurations']['hdfs-log4j']['content'] - if (('yarn-log4j' in config['configurations']) and ('content' in config['configurations']['yarn-log4j'])): - log4j_props += config['configurations']['yarn-log4j']['content'] -else: - log4j_props = None - -refresh_topology = False -command_params = config["commandParams"] if "commandParams" in config else None -if command_params is not None: - refresh_topology = bool(command_params["refresh_topology"]) if "refresh_topology" in command_params else False - -ambari_java_home = default("/commandParams/ambari_java_home", None) -ambari_jdk_name = default("/commandParams/ambari_jdk_name", None) -ambari_jce_name = default("/commandParams/ambari_jce_name", None) - -ambari_libs_dir = "/var/lib/ambari-agent/lib" -is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled'] -default_fs = config['configurations']['core-site']['fs.defaultFS'] - -#host info -all_hosts = default("/clusterHostInfo/all_hosts", []) -all_racks = default("/clusterHostInfo/all_racks", []) -all_ipv4_ips = default("/clusterHostInfo/all_ipv4_ips", []) -slave_hosts = default("/clusterHostInfo/slave_hosts", []) - -#topology files -net_topology_script_file_path = "/etc/hadoop/conf/topology_script.py" -net_topology_script_dir = os.path.dirname(net_topology_script_file_path) -net_topology_mapping_data_file_name = 'topology_mappings.data' -net_topology_mapping_data_file_path = os.path.join(net_topology_script_dir, net_topology_mapping_data_file_name) - -#Added logic to create /tmp and /user directory for HCFS stack. -has_core_site = 'core-site' in config['configurations'] -hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] -kinit_path_local = get_kinit_path() -stack_version_unformatted = config['hostLevelParams']['stack_version'] -stack_version_formatted = format_stack_version(stack_version_unformatted) -hadoop_bin_dir = stack_select.get_hadoop_dir("bin") -hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', None) -hdfs_site = config['configurations']['hdfs-site'] -smoke_user = config['configurations']['cluster-env']['smokeuser'] -smoke_hdfs_user_dir = format("/user/{smoke_user}") -smoke_hdfs_user_mode = 0770 - - -##### Namenode RPC ports - metrics config section start ##### - -# Figure out the rpc ports for current namenode -nn_rpc_client_port = None -nn_rpc_dn_port = None -nn_rpc_healthcheck_port = None - -namenode_id = None -namenode_rpc = None - -dfs_ha_enabled = False -dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.internal.nameservices', None) -if dfs_ha_nameservices is None: - dfs_ha_nameservices = default('/configurations/hdfs-site/dfs.nameservices', None) -dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None) - -dfs_ha_namemodes_ids_list = [] -other_namenode_id = None - -if dfs_ha_namenode_ids: - dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",") - dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list) - if dfs_ha_namenode_ids_array_len > 1: - dfs_ha_enabled = True - -if dfs_ha_enabled: - for nn_id in dfs_ha_namemodes_ids_list: - nn_host = config['configurations']['hdfs-site'][format('dfs.namenode.rpc-address.{dfs_ha_nameservices}.{nn_id}')] - if hostname.lower() in nn_host.lower(): - namenode_id = nn_id - namenode_rpc = nn_host - pass - pass -else: - namenode_rpc = default('/configurations/hdfs-site/dfs.namenode.rpc-address', default_fs) - -# if HDFS is not installed in the cluster, then don't try to access namenode_rpc -if "core-site" in config['configurations'] and namenode_rpc: - port_str = namenode_rpc.split(':')[-1].strip() - try: - nn_rpc_client_port = int(port_str) - except ValueError: - nn_rpc_client_port = None - -if dfs_ha_enabled: - dfs_service_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.servicerpc-address.{dfs_ha_nameservices}.{namenode_id}'), None) - dfs_lifeline_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.lifeline.rpc-address.{dfs_ha_nameservices}.{namenode_id}'), None) -else: - dfs_service_rpc_address = default('/configurations/hdfs-site/dfs.namenode.servicerpc-address', None) - dfs_lifeline_rpc_address = default(format('/configurations/hdfs-site/dfs.namenode.lifeline.rpc-address'), None) - -if dfs_service_rpc_address: - nn_rpc_dn_port = dfs_service_rpc_address.split(':')[1].strip() - -if dfs_lifeline_rpc_address: - nn_rpc_healthcheck_port = dfs_lifeline_rpc_address.split(':')[1].strip() - -is_nn_client_port_configured = False if nn_rpc_client_port is None else True -is_nn_dn_port_configured = False if nn_rpc_dn_port is None else True -is_nn_healthcheck_port_configured = False if nn_rpc_healthcheck_port is None else True - -##### end ##### - -import functools -#create partial functions with common arguments for every HdfsResource call -#to create/delete/copyfromlocal hdfs directories/files we need to call params.HdfsResource in code -HdfsResource = functools.partial( - HdfsResource, - user=hdfs_user, - hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", - security_enabled = security_enabled, - keytab = hdfs_user_keytab, - kinit_path_local = kinit_path_local, - hadoop_bin_dir = hadoop_bin_dir, - hadoop_conf_dir = hadoop_conf_dir, - principal_name = hdfs_principal_name, - hdfs_site = hdfs_site, - default_fs = default_fs, - immutable_paths = get_not_managed_resources(), - dfs_type = dfs_type -) http://git-wip-us.apache.org/repos/asf/ambari/blob/5b36cdfd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/rack_awareness.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/rack_awareness.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/rack_awareness.py deleted file mode 100644 index 48158bb..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/rack_awareness.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python - -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from resource_management.core.resources import File -from resource_management.core.source import StaticFile, Template -from resource_management.libraries.functions import format - - -def create_topology_mapping(): - import params - - File(params.net_topology_mapping_data_file_path, - content=Template("topology_mappings.data.j2"), - owner=params.hdfs_user, - group=params.user_group, - mode=0644, - only_if=format("test -d {net_topology_script_dir}")) - -def create_topology_script(): - import params - - File(params.net_topology_script_file_path, - content=StaticFile('topology_script.py'), - mode=0755, - only_if=format("test -d {net_topology_script_dir}")) - -def create_topology_script_and_mapping(): - import params - if params.has_hadoop_env: - create_topology_mapping() - create_topology_script()