Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 4DA23200B52 for ; Mon, 11 Jul 2016 05:54:35 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 4C3EA160A69; Mon, 11 Jul 2016 03:54:35 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 233B9160A66 for ; Mon, 11 Jul 2016 05:54:33 +0200 (CEST) Received: (qmail 21279 invoked by uid 500); 11 Jul 2016 03:54:33 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 21270 invoked by uid 99); 11 Jul 2016 03:54:33 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 11 Jul 2016 03:54:33 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id EC31BE09C5; Mon, 11 Jul 2016 03:54:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: smohanty@apache.org To: commits@ambari.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-17644. hiveserver2-site.xml is missing from /etc/hive2/conf/conf.server when hsi is set up by Ambari (smohanty) Date: Mon, 11 Jul 2016 03:54:32 +0000 (UTC) archived-at: Mon, 11 Jul 2016 03:54:35 -0000 Repository: ambari Updated Branches: refs/heads/trunk 49396c4ed -> 8858cb604 AMBARI-17644. hiveserver2-site.xml is missing from /etc/hive2/conf/conf.server when hsi is set up by Ambari (smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8858cb60 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8858cb60 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8858cb60 Branch: refs/heads/trunk Commit: 8858cb604a5488e84eb8fa68cf04d4790015394a Parents: 49396c4 Author: Sumit Mohanty Authored: Sun Jul 10 20:50:45 2016 -0700 Committer: Sumit Mohanty Committed: Sun Jul 10 20:50:45 2016 -0700 ---------------------------------------------------------------------- .../package/scripts/hive_interactive.py | 20 ++ .../package/scripts/hive_interactive.py.orig | 253 +++++++++++++++++++ .../hiveserver2-interactive-site.xml | 43 ++++ .../stacks/HDP/2.5/services/HIVE/metainfo.xml | 1 + .../stacks/2.5/HIVE/test_hive_server_int.py | 12 + .../python/stacks/2.5/configs/hsi_default.json | 24 +- 6 files changed, 345 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8858cb60/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py index 631b5a8..db269b8 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py @@ -121,6 +121,18 @@ def hive_interactive(name=None): group = params.user_group, mode = 0664) + ''' + + ''' + merged_hiveserver2_interactive_site = {} + if 'hiveserver2-site' in params.config['configurations']: + merged_hiveserver2_interactive_site.update(params.config['configurations']['hiveserver2-site']) + Logger.info("Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'.") + else: + Logger.error("'hiveserver2-site' couldn't be retrieved from passed-in configurations.") + merged_hiveserver2_interactive_site.update(params.config['configurations']['hiveserver2-interactive-site']) + + # Create config files under /etc/hive2/conf and /etc/hive2/conf/conf.server: # hive-site.xml # hive-env.sh @@ -139,6 +151,14 @@ def hive_interactive(name=None): group=params.user_group, mode=0644) + XmlConfig("hiveserver2-site.xml", + conf_dir=conf_dir, + configurations=merged_hiveserver2_interactive_site, + configuration_attributes=params.config['configuration_attributes']['hiveserver2-interactive-site'], + owner=params.hive_user, + group=params.user_group, + mode=0644) + hive_server_interactive_conf_dir = conf_dir File(format("{hive_server_interactive_conf_dir}/hive-env.sh"), http://git-wip-us.apache.org/repos/asf/ambari/blob/8858cb60/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py.orig ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py.orig b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py.orig new file mode 100644 index 0000000..631b5a8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py.orig @@ -0,0 +1,253 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +# Python Imports +import os +import glob +from urlparse import urlparse + +# Resource Management and Common Imports +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.version import compare_versions +from resource_management.core.resources.service import ServiceConfig +from resource_management.core.resources.system import File, Directory +from resource_management.core.source import Template, DownloadSource, InlineTemplate +from resource_management.core.shell import as_user +from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions.format import format +from resource_management.core.shell import as_sudo +from resource_management.core.shell import quote_bash_args +from resource_management.core.logger import Logger +from resource_management.core import utils + +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst +from hive import fill_conf_dir, jdbc_connector + + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def hive_interactive(name=None): + pass + +""" +Sets up the configs, jdbc connection and tarball copy to HDFS for Hive Server Interactive. +""" +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def hive_interactive(name=None): + import params + + # list of properties that should be excluded from the config + # this approach is a compromise against adding a dedicated config + # type for hive_server_interactive or needed config groups on a + # per component basis + exclude_list = ['hive.enforce.bucketing', + 'hive.enforce.sorting'] + + # Copy Tarballs in HDFS. + if params.stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted_major): + resource_created = copy_to_hdfs("tez_hive2", + params.user_group, + params.hdfs_user, + file_mode=params.tarballs_mode, + host_sys_prepped=params.host_sys_prepped) + + if resource_created: + params.HdfsResource(None, action="execute") + + Directory(params.hive_interactive_etc_dir_prefix, + mode=0755 + ) + + Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) + for conf_dir in params.hive_conf_dirs_list: + fill_conf_dir(conf_dir) + + ''' + As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml, + we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml. + ''' + merged_hive_interactive_site = {} + merged_hive_interactive_site.update(params.config['configurations']['hive-site']) + merged_hive_interactive_site.update(params.config['configurations']['hive-interactive-site']) + for item in exclude_list: + if item in merged_hive_interactive_site.keys(): + del merged_hive_interactive_site[item] + + ''' + Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook', + which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed. + ''' + remove_atlas_hook_if_exists(merged_hive_interactive_site) + + ''' + As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml, + we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml. + ''' + merged_tez_interactive_site = {} + if 'tez-site' in params.config['configurations']: + merged_tez_interactive_site.update(params.config['configurations']['tez-site']) + Logger.info("Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'.") + else: + Logger.error("Tez's 'tez-site' couldn't be retrieved from passed-in configurations.") + + merged_tez_interactive_site.update(params.config['configurations']['tez-interactive-site']) + XmlConfig("tez-site.xml", + conf_dir = params.tez_interactive_config_dir, + configurations = merged_tez_interactive_site, + configuration_attributes=params.config['configuration_attributes']['tez-interactive-site'], + owner = params.tez_interactive_user, + group = params.user_group, + mode = 0664) + + # Create config files under /etc/hive2/conf and /etc/hive2/conf/conf.server: + # hive-site.xml + # hive-env.sh + # llap-daemon-log4j2.properties + # llap-cli-log4j2.properties + # hive-log4j2.properties + # hive-exec-log4j2.properties + # beeline-log4j2.properties + + for conf_dir in params.hive_conf_dirs_list: + XmlConfig("hive-site.xml", + conf_dir=conf_dir, + configurations=merged_hive_interactive_site, + configuration_attributes=params.config['configuration_attributes']['hive-interactive-site'], + owner=params.hive_user, + group=params.user_group, + mode=0644) + + hive_server_interactive_conf_dir = conf_dir + + File(format("{hive_server_interactive_conf_dir}/hive-env.sh"), + owner=params.hive_user, + group=params.user_group, + content=InlineTemplate(params.hive_interactive_env_sh_template)) + + llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"), + mode=0644, + group=params.user_group, + owner=params.hive_user, + content=params.llap_daemon_log4j) + + llap_cli_log4j2_filename = 'llap-cli-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"), + mode=0644, + group=params.user_group, + owner=params.hive_user, + content=params.llap_cli_log4j2) + + hive_log4j2_filename = 'hive-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"), + mode=0644, + group=params.user_group, + owner=params.hive_user, + content=params.hive_log4j2) + + hive_exec_log4j2_filename = 'hive-exec-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"), + mode=0644, + group=params.user_group, + owner=params.hive_user, + content=params.hive_exec_log4j2) + + beeline_log4j2_filename = 'beeline-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"), + mode=0644, + group=params.user_group, + owner=params.hive_user, + content=params.beeline_log4j2) + + File(format("{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"), + owner=params.hive_user, + group=params.user_group, + content=Template("hadoop-metrics2-llapdaemon.j2")) + + File(format("{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"), + owner=params.hive_user, + group=params.user_group, + content=Template("hadoop-metrics2-llaptaskscheduler.j2")) + + + # On some OS this folder could be not exists, so we will create it before pushing there files + Directory(params.limits_conf_dir, + create_parents = True, + owner='root', + group='root') + + File(os.path.join(params.limits_conf_dir, 'hive.conf'), + owner='root', + group='root', + mode=0644, + content=Template("hive.conf.j2")) + + if not os.path.exists(params.target_hive_interactive): + jdbc_connector(params.target_hive_interactive, params.hive_intaractive_previous_jdbc_jar) + + File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), + content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), + mode = 0644) + File(params.start_hiveserver2_interactive_path, + mode=0755, + content=Template(format('{start_hiveserver2_interactive_script}'))) + + Directory(params.hive_pid_dir, + create_parents=True, + cd_access='a', + owner=params.hive_user, + group=params.user_group, + mode=0755) + Directory(params.hive_log_dir, + create_parents=True, + cd_access='a', + owner=params.hive_user, + group=params.user_group, + mode=0755) + Directory(params.hive_interactive_var_lib, + create_parents=True, + cd_access='a', + owner=params.hive_user, + group=params.user_group, + mode=0755) + +""" +Remove 'org.apache.atlas.hive.hook.HiveHook' value from Hive2/hive-site.xml config 'hive.exec.post.hooks', if exists. +""" +def remove_atlas_hook_if_exists(merged_hive_interactive_site): + if 'hive.exec.post.hooks' in merged_hive_interactive_site.keys(): + existing_hive_exec_post_hooks = merged_hive_interactive_site.get('hive.exec.post.hooks') + if existing_hive_exec_post_hooks: + hook_splits = existing_hive_exec_post_hooks.split(",") + updated_hook_splits = [hook for hook in hook_splits if not hook.strip() == 'org.apache.atlas.hive.hook.HiveHook'] + updated_hooks_str = ",".join((str(hook)).strip() for hook in updated_hook_splits) + if updated_hooks_str != existing_hive_exec_post_hooks: + merged_hive_interactive_site['hive.exec.post.hooks'] = updated_hooks_str + Logger.info("Updated Hive2/hive-site.xml 'hive.exec.post.hooks' value from : '{0}' to : '{1}'" + .format(existing_hive_exec_post_hooks, updated_hooks_str)) + else: + Logger.info("No change done to Hive2/hive-site.xml 'hive.exec.post.hooks' value.") + else: + Logger.debug("'hive.exec.post.hooks' doesn't exist in Hive2/hive-site.xml") http://git-wip-us.apache.org/repos/asf/ambari/blob/8858cb60/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hiveserver2-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hiveserver2-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hiveserver2-interactive-site.xml new file mode 100644 index 0000000..1652c91 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hiveserver2-interactive-site.xml @@ -0,0 +1,43 @@ + + + + + + hive.metastore.metrics.enabled + true + + boolean + + + + + hive.service.metrics.reporter + JSON_FILE, JMX, HADOOP2 + + + + hive.service.metrics.hadoop2.component + hiveserver2-interactive + + + + hive.service.metrics.file.location + /var/log/hive/hiveserver2Interactive-report.json + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/8858cb60/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml index d56522b..3b43b39 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/metainfo.xml @@ -107,6 +107,7 @@ hive-log4j2 hive-site hive-interactive-site + hiveserver2-interactive-site hive-interactive-env llap-cli-log4j2 llap-daemon-log4j http://git-wip-us.apache.org/repos/asf/ambari/blob/8858cb60/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py index dfa66ad..2a0674d 100644 --- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py @@ -255,6 +255,10 @@ class TestHiveServerInteractive(RMFTestCase): del hive_site_conf['hive.enforce.bucketing'] del hive_site_conf['hive.enforce.sorting'] + hiveserver2_site_conf = {} + hiveserver2_site_conf.update(self.getConfig()['configurations']['hiveserver2-site']) + hiveserver2_site_conf.update(self.getConfig()['configurations']['hiveserver2-interactive-site']) + mapred_site_conf = {} mapred_site_conf.update(self.getConfig()['configurations']['mapred-site']) @@ -302,6 +306,14 @@ class TestHiveServerInteractive(RMFTestCase): owner='hive', configurations=hive_site_conf, ) + self.assertResourceCalled('XmlConfig', 'hiveserver2-site.xml', + group='hadoop', + conf_dir=conf_dir, + mode=0644, + configuration_attributes={}, + owner='hive', + configurations=hiveserver2_site_conf, + ) self.assertResourceCalled('File', os.path.join(conf_dir, 'hive-env.sh'), content=InlineTemplate(self.getConfig()['configurations']['hive-interactive-env']['content']), owner='hive', http://git-wip-us.apache.org/repos/asf/ambari/blob/8858cb60/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json index 5411c05..78a0d04 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json @@ -366,6 +366,10 @@ "ipc.client.connection.maxidletime": "30000", "ipc.client.connect.max.retries": "50" }, + "hiveserver2-site": { + "c" : "d", + "a" : "e" + }, "hive-site": { "hive.enforce.sorting": "true", "javax.jdo.option.ConnectionPassword": "!`\"' 1", @@ -406,15 +410,18 @@ "hive.exec.scratchdir" : "/custompath/tmp/hive", "hive.exec.post.hooks" : "a, org.apache.atlas.hive.hook.HiveHook,org.apache.atlas.hive.hook.HiveHook , b , org.apache.atlas.hive.hook.HiveHook ,org.apache.hadoop.hive.ql.hooks.ATSHook" }, + "hiveserver2-interactive-site": { + "a" : "b" + }, "hive-interactive-env": { - "content" : "con\ntent", - "enable_hive_interactive" : "true", - "hive_server_interactive_host" : "c6401.ambari.apache.org", - "llap_queue_capacity" : "0", - "num_llap_nodes" : "1", - "num_retries_for_checking_llap_status" : 2, - "llap_heap_size":"280", - "llap_log_level":"INFO" + "content" : "con\ntent", + "enable_hive_interactive" : "true", + "hive_server_interactive_host" : "c6401.ambari.apache.org", + "llap_queue_capacity" : "0", + "num_llap_nodes" : "1", + "num_retries_for_checking_llap_status" : 2, + "llap_heap_size":"280", + "llap_log_level":"INFO" }, "hive-interactive-site": { "hive.enforce.sorting": "true", @@ -1040,6 +1047,7 @@ "hive.optimize.bucketmapjoin.sortedmerge": "true" } }, + "hiveserver2-interactive-site": {}, "hive-interactive-site": { "final": { "javax.jdo.option.ConnectionPassword": "true",