Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 25FE018FCD for ; Wed, 28 Oct 2015 04:57:03 +0000 (UTC) Received: (qmail 78419 invoked by uid 500); 28 Oct 2015 04:57:00 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 78379 invoked by uid 500); 28 Oct 2015 04:57:00 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 78370 invoked by uid 99); 28 Oct 2015 04:56:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 28 Oct 2015 04:56:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 99767E0441; Wed, 28 Oct 2015 04:56:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jonathanhurley@apache.org To: commits@ambari.apache.org Message-Id: <15e8c172435949b38e1aec7fb217369b@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-13588 - Express Upgrade Packs For Hive (jonathanhurley) Date: Wed, 28 Oct 2015 04:56:59 +0000 (UTC) Repository: ambari Updated Branches: refs/heads/trunk 9e085dfd0 -> deb782c79 AMBARI-13588 - Express Upgrade Packs For Hive (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/deb782c7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/deb782c7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/deb782c7 Branch: refs/heads/trunk Commit: deb782c796cf4ee792f1f1d98769583698b89f60 Parents: 9e085df Author: Jonathan Hurley Authored: Tue Oct 27 16:15:02 2015 -0400 Committer: Jonathan Hurley Committed: Wed Oct 28 00:56:28 2015 -0400 ---------------------------------------------------------------------- .../src/main/python/ambari_commons/constants.py | 9 ++++-- .../libraries/script/script.py | 5 +-- .../0.12.0.2.0/package/scripts/hive_client.py | 4 ++- .../package/scripts/hive_metastore.py | 28 ++++++++++++----- .../0.12.0.2.0/package/scripts/hive_server.py | 26 ++++++++++----- .../0.12.0.2.0/package/scripts/hive_service.py | 9 +++--- .../package/scripts/setup_ranger_hive.py | 5 +-- .../package/scripts/webhcat_server.py | 10 +++--- .../package/scripts/webhcat_service.py | 4 +-- .../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 2 +- .../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml | 2 +- .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 2 +- .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 2 +- .../stacks/2.0.6/HIVE/test_hive_client.py | 8 ++--- .../stacks/2.0.6/HIVE/test_hive_metastore.py | 4 ++- .../stacks/2.0.6/HIVE/test_hive_server.py | 8 ++--- .../stacks/2.0.6/HIVE/test_webhcat_server.py | 8 ++--- .../stacks/2.1/HIVE/test_hive_metastore.py | 33 ++++++++++++++------ .../src/test/python/stacks/utils/RMFTestCase.py | 8 ++++- 19 files changed, 115 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-common/src/main/python/ambari_commons/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/ambari_commons/constants.py b/ambari-common/src/main/python/ambari_commons/constants.py index 99ff679..525e024 100644 --- a/ambari-common/src/main/python/ambari_commons/constants.py +++ b/ambari-common/src/main/python/ambari_commons/constants.py @@ -1,6 +1,5 @@ #!/usr/bin/env python - -''' +""" Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information @@ -16,6 +15,10 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -''' + +""" AMBARI_SUDO_BINARY = "ambari-sudo.sh" + +UPGRADE_TYPE_ROLLING = "rolling" +UPGRADE_TYPE_NON_ROLLING = "nonrolling" http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 79bc535..10a9aa6 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -29,6 +29,7 @@ import platform import inspect import tarfile from ambari_commons import OSCheck, OSConst +from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from resource_management.libraries.resources import XmlConfig from resource_management.libraries.resources import PropertiesFile @@ -468,9 +469,9 @@ class Script(object): upgrade_type = None if restart_type.lower() == "rolling_upgrade": - upgrade_type = "rolling" + upgrade_type = UPGRADE_TYPE_ROLLING elif restart_type.lower() == "nonrolling_upgrade": - upgrade_type = "nonrolling" + upgrade_type = UPGRADE_TYPE_NON_ROLLING is_stack_upgrade = upgrade_type is not None http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py index 0cacb07..117b456 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py @@ -50,7 +50,9 @@ class HiveClientDefault(HiveClient): def get_stack_to_component(self): return {"HDP": "hadoop-client"} - def pre_rolling_restart(self, env): + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing Hive client Stack Upgrade pre-restart") + import params env.set_params(params) if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py index 8bdccdc..080225a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py @@ -48,16 +48,22 @@ class HiveMetastore(Script): import params self.install_packages(env, exclude_packages = params.hive_exclude_packages) - def start(self, env, rolling_restart=False): + + def start(self, env, upgrade_type=None): import params env.set_params(params) - self.configure(env) # FOR SECURITY - hive_service('metastore', action='start', rolling_restart=rolling_restart) - def stop(self, env, rolling_restart=False): + # writing configurations on start required for securtity + self.configure(env) + + hive_service('metastore', action='start', upgrade_type=upgrade_type) + + + def stop(self, env, upgrade_type=None): import params env.set_params(params) - hive_service('metastore', action='stop') + hive_service('metastore', action='stop', upgrade_type=upgrade_type) + def configure(self, env): import params @@ -78,6 +84,7 @@ class HiveMetastoreDefault(HiveMetastore): def get_stack_to_component(self): return {"HDP": "hive-metastore"} + def status(self, env): import status_params from resource_management.libraries.functions import check_process_status @@ -87,18 +94,23 @@ class HiveMetastoreDefault(HiveMetastore): # Recursively check all existing gmetad pid files check_process_status(pid_file) - def pre_rolling_restart(self, env): - Logger.info("Executing Metastore Rolling Upgrade pre-restart") + + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing Metastore Stack Upgrade pre-restart") import params env.set_params(params) if Script.is_hdp_stack_greater_or_equal("2.3"): + # ensure that configurations are written out before trying to upgrade the schema + # since the schematool needs configs and doesn't know how to use the hive conf override + self.configure(env) self.upgrade_schema(env) if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: conf_select.select(params.stack_name, "hive", params.version) hdp_select.select("hive-metastore", params.version) + def security_status(self, env): import status_params env.set_params(status_params) @@ -160,7 +172,7 @@ class HiveMetastoreDefault(HiveMetastore): databases. During an upgrade, it's possible that the library is not present, so this will also attempt to copy/download the appropriate driver. """ - Logger.info("Upgrading Hive Metastore") + Logger.info("Upgrading Hive Metastore Schema") import params env.set_params(params) http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py index d7542e9..96c05ca 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py @@ -36,6 +36,7 @@ if OSCheck.is_windows_family(): from resource_management.libraries.functions.windows_service_utils import check_windows_service_status from setup_ranger_hive import setup_ranger_hive from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons.constants import UPGRADE_TYPE_ROLLING from resource_management.core.logger import Logger import hive_server_upgrade @@ -77,26 +78,31 @@ class HiveServerDefault(HiveServer): def get_stack_to_component(self): return {"HDP": "hive-server2"} - def start(self, env, rolling_restart=False): + def start(self, env, upgrade_type=None): import params env.set_params(params) self.configure(env) # FOR SECURITY - setup_ranger_hive(rolling_upgrade=rolling_restart) - hive_service( 'hiveserver2', action = 'start', rolling_restart=rolling_restart) - if rolling_restart: + setup_ranger_hive(upgrade_type=upgrade_type) + hive_service('hiveserver2', action = 'start', upgrade_type=upgrade_type) + + # only perform this if upgrading and rolling; a non-rolling upgrade doesn't need + # to do this since hive is already down + if upgrade_type == UPGRADE_TYPE_ROLLING: hive_server_upgrade.post_upgrade_deregister() - def stop(self, env, rolling_restart=False): + + def stop(self, env, upgrade_type=None): import params env.set_params(params) # During rolling upgrade, HiveServer2 should not be stopped before new server is available. # Once new server is started, old one is stopped by the --deregister command which is # invoked by the 'hive_server_upgrade.post_upgrade_deregister()' method - if not rolling_restart: + if upgrade_type != UPGRADE_TYPE_ROLLING: hive_service( 'hiveserver2', action = 'stop' ) + def status(self, env): import status_params env.set_params(status_params) @@ -105,8 +111,9 @@ class HiveServerDefault(HiveServer): # Recursively check all existing gmetad pid files check_process_status(pid_file) - def pre_rolling_restart(self, env): - Logger.info("Executing HiveServer2 Rolling Upgrade pre-restart") + + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing Hive Server Stack Upgrade pre-restart") import params env.set_params(params) @@ -120,14 +127,17 @@ class HiveServerDefault(HiveServer): params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) + resource_created = copy_to_hdfs( "tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) or resource_created + if resource_created: params.HdfsResource(None, action="execute") + def security_status(self, env): import status_params env.set_params(status_params) http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py index ccb4f7d..456c018 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py @@ -21,6 +21,7 @@ limitations under the License. import os import time +from ambari_commons.constants import UPGRADE_TYPE_ROLLING from resource_management.core.logger import Logger from resource_management.core import shell from resource_management.libraries.functions.format import format @@ -35,7 +36,7 @@ from ambari_commons import OSConst @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) -def hive_service(name, action='start', rolling_restart=False): +def hive_service(name, action='start', upgrade_type=None): import params if name == 'metastore': if action == 'start' or action == 'stop': @@ -47,7 +48,7 @@ def hive_service(name, action='start', rolling_restart=False): @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) -def hive_service(name, action='start', rolling_restart=False): +def hive_service(name, action='start', upgrade_type=None): import params @@ -72,10 +73,10 @@ def hive_service(name, action='start', rolling_restart=False): # upgrading hiveserver2 (rolling_restart) means that there is an existing, # de-registering hiveserver2; the pid will still exist, but the new # hiveserver is spinning up on a new port, so the pid will be re-written - if rolling_restart: + if upgrade_type == UPGRADE_TYPE_ROLLING: process_id_exists_command = None - if (params.version): + if params.version: import os hadoop_home = format("/usr/hdp/{version}/hadoop") hive_bin = os.path.join(params.hive_bin, hive_bin) http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py index 5fdaa70..c17def0 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py @@ -19,7 +19,7 @@ limitations under the License. """ from resource_management.core.logger import Logger -def setup_ranger_hive(rolling_upgrade = False): +def setup_ranger_hive(upgrade_type = None): import params if params.has_ranger_admin: @@ -30,7 +30,8 @@ def setup_ranger_hive(rolling_upgrade = False): from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin hdp_version = None - if rolling_upgrade: + + if upgrade_type is not None: hdp_version = params.version if params.retryAble: http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py index 412edc7..f0f31a8 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py @@ -35,13 +35,13 @@ class WebHCatServer(Script): import params self.install_packages(env, exclude_packages=params.hive_exclude_packages) - def start(self, env, rolling_restart=False): + def start(self, env, upgrade_type=None): import params env.set_params(params) self.configure(env) # FOR SECURITY - webhcat_service(action='start', rolling_restart=rolling_restart) + webhcat_service(action='start', upgrade_type=upgrade_type) - def stop(self, env, rolling_restart=False): + def stop(self, env, upgrade_type=None): import params env.set_params(params) webhcat_service(action='stop') @@ -70,8 +70,8 @@ class WebHCatServerDefault(WebHCatServer): env.set_params(status_params) check_process_status(status_params.webhcat_pid_file) - def pre_rolling_restart(self, env): - Logger.info("Executing WebHCat Rolling Upgrade pre-restart") + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing WebHCat Stack Upgrade pre-restart") import params env.set_params(params) http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py index 46ad109..2553c70 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py @@ -31,7 +31,7 @@ def webhcat_service(action='start', rolling_restart=False): @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) -def webhcat_service(action='start', rolling_restart=False): +def webhcat_service(action='start', upgrade_type=None): import params environ = { @@ -41,7 +41,7 @@ def webhcat_service(action='start', rolling_restart=False): cmd = format('{webhcat_bin_dir}/webhcat_server.sh') if action == 'start': - if rolling_restart and params.version: + if upgrade_type is not None and params.version: environ['HADOOP_HOME'] = format("/usr/hdp/{version}/hadoop") daemon_cmd = format('cd {hcat_pid_dir} ; {cmd} start') http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml index 0f6b8f1..25fd6ab 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml @@ -246,7 +246,7 @@ - + http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml index b5add07..44413d3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml @@ -381,7 +381,7 @@ - + http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml index c091bb7..d3e209e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml @@ -610,7 +610,7 @@ - + http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index fe8e1fe..618b60a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -494,7 +494,7 @@ - + hive-site http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py index 97898b8..d600a65 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py @@ -175,7 +175,7 @@ class TestHiveClient(RMFTestCase): ) self.assertNoMoreResources() - def test_pre_rolling_restart(self): + def test_pre_upgrade_restart(self): config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) @@ -183,7 +183,7 @@ class TestHiveClient(RMFTestCase): json_content['commandParams']['version'] = version self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_client.py", classname = "HiveClient", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES) @@ -192,7 +192,7 @@ class TestHiveClient(RMFTestCase): self.assertNoMoreResources() @patch("resource_management.core.shell.call") - def test_pre_rolling_restart_23(self, call_mock): + def test_pre_upgrade_restart_23(self, call_mock): config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) @@ -202,7 +202,7 @@ class TestHiveClient(RMFTestCase): mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_client.py", classname = "HiveClient", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py index 7f3efcb..517d71c 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py @@ -364,6 +364,8 @@ class TestHiveMetastore(RMFTestCase): @patch("resource_management.core.shell.call") @patch("resource_management.libraries.functions.get_hdp_version") def test_start_ru(self, call_mock, get_hdp_version_mock): + from ambari_commons.constants import UPGRADE_TYPE_ROLLING + get_hdp_version_mock.return_value = '2.3.0.0-1234' config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" @@ -380,7 +382,7 @@ class TestHiveMetastore(RMFTestCase): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py", classname = "HiveMetastore", command = "start", - command_args = [True], + command_args = [UPGRADE_TYPE_ROLLING], config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES) http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py index 14ed232..51bc3a2 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py @@ -890,7 +890,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b""" @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=True)) @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") - def test_pre_rolling_restart(self, copy_to_hdfs_mock): + def test_pre_upgrade_restart(self, copy_to_hdfs_mock): copy_to_hdfs_mock.return_value = True config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" @@ -900,7 +900,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b""" json_content['commandParams']['version'] = version self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py", classname = "HiveServer", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES) @@ -925,7 +925,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b""" @patch("resource_management.core.shell.call") @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=True)) @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") - def test_pre_rolling_restart_23(self, copy_to_hdfs_mock, call_mock): + def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock, call_mock): config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) @@ -936,7 +936,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b""" mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py", classname = "HiveServer", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py index b0b34bc..b9b1ce9 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py @@ -343,7 +343,7 @@ class TestWebHCatServer(RMFTestCase): put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) - def test_pre_rolling_restart(self): + def test_pre_upgrade_restart(self): config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) @@ -351,7 +351,7 @@ class TestWebHCatServer(RMFTestCase): json_content['commandParams']['version'] = version self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/webhcat_server.py", classname = "WebHCatServer", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES) @@ -360,7 +360,7 @@ class TestWebHCatServer(RMFTestCase): self.assertNoMoreResources() @patch("resource_management.core.shell.call") - def test_pre_rolling_restart_23(self, call_mock): + def test_pre_upgrade_restart_23(self, call_mock): import sys config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" @@ -373,7 +373,7 @@ class TestWebHCatServer(RMFTestCase): mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/webhcat_server.py", classname = "WebHCatServer", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index 40f0b3b..2800224 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -483,7 +483,7 @@ class TestHiveMetastore(RMFTestCase): ) put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) - def test_pre_rolling_restart(self): + def test_pre_upgrade_restart(self): config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) @@ -491,7 +491,7 @@ class TestHiveMetastore(RMFTestCase): json_content['commandParams']['version'] = version self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py", classname = "HiveMetastore", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES) @@ -500,7 +500,7 @@ class TestHiveMetastore(RMFTestCase): self.assertNoMoreResources() @patch("resource_management.core.shell.call") - def test_pre_rolling_restart_23(self, call_mock): + def test_pre_upgrade_restart_23(self, call_mock): config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" with open(config_file, "r") as f: json_content = json.load(f) @@ -510,7 +510,7 @@ class TestHiveMetastore(RMFTestCase): mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py", classname = "HiveMetastore", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, @@ -535,12 +535,13 @@ class TestHiveMetastore(RMFTestCase): @patch("resource_management.core.shell.call") @patch("resource_management.libraries.functions.get_hdp_version") def test_upgrade_metastore_schema(self, get_hdp_version_mock, call_mock, os_path_exists_mock): - get_hdp_version_mock.return_value = '2.3.0.0-1234' def side_effect(path): if path == "/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar": return True + if ".j2" in path: + return True return False os_path_exists_mock.side_effect = side_effect @@ -556,7 +557,6 @@ class TestHiveMetastore(RMFTestCase): json_content['hostLevelParams']['stack_version'] = "2.3" json_content['hostLevelParams']['current_version'] = "2.2.7.0-1234" - # trigger the code to think it needs to copy the JAR json_content['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName'] = "com.mysql.jdbc.Driver" json_content['configurations']['hive-env']['hive_database'] = "Existing" @@ -564,13 +564,17 @@ class TestHiveMetastore(RMFTestCase): mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py", classname = "HiveMetastore", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, call_mocks = [(0, None), (0, None)], mocks_dict = mocks_dict) + # we don't care about configure here - the strings are different anyway because this + # is an upgrade, so just pop those resources off of the call stack + self.pop_resources(21) + self.assertResourceCalled('Execute', ('rm', '-f', '/usr/hdp/current/hive-server2/lib/ojdbc6.jar'), path = ['/bin', '/usr/bin/'], sudo = True, @@ -619,7 +623,14 @@ class TestHiveMetastore(RMFTestCase): get_hdp_version_mock.return_value = '2.3.0.0-1234' - os_path_exists_mock.return_value = False + def side_effect(path): + if path == "/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar": + return True + if ".j2" in path: + return True + return False + + os_path_exists_mock.side_effect = side_effect config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json" @@ -639,13 +650,17 @@ class TestHiveMetastore(RMFTestCase): mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py", classname = "HiveMetastore", - command = "pre_rolling_restart", + command = "pre_upgrade_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, call_mocks = [(0, None), (0, None)], mocks_dict = mocks_dict) + # we don't care about configure here - the strings are different anyway because this + # is an upgrade, so just pop those resources off of the call stack + self.pop_resources(25) + self.assertResourceCalled('Execute', ('rm', '-f', '/usr/hdp/current/hive-server2/lib/ojdbc6.jar'), path=["/bin", "/usr/bin/"], http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/utils/RMFTestCase.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py index a186527..ab4eed4 100644 --- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py +++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py @@ -223,7 +223,13 @@ class RMFTestCase(TestCase): s = self.reindent(s, intendation) print s print(self.reindent("self.assertNoMoreResources()", intendation)) - + + def pop_resources(self, count): + with patch.object(UnknownConfiguration, '__getattr__', return_value=lambda: "UnknownConfiguration()"): + self.assertNotEqual(len(RMFTestCase.env.resource_list), 0, "There was no more resources executed!") + for i in range(count): + RMFTestCase.env.resource_list.pop(0) + def assertResourceCalled(self, resource_type, name, **kwargs): with patch.object(UnknownConfiguration, '__getattr__', return_value=lambda: "UnknownConfiguration()"): self.assertNotEqual(len(RMFTestCase.env.resource_list), 0, "There was no more resources executed!")