ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-14073 - Hive Metastore Downgrade Fails Because Of upgradeSchema Invocation (jonathanhurley)
Date Wed, 25 Nov 2015 19:36:55 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 9a05c0591 -> ebba6caec


AMBARI-14073 - Hive Metastore Downgrade Fails Because Of upgradeSchema Invocation (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ebba6cae
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ebba6cae
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ebba6cae

Branch: refs/heads/trunk
Commit: ebba6caec221b7c915cdd9c0717c1b4d48075420
Parents: 9a05c05
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Wed Nov 25 12:26:58 2015 -0500
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Wed Nov 25 14:36:32 2015 -0500

----------------------------------------------------------------------
 .../package/scripts/hive_metastore.py           | 22 ++++---
 .../0.12.0.2.0/package/scripts/params_linux.py  |  5 +-
 .../stacks/2.1/HIVE/test_hive_metastore.py      | 63 +++++++++++++++-----
 3 files changed, 65 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/ebba6cae/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
index 084494d..d26e36f 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
@@ -24,6 +24,7 @@ from resource_management.core.resources.system import Execute, Directory
 from resource_management.libraries.script import Script
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import hdp_select
+from resource_management.libraries.functions.constants import Direction
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.version import format_hdp_stack_version
 from resource_management.libraries.functions.version import compare_versions
@@ -99,16 +100,12 @@ class HiveMetastoreDefault(HiveMetastore):
     Logger.info("Executing Metastore Stack Upgrade pre-restart")
     import params
 
-    if Script.is_hdp_stack_greater_or_equal("2.3"):
-      # disable schema creation on upgrade
-      params.init_metastore_schema = False
-
     env.set_params(params)
 
-    if Script.is_hdp_stack_greater_or_equal("2.3"):
-      # ensure that configurations are written out before trying to upgrade the schema
-      # since the schematool needs configs and doesn't know how to use the hive conf override
-      self.configure(env)
+    is_stack_hdp_23 = Script.is_hdp_stack_greater_or_equal("2.3")
+    is_upgrade = params.upgrade_direction == Direction.UPGRADE
+
+    if is_stack_hdp_23 and is_upgrade:
       self.upgrade_schema(env)
 
     if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
>= 0:
@@ -176,11 +173,20 @@ class HiveMetastoreDefault(HiveMetastore):
     The metastore schema upgrade requires a database driver library for most
     databases. During an upgrade, it's possible that the library is not present,
     so this will also attempt to copy/download the appropriate driver.
+
+    This function will also ensure that configurations are written out to disk before running
+    since the new configs will most likely not yet exist on an upgrade.
+
+    Should not be invoked for a DOWNGRADE; Metastore only supports schema upgrades.
     """
     Logger.info("Upgrading Hive Metastore Schema")
     import params
     env.set_params(params)
 
+    # ensure that configurations are written out before trying to upgrade the schema
+    # since the schematool needs configs and doesn't know how to use the hive conf override
+    self.configure(env)
+
     if params.security_enabled:
       kinit_command=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};
")
       Execute(kinit_command,user=params.smokeuser)

http://git-wip-us.apache.org/repos/asf/ambari/blob/ebba6cae/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 972d532..f360651 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -318,8 +318,9 @@ mysql_host = config['clusterHostInfo']['hive_mysql_host']
 mysql_adduser_path = format("{tmp_dir}/addMysqlUser.sh")
 mysql_deluser_path = format("{tmp_dir}/removeMysqlUser.sh")
 
-######## Metastore Schema
-init_metastore_schema = True
+#### Metastore
+# initialize the schema only if not in an upgrade/downgrade
+init_metastore_schema = upgrade_direction is None
 
 ########## HCAT
 hcat_dbroot = hcat_lib

http://git-wip-us.apache.org/repos/asf/ambari/blob/ebba6cae/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
index bdc0fe2..be3a956 100644
--- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
+++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
@@ -23,6 +23,7 @@ import os
 
 from mock.mock import MagicMock, call, patch
 from stacks.utils.RMFTestCase import *
+from resource_management.libraries.functions.constants import Direction
 from resource_management.libraries.script.script import Script
 
 @patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
@@ -534,31 +535,60 @@ class TestHiveMetastore(RMFTestCase):
       ('ambari-python-wrap', '/usr/bin/conf-select', 'create-conf-dir', '--package', 'hive',
'--stack-version', '2.3.0.0-1234', '--conf-version', '0'),
        mocks_dict['call'].call_args_list[0][0][0])
 
-  @patch.object(Script, "is_hdp_stack_greater_or_equal")
-  def test_pre_upgrade_restart_ims(self, is_hdp_stack_greater_or_equal_mock):
+  def test_pre_upgrade_restart_ims(self):
     """
     Tests the state of the init_metastore_schema property on update
     """
-    is_hdp_stack_greater_or_equal_mock.side_effect = [False, False, False, False, False,
False, True, False, False, False, False, False, False, False]
     config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
 
-    stack_version = '2.3.0.0-0'
-    version = '2.3.0.0-1234'
+    # first try it with a normal, non-upgrade
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
+      classname = "HiveMetastore",
+      command = "pre_upgrade_restart",
+      config_dict = json_content,
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES,
+      call_mocks = [(0, None, ''), (0, None)])
 
-    json_content['commandParams']['version'] = version
-    json_content['hostLevelParams']['stack_version'] = stack_version
+    self.assertEquals(True, RMFTestCase.env.config["params"]["init_metastore_schema"])
 
-    mocks_dict = {}
+    self.config_dict = None
+    config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+
+    json_content['commandParams']['version'] = '2.3.0.0-1234'
+    json_content['commandParams']['upgrade_direction'] = Direction.UPGRADE
+    json_content['hostLevelParams']['stack_version'] = '2.3.0.0-0'
+
+    # now try it in an upgrade
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
-                       classname = "HiveMetastore",
-                       command = "pre_upgrade_restart",
-                       config_dict = json_content,
-                       hdp_stack_version = self.STACK_VERSION,
-                       target = RMFTestCase.TARGET_COMMON_SERVICES,
-                       call_mocks = [(0, None, ''), (0, None)],
-                       mocks_dict = mocks_dict)
+      classname = "HiveMetastore",
+      command = "pre_upgrade_restart",
+      config_dict = json_content,
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES,
+      call_mocks = [(0, None, ''), (0, None)])
+
+    self.assertEquals(False, RMFTestCase.env.config["params"]["init_metastore_schema"])
+
+    self.config_dict = None
+    config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+
+    json_content['commandParams']['upgrade_direction'] = Direction.DOWNGRADE
+
+    # now try it in a downgrade
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
+      classname = "HiveMetastore",
+      command = "pre_upgrade_restart",
+      config_dict = json_content,
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES,
+      call_mocks = [(0, None, ''), (0, None)])
 
     self.assertEquals(False, RMFTestCase.env.config["params"]["init_metastore_schema"])
 
@@ -586,9 +616,11 @@ class TestHiveMetastore(RMFTestCase):
     # must be HDP 2.3+
     version = '2.3.0.0-1234'
     json_content['commandParams']['version'] = version
+    json_content['commandParams']['upgrade_direction'] = Direction.UPGRADE
     json_content['hostLevelParams']['stack_version'] = "2.3"
     json_content['hostLevelParams']['current_version'] = "2.2.7.0-1234"
 
+
     # trigger the code to think it needs to copy the JAR
     json_content['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName']
= "com.mysql.jdbc.Driver"
     json_content['configurations']['hive-env']['hive_database'] = "Existing"
@@ -674,6 +706,7 @@ class TestHiveMetastore(RMFTestCase):
     # must be HDP 2.3+
     version = '2.3.0.0-1234'
     json_content['commandParams']['version'] = version
+    json_content['commandParams']['upgrade_direction'] = Direction.UPGRADE
     json_content['hostLevelParams']['stack_version'] = "2.3"
 
     # trigger the code to think it needs to copy the JAR


Mime
View raw message