ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-13588 - Express Upgrade Packs For Hive (jonathanhurley)
Date Wed, 28 Oct 2015 04:56:59 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 9e085dfd0 -> deb782c79


AMBARI-13588 - Express Upgrade Packs For Hive (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/deb782c7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/deb782c7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/deb782c7

Branch: refs/heads/trunk
Commit: deb782c796cf4ee792f1f1d98769583698b89f60
Parents: 9e085df
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Tue Oct 27 16:15:02 2015 -0400
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Wed Oct 28 00:56:28 2015 -0400

----------------------------------------------------------------------
 .../src/main/python/ambari_commons/constants.py |  9 ++++--
 .../libraries/script/script.py                  |  5 +--
 .../0.12.0.2.0/package/scripts/hive_client.py   |  4 ++-
 .../package/scripts/hive_metastore.py           | 28 ++++++++++++-----
 .../0.12.0.2.0/package/scripts/hive_server.py   | 26 ++++++++++-----
 .../0.12.0.2.0/package/scripts/hive_service.py  |  9 +++---
 .../package/scripts/setup_ranger_hive.py        |  5 +--
 .../package/scripts/webhcat_server.py           | 10 +++---
 .../package/scripts/webhcat_service.py          |  4 +--
 .../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml |  2 +-
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml |  2 +-
 .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml     |  2 +-
 .../stacks/HDP/2.3/upgrades/config-upgrade.xml  |  2 +-
 .../stacks/2.0.6/HIVE/test_hive_client.py       |  8 ++---
 .../stacks/2.0.6/HIVE/test_hive_metastore.py    |  4 ++-
 .../stacks/2.0.6/HIVE/test_hive_server.py       |  8 ++---
 .../stacks/2.0.6/HIVE/test_webhcat_server.py    |  8 ++---
 .../stacks/2.1/HIVE/test_hive_metastore.py      | 33 ++++++++++++++------
 .../src/test/python/stacks/utils/RMFTestCase.py |  8 ++++-
 19 files changed, 115 insertions(+), 62 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-common/src/main/python/ambari_commons/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/constants.py b/ambari-common/src/main/python/ambari_commons/constants.py
index 99ff679..525e024 100644
--- a/ambari-common/src/main/python/ambari_commons/constants.py
+++ b/ambari-common/src/main/python/ambari_commons/constants.py
@@ -1,6 +1,5 @@
 #!/usr/bin/env python
-
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -16,6 +15,10 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+
+"""
 
 AMBARI_SUDO_BINARY = "ambari-sudo.sh"
+
+UPGRADE_TYPE_ROLLING = "rolling"
+UPGRADE_TYPE_NON_ROLLING = "nonrolling"

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 79bc535..10a9aa6 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -29,6 +29,7 @@ import platform
 import inspect
 import tarfile
 from ambari_commons import OSCheck, OSConst
+from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from resource_management.libraries.resources import XmlConfig
 from resource_management.libraries.resources import PropertiesFile
@@ -468,9 +469,9 @@ class Script(object):
 
     upgrade_type = None
     if restart_type.lower() == "rolling_upgrade":
-      upgrade_type = "rolling"
+      upgrade_type = UPGRADE_TYPE_ROLLING
     elif restart_type.lower() == "nonrolling_upgrade":
-      upgrade_type = "nonrolling"
+      upgrade_type = UPGRADE_TYPE_NON_ROLLING
 
     is_stack_upgrade = upgrade_type is not None
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py
index 0cacb07..117b456 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_client.py
@@ -50,7 +50,9 @@ class HiveClientDefault(HiveClient):
   def get_stack_to_component(self):
     return {"HDP": "hadoop-client"}
 
-  def pre_rolling_restart(self, env):
+  def pre_upgrade_restart(self, env, upgrade_type=None):
+    Logger.info("Executing Hive client Stack Upgrade pre-restart")
+
     import params
     env.set_params(params)
     if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
>= 0:

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
index 8bdccdc..080225a 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
@@ -48,16 +48,22 @@ class HiveMetastore(Script):
     import params
     self.install_packages(env, exclude_packages = params.hive_exclude_packages)
 
-  def start(self, env, rolling_restart=False):
+
+  def start(self, env, upgrade_type=None):
     import params
     env.set_params(params)
-    self.configure(env)  # FOR SECURITY
-    hive_service('metastore', action='start', rolling_restart=rolling_restart)
 
-  def stop(self, env, rolling_restart=False):
+    # writing configurations on start required for securtity
+    self.configure(env)
+
+    hive_service('metastore', action='start', upgrade_type=upgrade_type)
+
+
+  def stop(self, env, upgrade_type=None):
     import params
     env.set_params(params)
-    hive_service('metastore', action='stop')
+    hive_service('metastore', action='stop', upgrade_type=upgrade_type)
+
 
   def configure(self, env):
     import params
@@ -78,6 +84,7 @@ class HiveMetastoreDefault(HiveMetastore):
   def get_stack_to_component(self):
     return {"HDP": "hive-metastore"}
 
+
   def status(self, env):
     import status_params
     from resource_management.libraries.functions import check_process_status
@@ -87,18 +94,23 @@ class HiveMetastoreDefault(HiveMetastore):
     # Recursively check all existing gmetad pid files
     check_process_status(pid_file)
 
-  def pre_rolling_restart(self, env):
-    Logger.info("Executing Metastore Rolling Upgrade pre-restart")
+
+  def pre_upgrade_restart(self, env, upgrade_type=None):
+    Logger.info("Executing Metastore Stack Upgrade pre-restart")
     import params
     env.set_params(params)
 
     if Script.is_hdp_stack_greater_or_equal("2.3"):
+      # ensure that configurations are written out before trying to upgrade the schema
+      # since the schematool needs configs and doesn't know how to use the hive conf override
+      self.configure(env)
       self.upgrade_schema(env)
 
     if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
>= 0:
       conf_select.select(params.stack_name, "hive", params.version)
       hdp_select.select("hive-metastore", params.version)
 
+
   def security_status(self, env):
     import status_params
     env.set_params(status_params)
@@ -160,7 +172,7 @@ class HiveMetastoreDefault(HiveMetastore):
     databases. During an upgrade, it's possible that the library is not present,
     so this will also attempt to copy/download the appropriate driver.
     """
-    Logger.info("Upgrading Hive Metastore")
+    Logger.info("Upgrading Hive Metastore Schema")
     import params
     env.set_params(params)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
index d7542e9..96c05ca 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
@@ -36,6 +36,7 @@ if OSCheck.is_windows_family():
   from resource_management.libraries.functions.windows_service_utils import check_windows_service_status
 from setup_ranger_hive import setup_ranger_hive
 from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_commons.constants import UPGRADE_TYPE_ROLLING
 from resource_management.core.logger import Logger
 
 import hive_server_upgrade
@@ -77,26 +78,31 @@ class HiveServerDefault(HiveServer):
   def get_stack_to_component(self):
     return {"HDP": "hive-server2"}
 
-  def start(self, env, rolling_restart=False):
+  def start(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     self.configure(env) # FOR SECURITY
 
-    setup_ranger_hive(rolling_upgrade=rolling_restart)
-    hive_service( 'hiveserver2', action = 'start', rolling_restart=rolling_restart)
-    if rolling_restart:
+    setup_ranger_hive(upgrade_type=upgrade_type)
+    hive_service('hiveserver2', action = 'start', upgrade_type=upgrade_type)
+
+    # only perform this if upgrading and rolling; a non-rolling upgrade doesn't need
+    # to do this since hive is already down
+    if upgrade_type == UPGRADE_TYPE_ROLLING:
       hive_server_upgrade.post_upgrade_deregister()
 
-  def stop(self, env, rolling_restart=False):
+
+  def stop(self, env, upgrade_type=None):
     import params
     env.set_params(params)
 
     # During rolling upgrade, HiveServer2 should not be stopped before new server is available.
     # Once new server is started, old one is stopped by the --deregister command which is

     # invoked by the 'hive_server_upgrade.post_upgrade_deregister()' method
-    if not rolling_restart:
+    if upgrade_type != UPGRADE_TYPE_ROLLING:
       hive_service( 'hiveserver2', action = 'stop' )
 
+
   def status(self, env):
     import status_params
     env.set_params(status_params)
@@ -105,8 +111,9 @@ class HiveServerDefault(HiveServer):
     # Recursively check all existing gmetad pid files
     check_process_status(pid_file)
 
-  def pre_rolling_restart(self, env):
-    Logger.info("Executing HiveServer2 Rolling Upgrade pre-restart")
+
+  def pre_upgrade_restart(self, env, upgrade_type=None):
+    Logger.info("Executing Hive Server Stack Upgrade pre-restart")
     import params
     env.set_params(params)
 
@@ -120,14 +127,17 @@ class HiveServerDefault(HiveServer):
         params.user_group,
         params.hdfs_user,
         host_sys_prepped=params.host_sys_prepped)
+
       resource_created = copy_to_hdfs(
         "tez",
         params.user_group,
         params.hdfs_user,
         host_sys_prepped=params.host_sys_prepped) or resource_created
+
       if resource_created:
         params.HdfsResource(None, action="execute")
 
+
   def security_status(self, env):
     import status_params
     env.set_params(status_params)

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
index ccb4f7d..456c018 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
@@ -21,6 +21,7 @@ limitations under the License.
 import os
 import time
 
+from ambari_commons.constants import UPGRADE_TYPE_ROLLING
 from resource_management.core.logger import Logger
 from resource_management.core import shell
 from resource_management.libraries.functions.format import format
@@ -35,7 +36,7 @@ from ambari_commons import OSConst
 
 
 @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-def hive_service(name, action='start', rolling_restart=False):
+def hive_service(name, action='start', upgrade_type=None):
   import params
   if name == 'metastore':
     if action == 'start' or action == 'stop':
@@ -47,7 +48,7 @@ def hive_service(name, action='start', rolling_restart=False):
 
 
 @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def hive_service(name, action='start', rolling_restart=False):
+def hive_service(name, action='start', upgrade_type=None):
 
   import params
 
@@ -72,10 +73,10 @@ def hive_service(name, action='start', rolling_restart=False):
     # upgrading hiveserver2 (rolling_restart) means that there is an existing,
     # de-registering hiveserver2; the pid will still exist, but the new
     # hiveserver is spinning up on a new port, so the pid will be re-written
-    if rolling_restart:
+    if upgrade_type == UPGRADE_TYPE_ROLLING:
       process_id_exists_command = None
 
-      if (params.version):
+      if params.version:
         import os
         hadoop_home = format("/usr/hdp/{version}/hadoop")
         hive_bin = os.path.join(params.hive_bin, hive_bin)

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
index 5fdaa70..c17def0 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/setup_ranger_hive.py
@@ -19,7 +19,7 @@ limitations under the License.
 """
 from resource_management.core.logger import Logger
 
-def setup_ranger_hive(rolling_upgrade = False):
+def setup_ranger_hive(upgrade_type = None):
   import params
 
   if params.has_ranger_admin:
@@ -30,7 +30,8 @@ def setup_ranger_hive(rolling_upgrade = False):
       from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
     
     hdp_version = None
-    if rolling_upgrade:
+
+    if upgrade_type is not None:
       hdp_version = params.version
 
     if params.retryAble:

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py
index 412edc7..f0f31a8 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_server.py
@@ -35,13 +35,13 @@ class WebHCatServer(Script):
     import params
     self.install_packages(env, exclude_packages=params.hive_exclude_packages)
 
-  def start(self, env, rolling_restart=False):
+  def start(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     self.configure(env) # FOR SECURITY
-    webhcat_service(action='start', rolling_restart=rolling_restart)
+    webhcat_service(action='start', upgrade_type=upgrade_type)
 
-  def stop(self, env, rolling_restart=False):
+  def stop(self, env, upgrade_type=None):
     import params
     env.set_params(params)
     webhcat_service(action='stop')
@@ -70,8 +70,8 @@ class WebHCatServerDefault(WebHCatServer):
     env.set_params(status_params)
     check_process_status(status_params.webhcat_pid_file)
 
-  def pre_rolling_restart(self, env):
-    Logger.info("Executing WebHCat Rolling Upgrade pre-restart")
+  def pre_upgrade_restart(self, env, upgrade_type=None):
+    Logger.info("Executing WebHCat Stack Upgrade pre-restart")
     import params
     env.set_params(params)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py
index 46ad109..2553c70 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat_service.py
@@ -31,7 +31,7 @@ def webhcat_service(action='start', rolling_restart=False):
 
 
 @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def webhcat_service(action='start', rolling_restart=False):
+def webhcat_service(action='start', upgrade_type=None):
   import params
 
   environ = {
@@ -41,7 +41,7 @@ def webhcat_service(action='start', rolling_restart=False):
   cmd = format('{webhcat_bin_dir}/webhcat_server.sh')
 
   if action == 'start':
-    if rolling_restart and params.version:
+    if upgrade_type is not None and params.version:
       environ['HADOOP_HOME'] = format("/usr/hdp/{version}/hadoop")
 
     daemon_cmd = format('cd {hcat_pid_dir} ; {cmd} start')

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
index 0f6b8f1..25fd6ab 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
@@ -246,7 +246,7 @@
       </execute-stage>
 
       <execute-stage service="HIVE" component="HIVE_SERVER">
-        <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentification"/>
+        <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/>
       </execute-stage>
 
       <execute-stage service="HIVE" component="WEBHCAT_SERVER">

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
index b5add07..44413d3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
@@ -381,7 +381,7 @@
       </execute-stage>
 
       <execute-stage service="HIVE" component="HIVE_SERVER">
-        <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentification"/>
+        <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/>
       </execute-stage>
 
       <execute-stage service="HIVE" component="HIVE_SERVER">

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index c091bb7..d3e209e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -610,7 +610,7 @@
           
           <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_replace_auth_manager"/>
 
-          <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentification"/>
+          <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"/>
 
           <task xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_ranger_policy"/>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index fe8e1fe..618b60a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -494,7 +494,7 @@
             <replace key="hive.security.authorization.manager" find="com.xasecure.authorization.hive.authorizer.XaSecureHiveAuthorizerFactory"
replace-with="org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory"
/>
           </definition>
 
-          <definition xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentification"
summary="Configuring hive authentication">
+          <definition xsi:type="configure" id="hdp_2_3_0_0_hive_server_configure_authentication"
summary="Configuring hive authentication">
             <type>hive-site</type>
             <transfer operation="delete" delete-key="hive.server2.authentication.ldap.url"
if-key="hive.server2.authentication" if-type="hive-site" if-value="NONE"/>
             <transfer operation="delete" delete-key="hive.server2.authentication.ldap.baseDN"
if-key="hive.server2.authentication" if-type="hive-site" if-value="NONE"/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
index 97898b8..d600a65 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
@@ -175,7 +175,7 @@ class TestHiveClient(RMFTestCase):
     )
     self.assertNoMoreResources()
 
-  def test_pre_rolling_restart(self):
+  def test_pre_upgrade_restart(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -183,7 +183,7 @@ class TestHiveClient(RMFTestCase):
     json_content['commandParams']['version'] = version
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_client.py",
                        classname = "HiveClient",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -192,7 +192,7 @@ class TestHiveClient(RMFTestCase):
     self.assertNoMoreResources()
 
   @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart_23(self, call_mock):
+  def test_pre_upgrade_restart_23(self, call_mock):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -202,7 +202,7 @@ class TestHiveClient(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_client.py",
                        classname = "HiveClient",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
index 7f3efcb..517d71c 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
@@ -364,6 +364,8 @@ class TestHiveMetastore(RMFTestCase):
   @patch("resource_management.core.shell.call")
   @patch("resource_management.libraries.functions.get_hdp_version")
   def test_start_ru(self, call_mock, get_hdp_version_mock):
+    from ambari_commons.constants import UPGRADE_TYPE_ROLLING
+
     get_hdp_version_mock.return_value = '2.3.0.0-1234'
 
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
@@ -380,7 +382,7 @@ class TestHiveMetastore(RMFTestCase):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
                        classname = "HiveMetastore",
                        command = "start",
-                       command_args = [True],
+                       command_args = [UPGRADE_TYPE_ROLLING],
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index 14ed232..51bc3a2 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -890,7 +890,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
 
   @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=True))
   @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
-  def test_pre_rolling_restart(self, copy_to_hdfs_mock):
+  def test_pre_upgrade_restart(self, copy_to_hdfs_mock):
     copy_to_hdfs_mock.return_value = True
 
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
@@ -900,7 +900,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
     json_content['commandParams']['version'] = version
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
                        classname = "HiveServer",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -925,7 +925,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
   @patch("resource_management.core.shell.call")
   @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=True))
   @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
-  def test_pre_rolling_restart_23(self, copy_to_hdfs_mock, call_mock):
+  def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock, call_mock):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -936,7 +936,7 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
                        classname = "HiveServer",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
index b0b34bc..b9b1ce9 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
@@ -343,7 +343,7 @@ class TestWebHCatServer(RMFTestCase):
     put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
 
 
-  def test_pre_rolling_restart(self):
+  def test_pre_upgrade_restart(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -351,7 +351,7 @@ class TestWebHCatServer(RMFTestCase):
     json_content['commandParams']['version'] = version
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/webhcat_server.py",
                        classname = "WebHCatServer",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -360,7 +360,7 @@ class TestWebHCatServer(RMFTestCase):
     self.assertNoMoreResources()
 
   @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart_23(self, call_mock):
+  def test_pre_upgrade_restart_23(self, call_mock):
     import sys
 
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
@@ -373,7 +373,7 @@ class TestWebHCatServer(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/webhcat_server.py",
                        classname = "WebHCatServer",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
index 40f0b3b..2800224 100644
--- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
+++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
@@ -483,7 +483,7 @@ class TestHiveMetastore(RMFTestCase):
     )
     put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
 
-  def test_pre_rolling_restart(self):
+  def test_pre_upgrade_restart(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -491,7 +491,7 @@ class TestHiveMetastore(RMFTestCase):
     json_content['commandParams']['version'] = version
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
                        classname = "HiveMetastore",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
@@ -500,7 +500,7 @@ class TestHiveMetastore(RMFTestCase):
     self.assertNoMoreResources()
 
   @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart_23(self, call_mock):
+  def test_pre_upgrade_restart_23(self, call_mock):
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
@@ -510,7 +510,7 @@ class TestHiveMetastore(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
                        classname = "HiveMetastore",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,
@@ -535,12 +535,13 @@ class TestHiveMetastore(RMFTestCase):
   @patch("resource_management.core.shell.call")
   @patch("resource_management.libraries.functions.get_hdp_version")
   def test_upgrade_metastore_schema(self, get_hdp_version_mock, call_mock, os_path_exists_mock):
-
     get_hdp_version_mock.return_value = '2.3.0.0-1234'
 
     def side_effect(path):
       if path == "/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar":
         return True
+      if ".j2" in path:
+        return True
       return False
 
     os_path_exists_mock.side_effect = side_effect
@@ -556,7 +557,6 @@ class TestHiveMetastore(RMFTestCase):
     json_content['hostLevelParams']['stack_version'] = "2.3"
     json_content['hostLevelParams']['current_version'] = "2.2.7.0-1234"
 
-
     # trigger the code to think it needs to copy the JAR
     json_content['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName']
= "com.mysql.jdbc.Driver"
     json_content['configurations']['hive-env']['hive_database'] = "Existing"
@@ -564,13 +564,17 @@ class TestHiveMetastore(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
       classname = "HiveMetastore",
-      command = "pre_rolling_restart",
+      command = "pre_upgrade_restart",
       config_dict = json_content,
       hdp_stack_version = self.STACK_VERSION,
       target = RMFTestCase.TARGET_COMMON_SERVICES,
       call_mocks = [(0, None), (0, None)],
       mocks_dict = mocks_dict)
 
+    # we don't care about configure here - the strings are different anyway because this
+    # is an upgrade, so just pop those resources off of the call stack
+    self.pop_resources(21)
+
     self.assertResourceCalled('Execute', ('rm', '-f', '/usr/hdp/current/hive-server2/lib/ojdbc6.jar'),
         path = ['/bin', '/usr/bin/'],
         sudo = True,
@@ -619,7 +623,14 @@ class TestHiveMetastore(RMFTestCase):
 
     get_hdp_version_mock.return_value = '2.3.0.0-1234'
 
-    os_path_exists_mock.return_value = False
+    def side_effect(path):
+      if path == "/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar":
+        return True
+      if ".j2" in path:
+        return True
+      return False
+
+    os_path_exists_mock.side_effect = side_effect
 
     config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
 
@@ -639,13 +650,17 @@ class TestHiveMetastore(RMFTestCase):
     mocks_dict = {}
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
                        classname = "HiveMetastore",
-                       command = "pre_rolling_restart",
+                       command = "pre_upgrade_restart",
                        config_dict = json_content,
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,
                        call_mocks = [(0, None), (0, None)],
                        mocks_dict = mocks_dict)
 
+    # we don't care about configure here - the strings are different anyway because this
+    # is an upgrade, so just pop those resources off of the call stack
+    self.pop_resources(25)
+
     self.assertResourceCalled('Execute',
                               ('rm', '-f', '/usr/hdp/current/hive-server2/lib/ojdbc6.jar'),
                               path=["/bin", "/usr/bin/"],

http://git-wip-us.apache.org/repos/asf/ambari/blob/deb782c7/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
index a186527..ab4eed4 100644
--- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
+++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
@@ -223,7 +223,13 @@ class RMFTestCase(TestCase):
       s = self.reindent(s, intendation)
       print s
     print(self.reindent("self.assertNoMoreResources()", intendation))
-  
+
+  def pop_resources(self, count):
+    with patch.object(UnknownConfiguration, '__getattr__', return_value=lambda: "UnknownConfiguration()"):
+      self.assertNotEqual(len(RMFTestCase.env.resource_list), 0, "There was no more resources
executed!")
+      for i in range(count):
+        RMFTestCase.env.resource_list.pop(0)
+
   def assertResourceCalled(self, resource_type, name, **kwargs):
     with patch.object(UnknownConfiguration, '__getattr__', return_value=lambda: "UnknownConfiguration()"):
       self.assertNotEqual(len(RMFTestCase.env.resource_list), 0, "There was no more resources
executed!")


Mime
View raw message