ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-13459 - Hive Metastore Upgrade Retry Fails Because Of Driver Copy Issue (jonathanhurley)
Date Fri, 16 Oct 2015 20:07:57 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 4023dcf9e -> d8d19268a


AMBARI-13459 - Hive Metastore Upgrade Retry Fails Because Of Driver Copy Issue (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d8d19268
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d8d19268
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d8d19268

Branch: refs/heads/trunk
Commit: d8d19268a4c7078216d3a1dd3d589d9ff0cf23bb
Parents: 4023dcf
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Fri Oct 16 14:09:26 2015 -0400
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Fri Oct 16 16:07:50 2015 -0400

----------------------------------------------------------------------
 .../package/scripts/hive_metastore.py           | 30 +++++++++++++-------
 .../stacks/2.1/HIVE/test_hive_metastore.py      |  8 ++++--
 2 files changed, 24 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/d8d19268/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
index 796ec18..8a3833b 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
@@ -33,7 +33,6 @@ from resource_management.libraries.functions.security_commons import get_params_
 from resource_management.libraries.functions.security_commons import validate_security_config_properties
 from resource_management.libraries.functions.security_commons import FILE_TYPE_XML
 from resource_management.core.resources.system import File
-from resource_management.core.shell import as_sudo
 
 from hive import hive
 from hive import jdbc_connector
@@ -155,7 +154,7 @@ class HiveMetastoreDefault(HiveMetastore):
     """
     Executes the schema upgrade binary.  This is its own function because it could
     be called as a standalone task from the upgrade pack, but is safe to run it for each
-    metastore instance.
+    metastore instance. The schema upgrade on an already upgraded metastore is a NOOP.
 
     The metastore schema upgrade requires a database driver library for most
     databases. During an upgrade, it's possible that the library is not present,
@@ -173,28 +172,36 @@ class HiveMetastoreDefault(HiveMetastore):
     # present, then download it first
     if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
       target_directory = format("/usr/hdp/{version}/hive/lib")
-      if not os.path.exists(params.target):
-        # download it
+
+      # normally, the JDBC driver would be referenced by /usr/hdp/current/.../foo.jar
+      # but if hdp-select is called and the restart fails, then this means that current pointer
+      # is now pointing to the upgraded version location; that's bad for the cp command
+      source_jdbc_file = format(params.target.replace("/usr/hdp/current", "/usr/hdp/{current_version}"))
+
+      # download it if it does not exist
+      if not os.path.exists(source_jdbc_file):
         jdbc_connector()
 
+      target_directory_and_filename = os.path.join(target_directory, os.path.basename(source_jdbc_file))
+
       if params.sqla_db_used:
         target_native_libs_directory = format("{target_directory}/native/lib64")
 
         Execute(format("yes | {sudo} cp {jars_in_hive_lib} {target_directory}"))
 
-        Directory(target_native_libs_directory,
-                  recursive=True)
+        Directory(target_native_libs_directory, recursive=True)
 
         Execute(format("yes | {sudo} cp {libs_in_hive_lib} {target_native_libs_directory}"))
 
         Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"))
       else:
-        Execute(('cp', params.target, target_directory),
-                path=["/bin", "/usr/bin/"], sudo = True)
+        # copy the JDBC driver from the older metastore location to the new location only
+        # if it does not already exist
+        if not os.path.exists(target_directory_and_filename):
+          Execute(('cp', source_jdbc_file, target_directory),
+            path=["/bin", "/usr/bin/"], sudo = True)
 
-      File(os.path.join(target_directory, os.path.basename(params.target)),
-        mode = 0644,
-      )
+      File(target_directory_and_filename, mode = 0644)
 
     # build the schema tool command
     binary = format("/usr/hdp/{version}/hive/bin/schematool")
@@ -215,5 +222,6 @@ class HiveMetastoreDefault(HiveMetastore):
     command = format("{binary} -dbType {hive_metastore_db_type} -upgradeSchema")
     Execute(command, user=params.hive_user, tries=1, environment=env_dict, logoutput=True)
 
+
 if __name__ == "__main__":
   HiveMetastore().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/d8d19268/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
index a51f139..5e9fc1d 100644
--- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
+++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
@@ -539,7 +539,7 @@ class TestHiveMetastore(RMFTestCase):
     get_hdp_version_mock.return_value = '2.3.0.0-1234'
 
     def side_effect(path):
-      if path == "/usr/hdp/current/hive-server2/lib/mysql-connector-java.jar":
+      if path == "/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar":
         return True
       return False
 
@@ -554,6 +554,8 @@ class TestHiveMetastore(RMFTestCase):
     version = '2.3.0.0-1234'
     json_content['commandParams']['version'] = version
     json_content['hostLevelParams']['stack_version'] = "2.3"
+    json_content['hostLevelParams']['current_version'] = "2.2.7.0-1234"
+
 
     # trigger the code to think it needs to copy the JAR
     json_content['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName']
= "com.mysql.jdbc.Driver"
@@ -570,7 +572,7 @@ class TestHiveMetastore(RMFTestCase):
       mocks_dict = mocks_dict)
 
     self.assertResourceCalled('Execute',
-      ('cp', '/usr/hdp/current/hive-server2/lib/mysql-connector-java.jar', '/usr/hdp/2.3.0.0-1234/hive/lib'),
+      ('cp', '/usr/hdp/2.2.7.0-1234/hive-server2/lib/mysql-connector-java.jar', '/usr/hdp/2.3.0.0-1234/hive/lib'),
       path = ['/bin', '/usr/bin/'], sudo = True)
 
     self.assertResourceCalled('File', '/usr/hdp/2.3.0.0-1234/hive/lib/mysql-connector-java.jar',
@@ -578,7 +580,7 @@ class TestHiveMetastore(RMFTestCase):
     )
 
     self.assertResourceCalled('Execute', "/usr/hdp/2.3.0.0-1234/hive/bin/schematool -dbType
mysql -upgradeSchema",
-     logoutput = True, environment = {'HIVE_CONF_DIR': '/usr/hdp/current/hive-server2/conf/conf.server'},
+     logoutput = True, environment = {'HIVE_CONF_DIR': '/etc/hive/conf.server'},
       tries = 1, user = 'hive')
 
     self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set',
'hive-metastore', version), sudo=True,)


Mime
View raw message