ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-12279 - Oozie Rolling Upgrade Fails Due To Missing Database JAR (jonathanhurley)
Date Mon, 06 Jul 2015 20:27:57 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 6adee814f -> 2a253c056


AMBARI-12279 - Oozie Rolling Upgrade Fails Due To Missing Database JAR (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2a253c05
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2a253c05
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2a253c05

Branch: refs/heads/trunk
Commit: 2a253c056c9718e64d69b622d405efa17e69fd27
Parents: 6adee81
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Sat Jul 4 20:52:18 2015 -0400
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Mon Jul 6 16:10:12 2015 -0400

----------------------------------------------------------------------
 .../OOZIE/4.0.0.2.0/package/scripts/oozie.py    | 55 ++++++++++-----
 .../package/scripts/oozie_server_upgrade.py     | 24 +++----
 .../stacks/2.0.6/OOZIE/test_oozie_server.py     | 70 ++++++++++++++++++++
 .../stacks/2.2/configs/oozie-upgrade.json       | 10 +--
 4 files changed, 123 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/2a253c05/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index 8441f11..9b5177f 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -220,23 +220,8 @@ def oozie_server_specific():
     sudo = True,
   )
 
-  if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \
-     params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
-     params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
-    File(params.downloaded_custom_connector,
-         content = DownloadSource(params.driver_curl_source),
-    )
-
-
-    Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target),
-            #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide
jdbc
-            path=["/bin", "/usr/bin/"],
-            sudo = True)
-            
-    File ( params.target,
-      owner = params.oozie_user,
-      group = params.user_group
-    )
+  # download the database JAR
+  download_database_library_if_needed()
 
   #falcon el extension
   if params.has_falcon_host:
@@ -289,3 +274,39 @@ def oozie_server_specific():
   Execute(('chown', '-R', format("{oozie_user}:{user_group}"), params.oozie_server_dir),

           sudo=True
   )
+
+def download_database_library_if_needed(target_directory = None):
+  """
+  Downloads the library to use when connecting to the Oozie database, if
+  necessary. The library will be downloaded to 'params.target' unless
+  otherwise specified.
+  :param target_directory: the location where the database library will be
+  downloaded to.
+  :return:
+  """
+  import params
+  jdbc_drivers = ["com.mysql.jdbc.Driver",
+    "com.microsoft.sqlserver.jdbc.SQLServerDriver",
+    "oracle.jdbc.driver.OracleDriver"]
+
+  # check to see if the JDBC driver name is in the list of ones that need to
+  # be downloaded
+  if params.jdbc_driver_name not in jdbc_drivers:
+    return
+
+  # if the target directory is not specified
+  if target_directory is None:
+    target_jar_with_directory = params.target
+  else:
+    # create the full path using the supplied target directory and the JDBC JAR
+    target_jar_with_directory = target_directory + os.path.sep + params.jdbc_driver_jar
+
+  File(params.downloaded_custom_connector,
+    content = DownloadSource(params.driver_curl_source))
+
+  Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target_jar_with_directory),
+    path=["/bin", "/usr/bin/"],
+    sudo = True)
+
+  File(target_jar_with_directory, owner = params.oozie_user,
+    group = params.user_group)

http://git-wip-us.apache.org/repos/asf/ambari/blob/2a253c05/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
index bd423a2..fb19ddf 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
@@ -33,8 +33,8 @@ from resource_management.libraries.functions import hdp_select
 from resource_management.libraries.functions import format_hdp_stack_version
 from resource_management.libraries.functions import tar_archive
 from resource_management.libraries.script.script import Script
-from resource_management.core.resources import File
-from resource_management.core.source import DownloadSource
+
+import oozie
 
 BACKUP_TEMP_DIR = "oozie-upgrade-backup"
 BACKUP_CONF_ARCHIVE = "oozie-conf-backup.tar"
@@ -156,18 +156,7 @@ class OozieUpgrade(Script):
     shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)
 
     # Redownload jdbc driver to a new current location
-    if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \
-      params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
-      params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
-
-      File(params.downloaded_custom_connector,
-        content = DownloadSource(params.driver_curl_source))
-
-      Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target),
-        #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide
jdbc
-        path=["/bin", "/usr/bin/"], sudo = True)
-
-      File(params.target, owner = params.oozie_user, group = params.user_group)
+    oozie.download_database_library_if_needed()
 
 
   @staticmethod
@@ -230,6 +219,13 @@ class OozieUpgrade(Script):
 
     # upgrade oozie DB
     Logger.info('Upgrading the Oozie database...')
+
+    # the database upgrade requires the db driver JAR, but since we have
+    # not yet run hdp-select to upgrade the current points, we have to use
+    # the versioned libext directory as the location[[-vufdtffr,
+    versioned_libext_dir = "/usr/hdp/{0}/oozie/libext".format(stack_version)
+    oozie.download_database_library_if_needed(target_directory=versioned_libext_dir)
+
     database_upgrade_command = "/usr/hdp/{0}/oozie/bin/ooziedb.sh upgrade -run".format(stack_version)
     Execute(database_upgrade_command, user=params.oozie_user, logoutput=True)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/2a253c05/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index 0ce0f20..0759678 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -1153,3 +1153,73 @@ class TestOozieServer(RMFTestCase):
     self.assertResourceCalled('Execute', '/usr/hdp/2.3.0.0-1234/oozie/bin/oozie-setup.sh
sharelib create -fs hdfs://c6401.ambari.apache.org:8020',
       user='oozie', logoutput = True)
 
+
+  def test_upgrade_database_sharelib_existing_mysql(self):
+    """
+    Tests that the upgrade script runs the proper commands before the
+    actual upgrade begins when Oozie is using and external database. This
+    should ensure that the JDBC JAR is copied.
+    :return:
+    """
+    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/oozie-upgrade.json"
+
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+
+    version = '2.3.0.0-1234'
+    json_content['commandParams']['version'] = version
+    json_content['hostLevelParams']['stack_name'] = "HDP"
+
+    # use mysql external database
+    json_content['configurations']['oozie-site']['oozie.service.JPAService.jdbc.driver']
= "com.mysql.jdbc.Driver"
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server_upgrade.py",
+      classname = "OozieUpgrade", command = "upgrade_oozie_database_and_sharelib",
+      config_dict = json_content,
+      hdp_stack_version = self.UPGRADE_STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES )
+
+    self.assertResourceCalled('File', '/tmp/mysql-connector-java.jar',
+      content = DownloadSource('http://c6401.ambari.apache.org:8080/resources//mysql-jdbc-driver.jar')
)
+
+    self.assertResourceCalled('Execute', ('cp', '--remove-destination', '/tmp/mysql-connector-java.jar',
+      '/usr/hdp/2.3.0.0-1234/oozie/libext/mysql-connector-java.jar'),
+      path = ['/bin', '/usr/bin/'], sudo = True)
+
+    self.assertResourceCalled('File', '/usr/hdp/2.3.0.0-1234/oozie/libext/mysql-connector-java.jar',
+      owner = 'oozie', group = 'hadoop' )
+
+    self.assertResourceCalled('Execute', '/usr/hdp/2.3.0.0-1234/oozie/bin/ooziedb.sh upgrade
-run',
+      user = 'oozie', logoutput = True )
+
+    self.assertResourceCalled('HdfsResource', '/user/oozie/share',
+      security_enabled = False,
+      hadoop_bin_dir = '/usr/hdp/2.3.0.0-1234/hadoop/bin',
+      keytab = UnknownConfigurationMock(),
+      default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+      user = 'hdfs',
+      hdfs_site = UnknownConfigurationMock(),
+      kinit_path_local = '/usr/bin/kinit',
+      principal_name = UnknownConfigurationMock(),
+      recursive_chmod = True,
+      owner = 'oozie',
+      group = 'hadoop',
+      hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+      type = 'directory',
+      action = ['create_on_execute'],
+      mode = 0755 )
+
+    self.assertResourceCalled('HdfsResource', None,
+      security_enabled = False,
+      hadoop_bin_dir = '/usr/hdp/2.3.0.0-1234/hadoop/bin',
+      keytab = UnknownConfigurationMock(),
+      default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+      hdfs_site = UnknownConfigurationMock(),
+      kinit_path_local = '/usr/bin/kinit',
+      principal_name = UnknownConfigurationMock(),
+      user = 'hdfs',
+      action = ['execute'],
+      hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf' )
+
+    self.assertResourceCalled('Execute', '/usr/hdp/2.3.0.0-1234/oozie/bin/oozie-setup.sh
sharelib create -fs hdfs://c6401.ambari.apache.org:8020',
+      user='oozie', logoutput = True)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/2a253c05/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
index 1f98793..a2e8b1d 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
@@ -31,14 +31,14 @@
     "clusterName": "c1", 
     "hostname": "c6402.ambari.apache.org", 
     "hostLevelParams": {
-        "jdk_location": "http://hw10897.ix:8080/resources/", 
+        "jdk_location": "http://c6401.ambari.apache.org:8080/resources/",
         "ambari_db_rca_password": "mapred", 
         "java_home": "/usr/jdk64/jdk1.7.0_45",
         "java_version": "8",
-        "ambari_db_rca_url": "jdbc:postgresql://hw10897.ix/ambarirca", 
+        "ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca", 
         "stack_name": "HDP", 
         "custom_command": "RESTART", 
-        "oracle_jdbc_url": "http://hw10897.ix:8080/resources//ojdbc6.jar", 
+        "oracle_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//ojdbc6.jar", 
         "repo_info": "[{\"baseUrl\":\"http://repo.ambari.apache.org/hdp/centos6/HDP-2.2.0.0/\",\"osType\":\"redhat6\",\"repoId\":\"HDP-2.2\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/GA/2.2.0.0\",\"latestBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/GA/2.2.0.0\"},{\"baseUrl\":\"http://repo.ambari.apache.org/hdp/centos6/HDP-UTILS-1.1.0.20/\",\"osType\":\"redhat6\",\"repoId\":\"HDP-UTILS-1.1.0.20\",\"repoName\":\"HDP-UTILS\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6\",\"latestBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6\"}]",

         "group_list": "[\"hadoop\",\"nobody\",\"users\"]", 
         "agentCacheDir": "/var/lib/ambari-agent/cache", 
@@ -49,7 +49,7 @@
         "ambari_db_rca_username": "mapred", 
         "db_driver_filename": "mysql-connector-java.jar", 
         "user_list": "[\"nobody\",\"oozie\",\"hive\",\"mapred\",\"hbase\",\"ambari-qa\",\"zookeeper\",\"tez\",\"hdfs\",\"falcon\",\"yarn\",\"hcat\"]",

-        "mysql_jdbc_url": "http://hw10897.ix:8080/resources//mysql-connector-java.jar"
+        "mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar"
     }, 
     "commandType": "EXECUTION_COMMAND", 
     "roleParams": {
@@ -225,7 +225,7 @@
             "8670"
         ], 
         "ambari_server_host": [
-            "hw10897.ix"
+            "c6401.ambari.apache.org"
         ], 
         "rm_host": [
             "c6402.ambari.apache.org"


Mime
View raw message