ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [07/19] ambari git commit: Cross-stack upgrade, Oozie restart fails with ext-2.2.zip missing error, stack_tools.py is missing get_stack_name in __all__, disable BigInsights in UI (Alejandro Fernandez via smohanty)
Date Tue, 25 Jul 2017 14:35:11 GMT
Cross-stack upgrade, Oozie restart fails with ext-2.2.zip missing error, stack_tools.py is
missing get_stack_name in __all__, disable BigInsights in UI (Alejandro Fernandez via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0ad9d587
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0ad9d587
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0ad9d587

Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 0ad9d587d0d7cf88c53ae08c8ce9d7351be4a505
Parents: 421f3c6
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Fri Jul 21 12:07:34 2017 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Fri Jul 21 12:07:34 2017 -0700

----------------------------------------------------------------------
 .../OOZIE/4.0.0.2.0/package/scripts/oozie.py    | 40 +++++++++++++-------
 .../package/scripts/oozie_server_upgrade.py     | 35 ++++++++++-------
 .../stacks/2.0.6/OOZIE/test_oozie_server.py     | 32 ++++++++++++----
 3 files changed, 74 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad9d587/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index 142e962..aa5bc30 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -215,18 +215,25 @@ def oozie_ownership():
     group = params.user_group
   )
 
-def get_oozie_ext_zip_source_path(upgrade_type, params):
+def get_oozie_ext_zip_source_paths(upgrade_type, params):
   """
-  Get the Oozie ext zip file path from the source stack.
+  Get an ordered list of Oozie ext zip file paths from the source stack.
   :param upgrade_type:  Upgrade type will be None if not in the middle of a stack upgrade.
   :param params: Expected to contain fields for ext_js_path, upgrade_direction, source_stack_name,
and ext_js_file
-  :return: Source path to use for Oozie extension zip file
+  :return: Source paths to use for Oozie extension zip file
   """
   # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip
+  paths = []
   source_ext_js_path = params.ext_js_path
+  # Preferred location used by HDP and BigInsights 4.2.5
   if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE:
     source_ext_js_path = "/usr/share/" + params.source_stack_name.upper() + "-oozie/" + params.ext_js_file
-  return source_ext_js_path
+  paths.append(source_ext_js_path)
+
+  # Alternate location used by BigInsights 4.2.0 when migrating to another stack.
+  paths.append("/var/lib/oozie/" + params.ext_js_file)
+
+  return paths
 
 def oozie_server_specific(upgrade_type):
   import params
@@ -262,16 +269,23 @@ def oozie_server_specific(upgrade_type):
   )
 
   configure_cmds = []
-  # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip
-  source_ext_zip_path = get_oozie_ext_zip_source_path(upgrade_type, params)
-
-  configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir))
-  configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}')))
+  # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
+  source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params)
+  
+  # Copy the first oozie ext-2.2.zip file that is found.
+  # This uses a list to handle the cases when migrating from some versions of BigInsights
to HDP.
+  if source_ext_zip_paths is not None:
+    for source_ext_zip_path in source_ext_zip_paths:
+      if os.path.isfile(source_ext_zip_path):
+        configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir))
+        configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}')))
+
+        Execute(configure_cmds,
+                not_if=no_op_test,
+                sudo=True,
+                )
+        break
   
-  Execute( configure_cmds,
-    not_if  = no_op_test,
-    sudo = True,
-  )
   
   Directory(params.oozie_webapps_conf_dir,
             owner = params.oozie_user,

http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad9d587/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
index 719fb32..eb57c22 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
@@ -86,22 +86,31 @@ class OozieUpgrade(Script):
         raise Fail("There are no files at {0} matching {1}".format(
           hadoop_client_new_lib_dir, hadoop_lzo_pattern))
 
-    # Copy ext ZIP to libext dir
-    # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip
-    source_ext_zip_path = oozie.get_oozie_ext_zip_source_path(upgrade_type, params)
-
     # something like <stack-root>/current/oozie-server/libext/ext-2.2.zip
     oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, params.ext_js_file)
 
-    if not os.path.isfile(source_ext_zip_path):
-      raise Fail("Unable to copy {0} because it does not exist".format(source_ext_zip_path))
-
-    Logger.info("Copying {0} to {1}".format(source_ext_zip_path, params.oozie_libext_dir))
-    Execute(("cp", source_ext_zip_path, params.oozie_libext_dir), sudo=True)
-    Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
-    File(oozie_ext_zip_target_path,
-         mode=0644
-    )
+    # Copy ext ZIP to libext dir
+    # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
+    source_ext_zip_paths = oozie.get_oozie_ext_zip_source_paths(upgrade_type, params)
+
+    found_at_least_one_oozie_ext_file = False
+
+    # Copy the first oozie ext-2.2.zip file that is found.
+    # This uses a list to handle the cases when migrating from some versions of BigInsights
to HDP.
+    if source_ext_zip_paths is not None:
+      for source_ext_zip_path in source_ext_zip_paths:
+        if os.path.isfile(source_ext_zip_path):
+          found_at_least_one_oozie_ext_file = True
+          Logger.info("Copying {0} to {1}".format(source_ext_zip_path, params.oozie_libext_dir))
+          Execute(("cp", source_ext_zip_path, params.oozie_libext_dir), sudo=True)
+          Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path),
sudo=True)
+          File(oozie_ext_zip_target_path,
+               mode=0644
+               )
+          break
+
+    if not found_at_least_one_oozie_ext_file:
+      raise Fail("Unable to find any Oozie source extension files from the following paths
{0}".format(source_ext_zip_paths))
 
     # Redownload jdbc driver to a new current location
     oozie.download_database_library_if_needed()

http://git-wip-us.apache.org/repos/asf/ambari/blob/0ad9d587/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index 426c36a..5ef6ad9 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -44,7 +44,9 @@ class TestOozieServer(RMFTestCase):
 
   @patch.object(shell, "call")
   @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True]))
-  def test_configure_default(self, call_mocks):
+  @patch("os.path.isfile")
+  def test_configure_default(self, isfile_mock, call_mocks):
+    isfile_mock.return_value = True
     call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
                        classname = "OozieServer",
@@ -59,7 +61,10 @@ class TestOozieServer(RMFTestCase):
 
   @patch.object(shell, "call")
   @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, False, True]))
-  def test_configure_default_mysql(self, call_mocks):
+  @patch("os.path.isfile")
+  def test_configure_default_mysql(self, isfile_mock, iscall_mocks):
+    # Mock call when checking if need to copy oozie ext-2.2.zip file
+    isfile_mock.side_effect = [True, False]
     call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
                        classname = "OozieServer",
@@ -281,7 +286,10 @@ class TestOozieServer(RMFTestCase):
 
   @patch.object(shell, "call")
   @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, False, True]))
-  def test_configure_existing_sqla(self, call_mocks):
+  @patch("os.path.isfile")
+  def test_configure_existing_sqla(self, isfile_mock, call_mocks):
+    # Mock call when checking if need to copy oozie ext-2.2.zip file
+    isfile_mock.side_effect = [True, False]
     call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
                        classname = "OozieServer",
@@ -602,7 +610,10 @@ class TestOozieServer(RMFTestCase):
 
   @patch.object(shell, "call")
   @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True]))
-  def test_configure_secured(self, call_mocks):
+  @patch("os.path.isfile")
+  def test_configure_secured(self, isfile_mock, call_mocks):
+    # Mock call when checking if need to copy oozie ext-2.2.zip file
+    isfile_mock.side_effect = [True, False]
     call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
                        classname = "OozieServer",
@@ -616,8 +627,11 @@ class TestOozieServer(RMFTestCase):
     self.assertNoMoreResources()
 
   @patch.object(shell, "call")
+  @patch("os.path.isfile")
   @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True]))
-  def test_configure_secured_ha(self, call_mocks):
+  def test_configure_secured_ha(self, isfile_mock, call_mocks):
+    # Mock call when checking if need to copy oozie ext-2.2.zip file
+    isfile_mock.side_effects = [True, False]
     call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
 
     config_file = "stacks/2.0.6/configs/secured.json"
@@ -650,7 +664,8 @@ class TestOozieServer(RMFTestCase):
   @patch("os.path.isfile")
   @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True, True]))
   def test_start_secured(self, isfile_mock, call_mocks):
-    isfile_mock.return_value = True
+    # Mock call when checking if need to copy oozie ext-2.2.zip file
+    isfile_mock.side_effects = [True, False]
     call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server.py",
                          classname = "OozieServer",
@@ -1133,7 +1148,10 @@ class TestOozieServer(RMFTestCase):
 
   @patch.object(shell, "call")
   @patch('os.path.exists', new=MagicMock(side_effect = [False, True, False, True, True]))
-  def test_configure_default_hdp22(self, call_mocks):
+  @patch("os.path.isfile")
+  def test_configure_default_hdp22(self, isfile_mock, call_mocks):
+    # Mock call when checking if need to copy oozie ext-2.2.zip file
+    isfile_mock.side_effect = [True, False]
     call_mocks = MagicMock(return_value=(0, "New Oozie WAR file with added"))
     config_file = "stacks/2.0.6/configs/default.json"
     with open(config_file, "r") as f:


Mime
View raw message