ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-12778 - RU: Downgrade Fails For All Components From Finalize (jonathanhurley)
Date Mon, 17 Aug 2015 04:04:54 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 ba3909911 -> b99c56849


AMBARI-12778 - RU: Downgrade Fails For All Components From Finalize (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b99c5684
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b99c5684
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b99c5684

Branch: refs/heads/branch-2.1
Commit: b99c56849308e6b60dc9f2ae6dbe8fff39e0d1ff
Parents: ba39099
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Sat Aug 15 00:02:45 2015 -0400
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Mon Aug 17 00:04:47 2015 -0400

----------------------------------------------------------------------
 .../libraries/functions/conf_select.py          |   2 +-
 .../1.4.0.2.0/package/scripts/flume_handler.py  |  11 +-
 .../1.4.0.2.0/package/scripts/flume_upgrade.py  |   1 +
 .../1.4.0.2.0/package/scripts/params_linux.py   |   8 +-
 .../custom_actions/scripts/ru_set_all.py        |  81 +++++++++++++-
 .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml     |  22 ++++
 .../python/custom_actions/test_ru_set_all.py    | 107 +++++++++++++++++++
 7 files changed, 225 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/b99c5684/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
index a676f74..e039a77 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
@@ -326,5 +326,5 @@ def create_config_links(stack_id, stack_version):
           select(args[0], k, stack_version)
         except Exception, err:
           # don't ruin someone's day
-          Logger.logger.exception("Conf-select set failed to link '{k}'. Error: {0}".format(k,
str(err)))
+          Logger.logger.exception("'conf-select set' failed to link '{k}'. Error: {0}".format(k,
str(err)))
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/b99c5684/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py
b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py
index a305ce7..e1fe577 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_handler.py
@@ -63,19 +63,19 @@ class FlumeHandler(Script):
     env.set_params(params)
     self.configure(env)
     Service(service_mapping.flume_win_service_name, action="start")
-    # flume(action='start')
 
   @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)
     flume(action='stop')
-    if rolling_restart:
+
+    # only backup data on upgrade
+    if rolling_restart and params.upgrade_direction == Direction.UPGRADE:
       flume_upgrade.post_stop_backup()
 
   @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
   def stop(self, env):
-    import params
     Service(service_mapping.flume_win_service_name, action="stop")
 
   def configure(self, env):
@@ -123,7 +123,10 @@ class FlumeHandler(Script):
     Logger.info("Executing Flume Rolling Upgrade pre-restart")
     conf_select.select(params.stack_name, "flume", params.version)
     hdp_select.select("flume-server", params.version)
-    flume_upgrade.pre_start_restore()
+
+    # only restore on upgrade, not downgrade
+    if params.upgrade_direction == Direction.UPGRADE:
+      flume_upgrade.pre_start_restore()
 
 if __name__ == "__main__":
   FlumeHandler().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/b99c5684/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
index 7743619..64c0032 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
@@ -76,6 +76,7 @@ def pre_start_restore():
     if os.path.exists(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)):
       shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR))
 
+
 def _get_directory_mappings():
   """
   Gets a dictionary of directory to archive name that represents the

http://git-wip-us.apache.org/repos/asf/ambari/blob/b99c5684/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params_linux.py
index a7efbf6..22f3324 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/params_linux.py
@@ -16,12 +16,18 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
-from resource_management.libraries.functions import format
+from resource_management.libraries.functions.constants import Direction
+from resource_management.libraries.functions import default
 from resource_management.libraries.script.script import Script
 
 # server configurations
 config = Script.get_config()
 
+# upgrade params
+stack_name = default("/hostLevelParams/stack_name", None)
+upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE)
+stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+
 flume_conf_dir = '/etc/flume/conf'
 if Script.is_hdp_stack_greater_or_equal("2.2"):
   flume_conf_dir = '/usr/hdp/current/flume-server/conf'

http://git-wip-us.apache.org/repos/asf/ambari/blob/b99c5684/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
index f868002..6e5ddd2 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py
@@ -25,6 +25,7 @@ import shutil
 from ambari_commons.os_check import OSCheck
 from resource_management.libraries.script import Script
 from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions.constants import Direction
 from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions.version import compare_versions
 from resource_management.libraries.functions.version import format_hdp_stack_version
@@ -67,6 +68,82 @@ class UpgradeSetAll(Script):
           for dir_def in v:
             link_config(dir_def['conf_dir'], dir_def['current_dir'])
 
+
+  def unlink_all_configs(self, env):
+    """
+    Reverses the work performed in link_config. This should only be used when downgrading
from
+    HDP 2.3 to 2.2 in order to under the symlink work required for 2.3.
+    """
+    stack_name = default('/hostLevelParams/stack_name', "").upper()
+    downgrade_to_version = default('/commandParams/version', None)
+    downgrade_from_version = default('/commandParams/downgrade_from_version', None)
+    upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE)
+
+    # downgrade only
+    if upgrade_direction != Direction.DOWNGRADE:
+      Logger.warning("Unlinking configurations should only be performed on a downgrade.")
+      return
+
+    # HDP only
+    if stack_name != "HDP":
+      Logger.warning("Unlinking configurations should only be performed on the HDP stack.")
+      return
+
+    if downgrade_to_version is None or downgrade_from_version is None:
+      Logger.warning("Both 'commandParams/version' and 'commandParams/downgrade_from_version'
must be specified to unlink configs on downgrade.")
+      return
+
+
+    # normalize the versions
+    stack_23 = format_hdp_stack_version("2.3")
+    downgrade_to_version = format_hdp_stack_version(downgrade_to_version)
+    downgrade_from_version = format_hdp_stack_version(downgrade_from_version)
+
+    # downgrade-to-version must be 2.2 (less than 2.3)
+    if compare_versions(downgrade_to_version, stack_23) >= 0:
+      Logger.warning("Unlinking configurations should only be performed when downgrading
to HDP 2.2")
+      return
+
+    # downgrade-from-version must be 2.3+
+    if compare_versions(downgrade_from_version, stack_23) < 0:
+      Logger.warning("Unlinking configurations should only be performed when downgrading
from HDP 2.3 or later")
+      return
+
+    # iterate through all directory conf mappings and undo the symlinks
+    for key, value in conf_select.PACKAGE_DIRS.iteritems():
+      for directory_mapping in value:
+        original_config_directory = directory_mapping['conf_dir']
+        self._unlink_config(original_config_directory)
+
+
+  def _unlink_config(self, original_conf_directory):
+    """
+    Reverses the work performed in link_config. This should only be used when downgrading
from
+    HDP 2.3 to 2.2 in order to undo the conf symlink work required for 2.3.
+
+    1. Checks if conf.backup exists, if not then do no work
+    2. Check for existance of 'etc' symlink and remove it
+    3. Rename conf.back back to conf
+
+    :original_conf_directory: the original conf directory that was made into a symlink (/etc/component/conf)
+    """
+    if not os.path.islink(original_conf_directory):
+      Logger.info("Skipping the unlink of {0}; it is not a symlink or does not exist".format(original_conf_directory))
+      return
+
+    # calculate the parent and backup directories
+    original_conf_parent_directory = os.path.abspath(os.path.join(original_conf_directory,
os.pardir))
+    backup_conf_directory = os.path.join(original_conf_parent_directory, "conf.backup")
+
+    Logger.info("Unlinking {0} and restoring {1}".format(original_conf_directory, backup_conf_directory))
+
+    # remove the old symlink
+    Execute(("rm", original_conf_directory), sudo=True)
+
+    # rename the backup to the original name
+    Execute(("mv", backup_conf_directory, original_conf_directory), sudo=True)
+
+
 def link_config(old_conf, link_conf):
   """
   Creates a config link following:
@@ -74,7 +151,8 @@ def link_config(old_conf, link_conf):
   2. If it does, check if it's a link already
   3. Make a copy to /etc/[component]/conf.backup
   4. Remove the old directory and create a symlink to link_conf
-  :old_conf: the old config directory, ie /etc/[component]/config
+
+  :old_conf: the old config directory, ie /etc/[component]/conf
   :link_conf: the new target for the config directory, ie /usr/hdp/current/[component-dir]/conf
   """
   if not os.path.exists(old_conf):
@@ -98,5 +176,6 @@ def link_config(old_conf, link_conf):
   # link /etc/[component]/conf -> /usr/hdp/current/[component]-client/conf
   Link(old_conf, to = link_conf)
 
+
 if __name__ == "__main__":
   UpgradeSetAll().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/b99c5684/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index f6823c8..507dec3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -62,9 +62,31 @@
           <function>prepare</function>
         </task>
       </execute-stage>
+    </group>
 
+    <!--
+    On an HDP 2.3 to 2.2 downgrade, the /etc/component/conf must be turned back from a symlink
+    into a normal directory. This will avoid the circular symbolic link that will happen
after
+    hdp-select sets /usr/hdp/current/component back to 2.2:
+
+    # new symlink created for 2.3
+    /etc/flume/conf -> /usr/hdp/current/flume-server/conf
+
+    # circular reference from 2.2 (current)
+    /usr/hdp/2.2.0.0-1234/flume/conf -> /etc/flume/conf
+    /usr/hdp/current/flume-server/conf -> /etc/flume/conf
+    -->
+    <group xsi:type="cluster" name="RESTORE_CONFIG_DIRS" title="Restore Configuration
Directories">
+      <direction>DOWNGRADE</direction>
+      <execute-stage title="Restore configuration directories and remove HDP 2.3 symlinks">
+        <task xsi:type="execute">
+          <script>scripts/ru_set_all.py</script>
+          <function>unlink_all_configs</function>
+        </task>
+      </execute-stage>
     </group>
 
+
     <group name="ZOOKEEPER" title="ZooKeeper">
       <service name="ZOOKEEPER">
         <component>ZOOKEEPER_SERVER</component>

http://git-wip-us.apache.org/repos/asf/ambari/blob/b99c5684/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
index 211443b..b102b4e 100644
--- a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
+++ b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py
@@ -29,6 +29,7 @@ from mock.mock import MagicMock
 from stacks.utils.RMFTestCase import *
 from resource_management import Script, ConfigDictionary
 from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import conf_select
 from resource_management.core.logger import Logger
 from ambari_agent.AmbariConfig import AmbariConfig
 from ambari_agent.FileCache import FileCache
@@ -130,3 +131,109 @@ class TestRUSetAll(RMFTestCase):
     self.assertTrue(link_mock.called)
     call_mock.assert_called_with(('hdp-select', 'set', 'all', '2.3.0.0-1234'), sudo=True)
 
+
+  @patch("os.path.islink")
+  @patch("resource_management.core.shell.call")
+  @patch.object(Script, 'get_config')
+  @patch.object(OSCheck, 'is_redhat_family')
+  def test_downgrade_unlink_configs(self, family_mock, get_config_mock, call_mock, islink_mock):
+    """
+    Tests downgrading from 2.3 to 2.2 to ensure that conf symlinks are removed and the backup
+    directories restored.
+    """
+
+    # required for the test to run since the Execute calls need this
+    from resource_management.core.environment import Environment
+    env = Environment(test_mode=True)
+    env._instances.append(env)
+
+    # Mock the config objects
+    json_file_path = os.path.join(self.get_custom_actions_dir(), "ru_execute_tasks_namenode_prepare.json")
+    self.assertTrue(os.path.isfile(json_file_path))
+    with open(json_file_path, "r") as json_file:
+      json_payload = json.load(json_file)
+
+    # alter JSON for a downgrade from 2.3 to 2.2
+    json_payload['commandParams']['version'] = "2.2.0.0-1234"
+    json_payload['commandParams']['downgrade_from_version'] = "2.3.0.0-1234"
+    json_payload['commandParams']['original_stack'] = "HDP-2.2"
+    json_payload['commandParams']['target_stack'] = "HDP-2.3"
+    json_payload['commandParams']['upgrade_direction'] = "downgrade"
+    json_payload['hostLevelParams']['stack_version'] = "2.2"
+
+    config_dict = ConfigDictionary(json_payload)
+
+    family_mock.return_value = True
+    get_config_mock.return_value = config_dict
+    call_mock.side_effect = fake_call   # echo the command
+
+    # test the function
+    ru_execute = UpgradeSetAll()
+    ru_execute.unlink_all_configs(None)
+
+    # verify that os.path.islink was called for each conf
+    self.assertTrue(islink_mock.called)
+    for key, value in conf_select.PACKAGE_DIRS.iteritems():
+      for directory_mapping in value:
+        original_config_directory = directory_mapping['conf_dir']
+        is_link_called = False
+
+        for call in islink_mock.call_args_list:
+          call_tuple = call[0]
+          if original_config_directory in call_tuple:
+            is_link_called = True
+
+        if not is_link_called:
+          self.fail("os.path.islink({0}) was never called".format(original_config_directory))
+
+    # alter JSON for a downgrade from 2.3 to 2.3
+    with open(json_file_path, "r") as json_file:
+      json_payload = json.load(json_file)
+
+    json_payload['commandParams']['version'] = "2.3.0.0-1234"
+    json_payload['commandParams']['downgrade_from_version'] = "2.3.0.0-5678"
+    json_payload['commandParams']['original_stack'] = "HDP-2.3"
+    json_payload['commandParams']['target_stack'] = "HDP-2.3"
+    json_payload['commandParams']['upgrade_direction'] = "downgrade"
+    json_payload['hostLevelParams']['stack_version'] = "2.3"
+
+    # reset config
+    config_dict = ConfigDictionary(json_payload)
+    family_mock.return_value = True
+    get_config_mock.return_value = config_dict
+
+    # reset mock
+    islink_mock.reset_mock()
+
+    # test the function
+    ru_execute = UpgradeSetAll()
+    ru_execute.unlink_all_configs(None)
+
+    # ensure it wasn't called this time
+    self.assertFalse(islink_mock.called)
+
+    with open(json_file_path, "r") as json_file:
+      json_payload = json.load(json_file)
+
+    # alter JSON for a downgrade from 2.2 to 2.2
+    json_payload['commandParams']['version'] = "2.2.0.0-1234"
+    json_payload['commandParams']['downgrade_from_version'] = "2.2.0.0-5678"
+    json_payload['commandParams']['original_stack'] = "HDP-2.2"
+    json_payload['commandParams']['target_stack'] = "HDP-2.2"
+    json_payload['commandParams']['upgrade_direction'] = "downgrade"
+    json_payload['hostLevelParams']['stack_version'] = "2.2"
+
+    # reset config
+    config_dict = ConfigDictionary(json_payload)
+    family_mock.return_value = True
+    get_config_mock.return_value = config_dict
+
+    # reset mock
+    islink_mock.reset_mock()
+
+    # test the function
+    ru_execute = UpgradeSetAll()
+    ru_execute.unlink_all_configs(None)
+
+    # ensure it wasn't called this time
+    self.assertFalse(islink_mock.called)


Mime
View raw message