ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-14543 - Adding A Removed Host Back To a Cluster Post-Upgrade Does Not Install New Stacks Correctly (jonathanhurley)
Date Tue, 05 Jan 2016 19:09:16 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.2 883bde336 -> 21a4ed434


AMBARI-14543 - Adding A Removed Host Back To a Cluster Post-Upgrade Does Not Install New Stacks
Correctly (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/21a4ed43
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/21a4ed43
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/21a4ed43

Branch: refs/heads/branch-2.2
Commit: 21a4ed4346e2bba4b299506bf104818ceeb761c8
Parents: 883bde3
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Mon Jan 4 19:02:57 2016 -0500
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Tue Jan 5 14:09:07 2016 -0500

----------------------------------------------------------------------
 .../src/main/python/ambari_agent/HostCleanup.py | 12 -----------
 .../test/python/ambari_agent/TestHostCleanup.py | 15 -------------
 .../libraries/functions/hdp_select.py           | 22 ++++++++++++++++++--
 .../2.0.6/hooks/after-INSTALL/scripts/hook.py   |  5 ++---
 .../scripts/shared_initialization.py            | 22 ++++++++++----------
 .../hooks/after-INSTALL/test_after_install.py   | 16 ++++----------
 6 files changed, 37 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/21a4ed43/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
index 4d3cd83..1655fbd 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
@@ -101,8 +101,6 @@ def get_erase_cmd():
 
 class HostCleanup:
 
-  SELECT_ALL_PERFORMED_MARKER = "/var/lib/ambari-agent/data/hdp-select-set-all.performed"
-
   def resolve_ambari_config(self):
     try:
       config = AmbariConfig()
@@ -148,8 +146,6 @@ class HostCleanup:
       if packageList and not PACKAGE_SECTION in SKIP_LIST:
         logger.info("Deleting packages: " + str(packageList) + "\n")
         self.do_erase_packages(packageList)
-        # Removing packages means that we have to rerun hdp-select
-        self.do_remove_hdp_select_marker()
       if userList and not USER_SECTION in SKIP_LIST:
         logger.info("\n" + "Deleting users: " + str(userList))
         self.do_delete_users(userList)
@@ -276,14 +272,6 @@ class HostCleanup:
       self.do_erase_files_silent(remList)
 
 
-  def do_remove_hdp_select_marker(self):
-    """
-    Remove marker file for 'hdp-select set all' invocation
-    """
-    if os.path.isfile(self.SELECT_ALL_PERFORMED_MARKER):
-      os.unlink(self.SELECT_ALL_PERFORMED_MARKER)
-
-
   # Alternatives exist as a stack of symlinks under /var/lib/alternatives/$name
   # Script expects names of the alternatives as input
   # We find all the symlinks using command, #] alternatives --display $name

http://git-wip-us.apache.org/repos/asf/ambari/blob/21a4ed43/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
index 7cfebe8..165c5f0 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
@@ -408,21 +408,6 @@ class TestHostCleanup(TestCase):
     sys.stdout = sys.__stdout__
 
 
-  @patch('os.path.isfile')
-  @patch('os.unlink')
-  def test_do_remove_hdp_select_marker(self, unlink_mock, isfile_mock):
-    out = StringIO.StringIO()
-    sys.stdout = out
-
-    isfile_mock.return_value = True
-
-    self.hostcleanup.do_remove_hdp_select_marker()
-
-    self.assertTrue(unlink_mock.called)
-
-    sys.stdout = sys.__stdout__
-
-
   @patch.object(HostCleanup.HostCleanup, 'get_files_in_dir')
   @patch.object(OSCheck, "get_os_type")
   def test_find_repo_files_for_repos(self, get_os_type_method,

http://git-wip-us.apache.org/repos/asf/ambari/blob/21a4ed43/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
index 7d90dca..7cae627 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
@@ -25,11 +25,13 @@ from resource_management.core.exceptions import Fail
 from resource_management.core.resources.system import Execute
 from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions.get_hdp_version import get_hdp_version
+from resource_management.libraries.functions.format import format
 from resource_management.libraries.script.script import Script
 from resource_management.core.shell import call
 from resource_management.libraries.functions.version import format_hdp_stack_version
 from resource_management.libraries.functions.version_select_util import get_versions_from_stack_root
 
+
 # hdp-select set oozie-server 2.2.0.0-1234
 TEMPLATE = ('hdp-select', 'set')
 
@@ -100,6 +102,21 @@ HADOOP_DIR_DEFAULTS = {
   "lib": "/usr/lib/hadoop/lib"
 }
 
+def select_all(stack_version):
+  """
+  Executes hdp-select on every component for the latest installed version of the specified
stack.
+  For example, if stack_version is "2.3", then this will find the latest installed version
which
+  could be "2.3.0.0-9999".
+  :param stack_version: the stack version to use when calculating the latest actual version,
+  such as "2.3".
+  """
+  Logger.info("Executing hdp-select set all on the latest calculated version for stack {0}".format(stack_version))
+
+  command = format('{sudo} /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select
versions | grep ^{stack_version} | tail -1`')
+  only_if_command = format('ls -d /usr/hdp/{stack_version}*')
+  Execute(command, only_if = only_if_command)
+
+
 def select(component, version):
   """
   Executes hdp-select on the specific component and version. Some global
@@ -109,7 +126,8 @@ def select(component, version):
   recalculated is to call reload(...) on each module that has global parameters.
   After invoking hdp-select, this function will also reload params, status_params,
   and params_linux.
-  :param component: the hdp-select component, such as oozie-server
+  :param component: the hdp-select component, such as oozie-server. If "all", then all components
+  will be updated.
   :param version: the version to set the component to, such as 2.2.0.0-1234
   """
   command = TEMPLATE + (component, version)
@@ -125,7 +143,7 @@ def select(component, version):
     if moduleName in modules:
       module = modules.get(moduleName)
       reload(module)
-      Logger.info("After hdp-select {0}, reloaded module {1}".format(component, moduleName))
+      Logger.info("After {0}, reloaded module {1}".format(command, moduleName))
 
 
 def get_role_component_current_hdp_version():

http://git-wip-us.apache.org/repos/asf/ambari/blob/21a4ed43/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/hook.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/hook.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/hook.py
index d41a889..73412e0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/hook.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/hook.py
@@ -20,8 +20,7 @@ limitations under the License.
 from resource_management.libraries.script.hook import Hook
 from shared_initialization import link_configs
 from shared_initialization import setup_config
-from shared_initialization import setup_hdp_install_directory
-from resource_management.libraries.script import Script
+from shared_initialization import setup_hdp_symlinks
 
 class AfterInstallHook(Hook):
 
@@ -29,7 +28,7 @@ class AfterInstallHook(Hook):
     import params
 
     env.set_params(params)
-    setup_hdp_install_directory()
+    setup_hdp_symlinks()
     setup_config()
 
     link_configs(self.stroutfile)

http://git-wip-us.apache.org/repos/asf/ambari/blob/21a4ed43/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
index 85bfbde..1b7b57a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
@@ -17,29 +17,29 @@ limitations under the License.
 
 """
 import os
-import shutil
 
 import ambari_simplejson as json
 from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute
-from resource_management.core.shell import as_sudo
 from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import hdp_select
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.version import compare_versions
 from resource_management.libraries.resources.xml_config import XmlConfig
 from resource_management.libraries.script import Script
 
 
-def setup_hdp_install_directory():
-  # This is a name of marker file.
-  SELECT_ALL_PERFORMED_MARKER = "/var/lib/ambari-agent/data/hdp-select-set-all.performed"
+def setup_hdp_symlinks():
+  """
+  Invokes hdp-select set all against a calculated fully-qualified, "normalized" version based
on a
+  stack version, such as "2.3". This should always be called after a component has been
+  installed to ensure that all HDP pointers are correct. The stack upgrade logic does not
+  interact with this since it's done via a custom command and will not trigger this hook.
+  :return:
+  """
   import params
   if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2')
>= 0:
-    Execute(as_sudo(['touch', SELECT_ALL_PERFORMED_MARKER]) + ' ; ' +
-                   format('{sudo} /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select
versions | grep ^{stack_version_unformatted} | tail -1`'),
-            only_if=format('ls -d /usr/hdp/{stack_version_unformatted}*'),   # If any HDP
version is installed
-            not_if=format("test -f {SELECT_ALL_PERFORMED_MARKER}")           # Do that only
once (otherwise we break stack upgrade logic)
-    )
+    hdp_select.select_all(params.stack_version_unformatted)
+
 
 def setup_config():
   import params

http://git-wip-us.apache.org/repos/asf/ambari/blob/21a4ed43/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
index 4d5c0c2..be35a99 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
@@ -69,12 +69,8 @@ class TestHookAfterInstall(RMFTestCase):
                        config_dict = json_content)
 
 
-    self.assertResourceCalled("Execute",
-      "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E touch /var/lib/ambari-agent/data/hdp-select-set-all.performed
; " \
-      "ambari-sudo.sh /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select
versions | grep ^2.3 | tail -1`",
-      only_if = "ls -d /usr/hdp/2.3*",
-      not_if = "test -f /var/lib/ambari-agent/data/hdp-select-set-all.performed")
-
+    self.assertResourceCalled('Execute', 'ambari-sudo.sh /usr/bin/hdp-select set all `ambari-python-wrap
/usr/bin/hdp-select versions | grep ^2.3 | tail -1`',
+      only_if = 'ls -d /usr/hdp/2.3*')
 
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
@@ -310,12 +306,8 @@ class TestHookAfterInstall(RMFTestCase):
                        config_dict = json_content)
 
 
-    self.assertResourceCalled("Execute",
-      "ambari-sudo.sh [RMF_ENV_PLACEHOLDER] -H -E touch /var/lib/ambari-agent/data/hdp-select-set-all.performed
; " \
-      "ambari-sudo.sh /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select
versions | grep ^2.3 | tail -1`",
-      only_if = "ls -d /usr/hdp/2.3*",
-      not_if = "test -f /var/lib/ambari-agent/data/hdp-select-set-all.performed")
-
+    self.assertResourceCalled('Execute', 'ambari-sudo.sh /usr/bin/hdp-select set all `ambari-python-wrap
/usr/bin/hdp-select versions | grep ^2.3 | tail -1`',
+      only_if = 'ls -d /usr/hdp/2.3*')
 
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',


Mime
View raw message