ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-14543 - Adding A Removed Host Back To a Cluster Post-Upgrade Does Not Install New Stacks Correctly (part2) (jonathanhurley)
Date Wed, 06 Jan 2016 19:47:57 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.2 53485c89b -> 6d7f13bd4


AMBARI-14543 - Adding A Removed Host Back To a Cluster Post-Upgrade Does Not Install New Stacks
Correctly (part2) (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6d7f13bd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6d7f13bd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6d7f13bd

Branch: refs/heads/branch-2.2
Commit: 6d7f13bd4e4a03e178c83304ffc392060a3a42fb
Parents: 53485c8
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Wed Jan 6 11:29:35 2016 -0500
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Wed Jan 6 14:46:09 2016 -0500

----------------------------------------------------------------------
 .../libraries/functions/hdp_select.py           | 23 ++++++-----
 .../controller/AmbariActionExecutionHelper.java |  3 +-
 .../2.0.6/hooks/after-INSTALL/scripts/params.py |  6 +--
 .../scripts/shared_initialization.py            |  5 ++-
 .../hooks/after-INSTALL/test_after_install.py   | 40 ++++++++++++++++++++
 5 files changed, 63 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/6d7f13bd/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
index 7cae627..9a3201e 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
@@ -102,18 +102,23 @@ HADOOP_DIR_DEFAULTS = {
   "lib": "/usr/lib/hadoop/lib"
 }
 
-def select_all(stack_version):
+def select_all(version_to_select):
   """
-  Executes hdp-select on every component for the latest installed version of the specified
stack.
-  For example, if stack_version is "2.3", then this will find the latest installed version
which
-  could be "2.3.0.0-9999".
-  :param stack_version: the stack version to use when calculating the latest actual version,
-  such as "2.3".
+  Executes hdp-select on every component for the specified version. If the value passed in
is a
+  stack version such as "2.3", then this will find the latest installed version which
+  could be "2.3.0.0-9999". If a version is specified instead, such as 2.3.0.0-1234, it will
use
+  that exact version.
+  :param version_to_select: the version to hdp-select on, such as "2.3" or "2.3.0.0-1234"
   """
-  Logger.info("Executing hdp-select set all on the latest calculated version for stack {0}".format(stack_version))
+  # it's an error, but it shouldn't really stop anything from working
+  if version_to_select is None:
+    Logger.error("Unable to execute hdp-select after installing because there was no version
specified")
+    return
 
-  command = format('{sudo} /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select
versions | grep ^{stack_version} | tail -1`')
-  only_if_command = format('ls -d /usr/hdp/{stack_version}*')
+  Logger.info("Executing hdp-select set all on {0}".format(version_to_select))
+
+  command = format('{sudo} /usr/bin/hdp-select set all `ambari-python-wrap /usr/bin/hdp-select
versions | grep ^{version_to_select} | tail -1`')
+  only_if_command = format('ls -d /usr/hdp/{version_to_select}*')
   Execute(command, only_if = only_if_command)
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d7f13bd/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
index 6e7e06b..2039751 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.controller;
 
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMPONENT_CATEGORY;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
 import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
@@ -495,7 +496,7 @@ public class AmbariActionExecutionHelper {
       }
     }
 
-    hostLevelParams.put("repo_info", rootJsonObject.toString());
+    hostLevelParams.put(REPO_INFO, rootJsonObject.toString());
 
     StackId stackId = cluster.getCurrentStackVersion();
     hostLevelParams.put(STACK_NAME, stackId.getStackName());

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d7f13bd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py
index ba261fe..7039f3e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py
@@ -25,15 +25,15 @@ from resource_management.libraries.functions import hdp_select
 from resource_management.libraries.functions import format_jvm_option
 from resource_management.libraries.functions.version import format_hdp_stack_version
 
-from resource_management.core.system import System
-from ambari_commons.os_check import OSCheck
-
 config = Script.get_config()
 sudo = AMBARI_SUDO_BINARY
 
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 
+# current host stack version
+current_version = default("/hostLevelParams/current_version", None)
+
 # default hadoop params
 mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
 hadoop_libexec_dir = hdp_select.get_hadoop_dir("libexec")

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d7f13bd/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
index 1b7b57a..00ae426 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
@@ -38,7 +38,10 @@ def setup_hdp_symlinks():
   """
   import params
   if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2')
>= 0:
-    hdp_select.select_all(params.stack_version_unformatted)
+    # try using the exact version first, falling back in just the stack if it's not defined
+    # which would only be during an intial cluster installation
+    version = params.current_version if params.current_version is not None else params.stack_version_unformatted
+    hdp_select.select_all(version)
 
 
 def setup_config():

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d7f13bd/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
index be35a99..0498b3a 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
@@ -512,3 +512,43 @@ class TestHookAfterInstall(RMFTestCase):
         to = '/usr/hdp/current/hive-client/conf')
 
     self.assertNoMoreResources()
+
+
+  @patch("shared_initialization.load_version", new = MagicMock(return_value="2.3.0.0-1243"))
+  @patch("resource_management.libraries.functions.conf_select.create")
+  @patch("resource_management.libraries.functions.conf_select.select")
+  @patch("os.symlink")
+  @patch("shutil.rmtree")
+  def test_hook_default_hdp_select_specific_version(self, rmtree_mock, symlink_mock, conf_select_select_mock,
conf_select_create_mock):
+    """
+    Tests that hdp-select set all on a specific version, not a 2.3* wildcard is used when
+    installing a component when the cluster version is already set.
+
+    :param rmtree_mock:
+    :param symlink_mock:
+    :param conf_select_select_mock:
+    :param conf_select_create_mock:
+    :return:
+    """
+
+    def mocked_conf_select(arg1, arg2, arg3, dry_run = False):
+      return "/etc/{0}/{1}/0".format(arg2, arg3)
+
+    conf_select_create_mock.side_effect = mocked_conf_select
+
+    config_file = self.get_src_folder() + "/test/python/stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+
+    version = '2.3.0.0-1234'
+    json_content['commandParams']['version'] = version
+    json_content['hostLevelParams']['stack_version'] = "2.3"
+    json_content['hostLevelParams']['current_version'] = "2.3.0.0-1234"
+
+    self.executeScript("2.0.6/hooks/after-INSTALL/scripts/hook.py",
+      classname="AfterInstallHook",
+      command="hook",
+      config_dict = json_content)
+
+    self.assertResourceCalled('Execute', 'ambari-sudo.sh /usr/bin/hdp-select set all `ambari-python-wrap
/usr/bin/hdp-select versions | grep ^2.3.0.0-1234 | tail -1`',
+      only_if = 'ls -d /usr/hdp/2.3.0.0-1234*')


Mime
View raw message