ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jlun...@apache.org
Subject ambari git commit: AMBARI-13210: RU - Install version stuck (jluniya)
Date Thu, 24 Sep 2015 23:15:09 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 e6ce33e3d -> 3815ed073


AMBARI-13210: RU - Install version stuck (jluniya)

Conflicts:
	ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
	ambari-server/src/main/resources/custom_actions/scripts/install_packages.py


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3815ed07
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3815ed07
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3815ed07

Branch: refs/heads/branch-2.1
Commit: 3815ed073a9f62f7811a4c36b0dc42d1b193e831
Parents: e6ce33e
Author: Jayush Luniya <jluniya@hortonworks.com>
Authored: Thu Sep 24 15:55:36 2015 -0700
Committer: Jayush Luniya <jluniya@hortonworks.com>
Committed: Thu Sep 24 16:14:45 2015 -0700

----------------------------------------------------------------------
 .../libraries/functions/hdp_select.py           |  19 ++-
 .../libraries/functions/version_select_util.py  |  20 ++-
 .../DistributeRepositoriesActionListener.java   |  13 +-
 .../custom_actions/scripts/install_packages.py  | 128 +++++++++++++++----
 .../custom_actions/TestInstallPackages.py       |  20 ++-
 5 files changed, 159 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/3815ed07/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
index 0c42823..f4f0efc 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
@@ -18,6 +18,7 @@ limitations under the License.
 
 """
 
+import os
 import sys
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
@@ -27,6 +28,7 @@ from resource_management.libraries.functions.get_hdp_version import get_hdp_vers
 from resource_management.libraries.script.script import Script
 from resource_management.core.shell import call
 from resource_management.libraries.functions.version import format_hdp_stack_version
+from resource_management.libraries.functions.version_select_util import get_versions_from_stack_root
 
 # hdp-select set oozie-server 2.2.0.0-1234
 TEMPLATE = ('hdp-select', 'set')
@@ -239,12 +241,19 @@ def _get_upgrade_stack():
   return None
 
 
-def get_hdp_versions():
+def get_hdp_versions(stack_root):
+  """
+  Gets list of stack versions installed on the host.
+  Be default a call to hdp-select versions is made to get the list of installed stack versions.
+  As a fallback list of installed versions is collected from stack version directories in
stack install root.
+  :param stack_root: Stack install root
+  :return: Returns list of installed stack versions.
+  """
   code, out = call("hdp-select versions")
+  versions = []
   if 0 == code:
-    versions = []
     for line in out.splitlines():
       versions.append(line.rstrip('\n'))
-    return versions
-  else:
-    return []
+  if not versions:
+    versions = get_versions_from_stack_root(stack_root)
+  return versions

http://git-wip-us.apache.org/repos/asf/ambari/blob/3815ed07/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
index d1649df..f1a484b 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
@@ -19,6 +19,7 @@ limitations under the License.
 Ambari Agent
 
 """
+import os
 import re
 import tempfile
 
@@ -74,4 +75,21 @@ def get_component_version(stack_name, component_name):
   else:
     Logger.error("Could not find a stack for stack name: %s" % str(stack_name))
 
-  return version
\ No newline at end of file
+  return version
+
+
+def get_versions_from_stack_root(stack_root):
+  """
+  Given a stack install root (/usr/hdp), returns a list of stack versions currently installed.
+  The list of installed stack versions is determined purely based on the stack version directories
+  found in the stack install root.
+  Because each stack name may have different logic, the input is a generic dictionary.
+  :param stack_root: Stack install root directory
+  :return: Returns list of installed stack versions
+  """
+  if stack_root is None or not os.path.exists(stack_root):
+    return []
+
+  installed_stack_versions = [f for f in os.listdir(stack_root) if os.path.isdir(os.path.join(stack_root,
f))
+                              and re.match("([\d\.]+(-\d+)?)", f)]
+  return installed_stack_versions

http://git-wip-us.apache.org/repos/asf/ambari/blob/3815ed07/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java
b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java
index 5b7c2d6..129c404 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java
@@ -99,20 +99,23 @@ public class DistributeRepositoriesActionListener {
     String repositoryVersion = null;
 
     if (event.getCommandReport() == null) {
-      LOG.error("Command report is null, will set all INSTALLING versions for host {} to
INSTALL_FAILED.", event.getHostname());
+      LOG.error(
+          "Command report is null, will set all INSTALLING versions for host {} to INSTALL_FAILED.",
+          event.getHostname());
+    } else if (!event.getCommandReport().getStatus().equals(HostRoleStatus.COMPLETED.toString()))
{
+      LOG.warn(
+          "Distribute repositories did not complete, will set all INSTALLING versions for
host {} to INSTALL_FAILED.",
+          event.getHostname());
     } else {
       // Parse structured output
       try {
+        newHostState = RepositoryVersionState.INSTALLED;
         DistributeRepositoriesStructuredOutput structuredOutput = StageUtils.getGson().fromJson(
                 event.getCommandReport().getStructuredOut(),
                 DistributeRepositoriesStructuredOutput.class);
 
         repositoryVersion = structuredOutput.getInstalledRepositoryVersion();
 
-        if (event.getCommandReport().getStatus().equals(HostRoleStatus.COMPLETED.toString()))
{
-          newHostState = RepositoryVersionState.INSTALLED;
-        }
-
         // Handle the case in which the version to install did not contain the build number,
         // but the structured output does contain the build number.
         if (null != structuredOutput.getActualVersion() && !structuredOutput.getActualVersion().isEmpty()
&&

http://git-wip-us.apache.org/repos/asf/ambari/blob/3815ed07/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
index d2f760d..dd43c8f 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
@@ -88,9 +88,12 @@ class InstallPackages(Script):
           self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[stack_name]
     if self.stack_root_folder is None:
       raise Fail("Cannot determine the stack's root directory by parsing the stack_id property,
{0}".format(str(stack_id)))
+    if self.repository_version is None:
+      raise Fail("Cannot determine the repository version to install")
 
     self.repository_version = self.repository_version.strip()
 
+
     # Install/update repositories
     installed_repositories = []
     self.current_repositories = []
@@ -130,24 +133,10 @@ class InstallPackages(Script):
     if num_errors > 0:
       raise Fail("Failed to distribute repositories/install packages")
 
-    # If the repo contains a build number, optimistically assume it to be the actual_version.
It will get changed
-    # to correct value if it is not
-    self.actual_version = None
-    if self.repository_version:
-      m = re.search("[\d\.]+-\d+", self.repository_version)
-      if m:
-        # Contains a build number
-        self.repo_version_with_build_number = self.repository_version
-        self.structured_output['actual_version'] = self.repo_version_with_build_number  #
This is the best value known so far.
-        self.put_structured_out(self.structured_output)
-      else:
-        self.repo_version_with_build_number = None
-
     # Initial list of versions, used to compute the new version installed
-    self.old_versions = get_hdp_versions()
+    self.old_versions = get_hdp_versions(self.stack_root_folder)
 
     try:
-      # It's possible for the process to receive a SIGTERM while installing the packages
       ret_code = self.install_packages(package_list)
       if ret_code == 0:
         self.structured_output['package_installation_result'] = 'SUCCESS'
@@ -169,18 +158,31 @@ class InstallPackages(Script):
 
   def compute_actual_version(self):
     """
-    After packages are installed, determine what the new actual version is, in order to save
it.
+    After packages are installed, determine what the new actual version is.
     """
+
+    # If the repo contains a build number, optimistically assume it to be the actual_version.
It will get changed
+    # to correct value if it is not
+    self.actual_version = None
+    self.repo_version_with_build_number = None
+    if self.repository_version:
+      m = re.search("[\d\.]+-\d+", self.repository_version)
+      if m:
+        # Contains a build number
+        self.repo_version_with_build_number = self.repository_version
+        self.structured_output['actual_version'] = self.repo_version_with_build_number  #
This is the best value known so far.
+        self.put_structured_out(self.structured_output)
+
     Logger.info("Attempting to determine actual version with build number.")
     Logger.info("Old versions: {0}".format(self.old_versions))
 
-    new_versions = get_hdp_versions()
+    new_versions = get_hdp_versions(self.stack_root_folder)
     Logger.info("New versions: {0}".format(new_versions))
 
     deltas = set(new_versions) - set(self.old_versions)
     Logger.info("Deltas: {0}".format(deltas))
 
-    # Get HDP version without build number
+    # Get version without build number
     normalized_repo_version = self.repository_version.split('-')[0]
 
     if 1 == len(deltas):
@@ -188,18 +190,22 @@ class InstallPackages(Script):
       self.structured_output['actual_version'] = self.actual_version
       self.put_structured_out(self.structured_output)
       write_actual_version_to_history_file(normalized_repo_version, self.actual_version)
+      Logger.info(
+        "Found actual version {0} by checking the delta between versions before and after
installing packages".format(
+          self.actual_version))
     else:
-      Logger.info("Cannot determine a new actual version installed by using the delta method.")
       # If the first install attempt does a partial install and is unable to report this
to the server,
-      # then a subsequent attempt will report an empty delta. For this reason, it is important
to search the
-      # repo version history file to determine if we previously did write an actual_version.
-      self.actual_version = read_actual_version_from_history_file(normalized_repo_version)
+      # then a subsequent attempt will report an empty delta. For this reason, we search
for a best fit version for the repo version
+      Logger.info("Cannot determine actual version installed by checking the delta between
versions "
+                  "before and after installing package")
+      Logger.info("Will try to find for the actual version by searching for best possible
match in the list of versions installed")
+      self.actual_version = self.find_best_fit_version(new_versions, self.repository_version)
       if self.actual_version is not None:
         self.actual_version = self.actual_version.strip()
         self.structured_output['actual_version'] = self.actual_version
         self.put_structured_out(self.structured_output)
-        Logger.info("Found actual version {0} by parsing file {1}".format(self.actual_version,
REPO_VERSION_HISTORY_FILE))
-      elif self.repo_version_with_build_number is None:
+        Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version))
+      else:
         # It's likely that this host does not have any Stack Components installed, so only
contains AMS.
         # So just use repo version value provided by server (we already put it to structured
output)
         if not os.path.exists(self.stack_root_folder):
@@ -212,6 +218,73 @@ class InstallPackages(Script):
                 "Try reinstalling packages again.".format(self.stack_root_folder)
           raise Fail(msg)
 
+  def check_partial_install(self):
+    """
+    If an installation did not complete successfully, check if installation was partially
complete and
+    log the partially completed version to REPO_VERSION_HISTORY_FILE.
+    :return:
+    """
+    Logger.info("Installation of packages failed. Checking if installation was partially
complete")
+    Logger.info("Old versions: {0}".format(self.old_versions))
+
+    new_versions = get_hdp_versions(self.stack_root_folder)
+    Logger.info("New versions: {0}".format(new_versions))
+
+    deltas = set(new_versions) - set(self.old_versions)
+    Logger.info("Deltas: {0}".format(deltas))
+
+    # Get version without build number
+    normalized_repo_version = self.repository_version.split('-')[0]
+
+    if 1 == len(deltas):
+      # Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE
+      partial_install_version = next(iter(deltas)).strip()
+      write_actual_version_to_history_file(normalized_repo_version, partial_install_version)
+      Logger.info("Version {0} was partially installed. ".format(partial_install_version))
+
+  def find_best_fit_version(self, versions, repo_version):
+    """
+    Given a list of installed versions and a repo version, search for a version that best
fits the repo version
+    If the repo version is found in the list of installed versions, return the repo version
itself.
+    If the repo version is not found in the list of installed versions
+    normalize the repo version and use the REPO_VERSION_HISTORY_FILE file to search the list.
+
+    :param versions: List of versions installed
+    :param repo_version: Repo version to search
+    :return: Matching version, None if no match was found.
+    """
+    if versions is None or repo_version is None:
+      return None
+
+    build_num_match = re.search("[\d\.]+-\d+", repo_version)
+    if build_num_match and repo_version in versions:
+      # If repo version has build number and is found in the list of versions, return it
as the matching version
+      Logger.info("Best Fit Version: Resolved from repo version with valid build number:
{0}".format(repo_version))
+      return repo_version
+
+    # Get version without build number
+    normalized_repo_version = repo_version.split('-')[0]
+
+    # Find all versions that match the normalized repo version
+    match_versions = filter(lambda x: x.startswith(normalized_repo_version), versions)
+    if match_versions:
+
+      if len(match_versions) == 1:
+        # Resolved without conflicts
+        Logger.info("Best Fit Version: Resolved from normalized repo version without conflicts:
{0}".format(match_versions[0]))
+        return match_versions[0]
+
+      # Resolve conflicts using REPO_VERSION_HISTORY_FILE
+      history_version = read_actual_version_from_history_file(normalized_repo_version)
+
+      # Validate history version retrieved is valid
+      if history_version in match_versions:
+        Logger.info("Best Fit Version: Resolved from normalized repo version using {0}: {1}".format(REPO_VERSION_HISTORY_FILE,
history_version))
+        return history_version
+
+    # No matching version
+    return None
+
 
   def install_packages(self, package_list):
     """
@@ -254,7 +327,10 @@ class InstallPackages(Script):
             Package(package, action="remove")
     # Compute the actual version in order to save it in structured out
     try:
-      self.compute_actual_version()
+      if ret_code == 0:
+         self.compute_actual_version()
+      else:
+        self.check_partial_install()
     except Fail, err:
       ret_code = 1
       Logger.logger.exception("Failure while computing actual version. Error: {0}".format(str(err)))
@@ -306,7 +382,7 @@ class InstallPackages(Script):
 
   def abort_handler(self, signum, frame):
     Logger.error("Caught signal {0}, will handle it gracefully. Compute the actual version
if possible before exiting.".format(signum))
-    self.compute_actual_version()
+    self.check_partial_install()
 
 
 if __name__ == "__main__":

http://git-wip-us.apache.org/repos/asf/ambari/blob/3815ed07/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
index e85ee39..7feef96 100644
--- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
+++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py
@@ -66,7 +66,10 @@ class TestInstallPackages(RMFTestCase):
                             read_actual_version_from_history_file_mock,
                             hdp_versions_mock,
                             put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock):
-    read_actual_version_from_history_file_mock.return_value = VERSION_STUB
+    hdp_versions_mock.side_effect = [
+      [],  # before installation attempt
+      [VERSION_STUB]
+    ]
     allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
     list_ambari_managed_repos_mock.return_value=[]
     self.executeScript("scripts/install_packages.py",
@@ -121,7 +124,10 @@ class TestInstallPackages(RMFTestCase):
                             read_actual_version_from_history_file_mock,
                             hdp_versions_mock, put_structured_out_mock, allInstalledPackages_mock,
list_ambari_managed_repos_mock, is_suse_family_mock):
     is_suse_family_mock = True
-    read_actual_version_from_history_file_mock.return_value = VERSION_STUB
+    hdp_versions_mock.side_effect = [
+      [],  # before installation attempt
+      [VERSION_STUB]
+    ]
     allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
     list_ambari_managed_repos_mock.return_value=[]
     self.executeScript("scripts/install_packages.py",
@@ -178,7 +184,10 @@ class TestInstallPackages(RMFTestCase):
                                  hdp_versions_mock,
                                  allInstalledPackages_mock, put_structured_out_mock,
                                  is_redhat_family_mock, list_ambari_managed_repos_mock):
-    read_actual_version_from_history_file_mock.return_value = VERSION_STUB
+    hdp_versions_mock.side_effect = [
+      [],  # before installation attempt
+      [VERSION_STUB]
+    ]
     allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
     list_ambari_managed_repos_mock.return_value=["HDP-UTILS-2.2.0.1-885"]
     is_redhat_family_mock.return_value = True
@@ -269,7 +278,6 @@ class TestInstallPackages(RMFTestCase):
     self.assertTrue(put_structured_out_mock.called)
     self.assertEquals(put_structured_out_mock.call_args[0][0],
                       {'stack_id': 'HDP-2.2',
-                      'actual_version': VERSION_STUB,
                       'installed_repository_version': VERSION_STUB,
                       'ambari_repositories': [],
                       'package_installation_result': 'FAIL'})
@@ -308,6 +316,10 @@ class TestInstallPackages(RMFTestCase):
                                hdp_versions_mock,
                                allInstalledPackages_mock, put_structured_out_mock,
                                package_mock, is_suse_family_mock):
+    hdp_versions_mock.side_effect = [
+      [],  # before installation attempt
+      [VERSION_STUB]
+    ]
     read_actual_version_from_history_file_mock.return_value = VERSION_STUB
     allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages)
     is_suse_family_mock.return_value = True


Mime
View raw message