ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hapyles...@apache.org
Subject [7/8] ambari git commit: AMBARI-21398 Backend need to distinguish repo_version records by using the Stack Name (dgrinenko)
Date Thu, 17 Aug 2017 10:04:20 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 54a7e6d..313b666 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -51,13 +51,13 @@ from resource_management.libraries.functions.version import compare_versions
 from resource_management.libraries.functions.version import format_stack_version
 from resource_management.libraries.functions import stack_tools
 from resource_management.libraries.functions.constants import Direction
-from resource_management.libraries.functions import packages_analyzer
 from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration
 from resource_management.core.resources.system import Execute
 from contextlib import closing
 from resource_management.libraries.functions.stack_features import check_stack_feature
 from resource_management.libraries.functions.constants import StackFeature
 from resource_management.libraries.functions.show_logs import show_logs
+from resource_management.core.providers import get_provider
 from resource_management.libraries.functions.fcntl_based_process_lock import FcntlBasedProcessLock
 
 import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json
module and has the same functions set.
@@ -435,7 +435,9 @@ class Script(object):
     if not Script.stack_version_from_distro_select or '*' in Script.stack_version_from_distro_select:
       # FIXME: this method is not reliable to get stack-selector-version
       # as if there are multiple versions installed with different <stack-selector-tool>,
we won't detect the older one (if needed).
-      Script.stack_version_from_distro_select = packages_analyzer.getInstalledPackageVersion(
+      pkg_provider = get_provider("Package")
+
+      Script.stack_version_from_distro_select = pkg_provider.get_installed_package_version(
               stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME))
 
     return Script.stack_version_from_distro_select
@@ -456,12 +458,19 @@ class Script(object):
         return package
 
 
-  def format_package_name(self, name):
+  def format_package_name(self, name, repo_version=None):
     from resource_management.libraries.functions.default import default
     """
     This function replaces ${stack_version} placeholder with actual version.  If the package
     version is passed from the server, use that as an absolute truth.
+    
+    :param name name of the package
+    :param repo_version actual version of the repo currently installing
     """
+    stack_version_package_formatted = ""
+
+    if not repo_version:
+      repo_version = self.get_stack_version_before_packages_installed()
 
     package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_'
 
@@ -497,7 +506,7 @@ class Script(object):
 
     # Wildcards cause a lot of troubles with installing packages, if the version contains
wildcards we try to specify it.
     if not package_version or '*' in package_version:
-      stack_version_package_formatted = self.get_stack_version_before_packages_installed().replace('.',
package_delimiter).replace('-', package_delimiter) if STACK_VERSION_PLACEHOLDER in name else
name
+      stack_version_package_formatted = repo_version.replace('.', package_delimiter).replace('-',
package_delimiter) if STACK_VERSION_PLACEHOLDER in name else name
 
     package_name = name.replace(STACK_VERSION_PLACEHOLDER, stack_version_package_formatted)
 
@@ -703,7 +712,7 @@ class Script(object):
 
     if 'host_sys_prepped' in config['hostLevelParams']:
       # do not install anything on sys-prepped host
-      if config['hostLevelParams']['host_sys_prepped'] == True:
+      if config['hostLevelParams']['host_sys_prepped'] is True:
         Logger.info("Node has all packages pre-installed. Skipping.")
         return
       pass
@@ -711,8 +720,9 @@ class Script(object):
       package_list_str = config['hostLevelParams']['package_list']
       agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability'])
       agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count'])
+      pkg_provider = get_provider("Package")
       try:
-        available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(config['repositoryFile']['repositories'])
+        available_packages_in_repos = pkg_provider.get_available_packages_in_repos(config['repositoryFile']['repositories'])
       except Exception as err:
         available_packages_in_repos = []
       if isinstance(package_list_str, basestring) and len(package_list_str) > 0:

http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
index 358c104..a485415 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
@@ -27,7 +27,6 @@ import socket
 import getpass
 import tempfile
 
-from resource_management.libraries.functions import packages_analyzer
 from resource_management.libraries.functions.default import default
 from ambari_commons import os_utils
 from ambari_commons.os_check import OSCheck, OSConst
@@ -40,6 +39,7 @@ from resource_management.core.exceptions import Fail
 from ambari_commons.constants import AMBARI_SUDO_BINARY
 from resource_management.core import shell
 from resource_management.core.logger import Logger
+from resource_management.core.providers import get_provider
 
 # WARNING. If you are adding a new host check that is used by cleanup, add it to BEFORE_CLEANUP_HOST_CHECKS
 # It is used by HostCleanup.py
@@ -113,6 +113,7 @@ class CheckHost(Script):
   
   def __init__(self):
     self.reportFileHandler = HostCheckReportFileHandler()
+    self.pkg_provider = get_provider("Package")
   
   def actionexecute(self, env):
     Logger.info("Host checks started.")
@@ -229,21 +230,18 @@ class CheckHost(Script):
   def execute_existing_repos_and_installed_packages_check(self, config):
       Logger.info("Installed packages and existing repos checks started.")
 
-      installedPackages = []
-      availablePackages = []
-      packages_analyzer.allInstalledPackages(installedPackages)
-      packages_analyzer.allAvailablePackages(availablePackages)
-
-      repos = []
-      packages_analyzer.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages,
-                                      self.IGNORE_PACKAGES_FROM_REPOS, repos)
-      packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES,
installedPackages)
-      additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames(
-        self.ADDITIONAL_PACKAGES, installedPackages)
+      installedPackages = self.pkg_provider.all_installed_packages()
+      availablePackages = self.pkg_provider.all_available_packages()
+
+      repos = self.pkg_provider.get_installed_repos(self.PACKAGES, installedPackages + availablePackages,
+                                                    self.IGNORE_PACKAGES_FROM_REPOS)
+
+      packagesInstalled = self.pkg_provider.get_installed_pkgs_by_repo(repos, self.IGNORE_PACKAGES,
installedPackages)
+      additionalPkgsInstalled = self.pkg_provider.get_installed_pkgs_by_names(self.ADDITIONAL_PACKAGES,
installedPackages)
       allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
       
-      installedPackages = packages_analyzer.getPackageDetails(installedPackages, allPackages)
-      repos = packages_analyzer.getReposToRemove(repos, self.IGNORE_REPOS)
+      installedPackages = self.pkg_provider.get_package_details(installedPackages, allPackages)
+      repos = self.pkg_provider.get_repos_to_remove(repos, self.IGNORE_REPOS)
 
       Logger.info("Installed packages and existing repos checks completed.")
       return installedPackages, repos

http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
index a86a71d..189043b 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
@@ -28,20 +28,21 @@ from ambari_commons.str_utils import cbool, cint
 from resource_management.core.exceptions import Fail
 from resource_management.core.logger import Logger
 from resource_management.core.resources import Package
-from resource_management.libraries.functions.packages_analyzer import allInstalledPackages,
verifyDependencies
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_tools
 from resource_management.libraries.functions.stack_select import get_stack_versions
 from resource_management.libraries.functions.version import format_stack_version
 from resource_management.libraries.functions.repo_version_history \
     import read_actual_version_from_history_file, write_actual_version_to_history_file, REPO_VERSION_HISTORY_FILE
+from resource_management.core.providers import get_provider
 from resource_management.core.resources.system import Link
 from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions import packages_analyzer
 from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository
 from resource_management.libraries.functions.stack_features import check_stack_feature
 from resource_management.libraries.resources.repository import Repository
 from resource_management.libraries.script.script import Script
+from resource_management.core import sudo
+
 
 class InstallPackages(Script):
   """
@@ -53,6 +54,11 @@ class InstallPackages(Script):
 
   UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
 
+  def __init__(self):
+    super(InstallPackages, self).__init__()
+
+    self.pkg_provider = get_provider("Package")
+
   def actionexecute(self, env):
     num_errors = 0
 
@@ -83,8 +89,6 @@ class InstallPackages(Script):
     except KeyError:
       pass
 
-
-
     self.stack_name = Script.get_stack_name()
     if self.stack_name is None:
       raise Fail("Cannot determine the stack name")
@@ -99,7 +103,7 @@ class InstallPackages(Script):
     self.repository_version = self.repository_version.strip()
 
     try:
-      if 0 == len(command_repository.repositories):
+      if not command_repository.repositories:
         Logger.warning(
           "Repository list is empty. Ambari may not be managing the repositories for {0}.".format(
             self.repository_version))
@@ -171,13 +175,12 @@ class InstallPackages(Script):
 
     # After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this
symlink
     if self.stack_name.upper() == "HDF" \
-            and not os.path.exists("/usr/bin/conf-select") and os.path.exists("/usr/bin/hdfconf-select"):
-      Link("/usr/bin/conf-select", to = "/usr/bin/hdfconf-select")
+            and not sudo.path_exists("/usr/bin/conf-select") and sudo.path_exists("/usr/bin/hdfconf-select"):
+      Link("/usr/bin/conf-select", to="/usr/bin/hdfconf-select")
 
     for package_name, directories in conf_select.get_package_dirs().iteritems():
       conf_select.select(self.stack_name, package_name, stack_version, ignore_errors = True)
 
-
   def compute_actual_version(self):
     """
     After packages are installed, determine what the new actual version is.
@@ -313,6 +316,7 @@ class InstallPackages(Script):
 
     # Install packages
     packages_were_checked = False
+    packages_installed_before = []
     stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
     try:
       Package(stack_selector_package,
@@ -321,13 +325,12 @@ class InstallPackages(Script):
               retry_count=agent_stack_retry_count
       )
 
-      packages_installed_before = []
-      allInstalledPackages(packages_installed_before)
+      packages_installed_before = self.pkg_provider.all_installed_packages()
       packages_installed_before = [package[0] for package in packages_installed_before]
       packages_were_checked = True
       filtered_package_list = self.filter_package_list(package_list)
       try:
-        available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(config['repositoryFile']['repositories'])
+        available_packages_in_repos = self.pkg_provider.get_available_packages_in_repos(config['repositoryFile']['repositories'])
       except Exception:
         available_packages_in_repos = []
       for package in filtered_package_list:
@@ -343,8 +346,7 @@ class InstallPackages(Script):
 
       # Remove already installed packages in case of fail
       if packages_were_checked and packages_installed_before:
-        packages_installed_after = []
-        allInstalledPackages(packages_installed_after)
+        packages_installed_after = self.pkg_provider.all_installed_packages()
         packages_installed_after = [package[0] for package in packages_installed_after]
         packages_installed_before = set(packages_installed_before)
         new_packages_installed = [package for package in packages_installed_after if package
not in packages_installed_before]
@@ -361,7 +363,7 @@ class InstallPackages(Script):
 
     if self.ignore_package_dependencies:
       Logger.info("Ignoring package dependencies")
-    elif not verifyDependencies():
+    elif not self.pkg_provider.verify_dependencies():
       ret_code = 1
       Logger.logger.error("Failure while verifying dependencies")
       Logger.logger.error("*******************************************************************************")
@@ -379,36 +381,6 @@ class InstallPackages(Script):
       Logger.logger.exception("Failure while computing actual version. Error: {0}".format(str(err)))
     return ret_code
 
-  def install_repository(self, url_info, append_to_file, template):
-
-    repo = {
-      'repoName': "{0}-{1}".format(url_info['name'], self.repository_version)
-    }
-
-    if not 'baseUrl' in url_info:
-      repo['baseurl'] = None
-    else:
-      repo['baseurl'] = url_info['baseUrl']
-
-    if not 'mirrorsList' in url_info:
-      repo['mirrorsList'] = None
-    else:
-      repo['mirrorsList'] = url_info['mirrorsList']
-
-    ubuntu_components = [url_info['name']] + self.UBUNTU_REPO_COMPONENTS_POSTFIX
-    file_name = self.stack_name + "-" + self.repository_version
-
-    Repository(repo['repoName'],
-      action = "create",
-      base_url = repo['baseurl'],
-      mirror_list = repo['mirrorsList'],
-      repo_file_name = file_name,
-      repo_template = template,
-      append_to_file = append_to_file,
-      components = ubuntu_components,  # ubuntu specific
-    )
-    return repo['repoName'], file_name
-
   def abort_handler(self, signum, frame):
     Logger.error("Caught signal {0}, will handle it gracefully. Compute the actual version
if possible before exiting.".format(signum))
     self.check_partial_install()

http://git-wip-us.apache.org/repos/asf/ambari/blob/33879743/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
index 958b800..2b50171 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
@@ -25,8 +25,8 @@ import os
 from resource_management import Script, format, Package, Execute, Fail
 from resource_management.core.logger import Logger
 from resource_management.libraries.functions import stack_tools
-from resource_management.libraries.functions.packages_analyzer import allInstalledPackages
 from resource_management.libraries.functions.stack_select import get_stack_versions
+from resource_management.core.providers import get_provider
 
 CURRENT_ = "/current/"
 stack_root = Script.get_stack_root()
@@ -43,6 +43,7 @@ class RemovePreviousStacks(Script):
     self.stack_tool_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
 
     versions_to_remove = self.get_lower_versions(version)
+    self.pkg_provider = get_provider("Package")
 
     for low_version in versions_to_remove:
       self.remove_stack_version(structured_output, low_version)
@@ -73,8 +74,8 @@ class RemovePreviousStacks(Script):
   def get_packages_to_remove(self, version):
     packages = []
     formated_version = version.replace('.', '_').replace('-', '_')
-    all_installed_packages = []
-    allInstalledPackages(all_installed_packages)
+    all_installed_packages = self.pkg_provider.all_installed_packages()
+
     all_installed_packages = [package[0] for package in all_installed_packages]
     for package in all_installed_packages:
       if formated_version in package and self.stack_tool_package not in package:


Mime
View raw message