ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject ambari git commit: AMBARI-22561. Need to address HDP-GPL repo update after user accepts license in post-install scenario (aonishuk)
Date Fri, 01 Dec 2017 12:48:21 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.6 64cb27fae -> 3ac717a49


AMBARI-22561. Need to address HDP-GPL repo update after user accepts license in post-install
scenario (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3ac717a4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3ac717a4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3ac717a4

Branch: refs/heads/branch-2.6
Commit: 3ac717a497efc60ca9b51f8ca7bef21f3df26fb3
Parents: 64cb27f
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Fri Dec 1 14:47:47 2017 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Fri Dec 1 14:47:47 2017 +0200

----------------------------------------------------------------------
 .../libraries/functions/lzo_utils.py            |  9 +-
 .../libraries/functions/repository_util.py      | 92 ++++++++++++--------
 .../libraries/script/script.py                  | 10 ++-
 .../ambari/server/agent/CommandRepository.java  |  6 ++
 .../custom_actions/scripts/install_packages.py  |  8 +-
 .../scripts/repo_initialization.py              |  9 +-
 .../src/test/python/stacks/utils/RMFTestCase.py |  2 +
 7 files changed, 85 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac717a4/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
index 68ee607..c505969 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
@@ -54,6 +54,9 @@ def get_lzo_packages():
 
   return lzo_packages
 
+def is_gpl_license_accepted():
+  return default("/hostLevelParams/gpl_license_accepted", False)
+
 def should_install_lzo():
   """
   Return true if lzo is enabled via core-site.xml and GPL license (required for lzo) is accepted.
@@ -65,8 +68,7 @@ def should_install_lzo():
   if not lzo_enabled:
     return False
 
-  is_gpl_license_accepted = default("/hostLevelParams/gpl_license_accepted", False)
-  if not is_gpl_license_accepted:
+  if not is_gpl_license_accepted():
     Logger.warning(INSTALLING_LZO_WITHOUT_GPL)
     return False
 
@@ -79,6 +81,9 @@ def install_lzo_if_needed():
   if not should_install_lzo():
     return
 
+  # If user has just accepted GPL license. GPL repository can not yet be present.
+  Script.repository_util.create_repo_files()
+
   lzo_packages = get_lzo_packages()
 
   config = Script.get_config()

http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac717a4/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
index f1c8ef1..5d73b5d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
@@ -17,60 +17,75 @@ limitations under the License.
 
 """
 
+from ambari_commons.os_check import OSCheck
 from resource_management.core.exceptions import Fail
 from resource_management.core.logger import Logger
 from resource_management.libraries.resources.repository import Repository
+from resource_management.libraries.functions.is_empty import is_empty
 import ambari_simplejson as json
 
 
-__all__ = ["create_repo_files", "CommandRepository"]
+__all__ = ["RepositoryUtil", "CommandRepository"]
 
 # components_lits = repoName + postfix
 UBUNTU_REPO_COMPONENTS_POSTFIX = "main"
 
+class RepositoryUtil:
+  def __init__(self, config, tags_to_skip):
+    self.tags_to_skip = tags_to_skip
 
-def create_repo_files(template, command_repository):
-  """
-  Creates repositories in a consistent manner for all types
-  :param command_repository: a CommandRepository instance
-  :type command_repository CommandRepository
-  :return: a dictionary with repo ID => repo file name mapping
-  """
-
-  if command_repository.version_id is None:
-    raise Fail("The command repository was not parsed correctly")
-
-  if 0 == len(command_repository.items):
-    Logger.warning(
-      "Repository for {0}/{1} has no repositories.  Ambari may not be managing this version.".format(
-        command_repository.stack_name, command_repository.version_string))
-    return {}
+    # repo templates
+    repo_file = config['repositoryFile']
+    repo_rhel_suse =  config['configurations']['cluster-env']['repo_suse_rhel_template']
+    repo_ubuntu =  config['configurations']['cluster-env']['repo_ubuntu_template']
 
-  append_to_file = False  # initialize to False to create the file anew.
-  repo_files = {}
+    if is_empty(repo_file):
+      return
 
-  for repository in command_repository.items:
+    self.template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family()
else repo_ubuntu
+    self.command_repository = CommandRepository(repo_file)
 
-    if repository.repo_id is None:
-      raise Fail("Repository with url {0} has no id".format(repository.base_url))
+  def create_repo_files(self):
+    """
+    Creates repositories in a consistent manner for all types
+    :return: a dictionary with repo ID => repo file name mapping
+    """
+    if self.command_repository.version_id is None:
+      raise Fail("The command repository was not parsed correctly")
 
-    if not repository.ambari_managed:
+    if 0 == len(self.command_repository.items):
       Logger.warning(
-        "Repository for {0}/{1}/{2} is not managed by Ambari".format(
-          command_repository.stack_name, command_repository.version_string, repository.repo_id))
-    else:
-      Repository(repository.repo_id,
-                 action="create",
-                 base_url=repository.base_url,
-                 mirror_list=repository.mirrors_list,
-                 repo_file_name=command_repository.repo_filename,
-                 repo_template=template,
-                 components=repository.ubuntu_components,
-                 append_to_file=append_to_file)
-      append_to_file = True
-      repo_files[repository.repo_id] = command_repository.repo_filename
-
-  return repo_files
+        "Repository for {0}/{1} has no repositories.  Ambari may not be managing this version.".format(
+          self.command_repository.stack_name, self.command_repository.version_string))
+      return {}
+
+    append_to_file = False  # initialize to False to create the file anew.
+    repo_files = {}
+    for repository in self.command_repository.items:
+      if repository.repo_id is None:
+        raise Fail("Repository with url {0} has no id".format(repository.base_url))
+
+      if self.tags_to_skip & repository.tags:
+        Logger.info("Repository with url {0} is not created due to its tags: {1}".format(repository.base_url,
repository.tags))
+        continue
+
+      if not repository.ambari_managed:
+        Logger.warning(
+          "Repository for {0}/{1}/{2} is not managed by Ambari".format(
+            self.command_repository.stack_name, self.command_repository.version_string, repository.repo_id))
+      else:
+        Repository(repository.repo_id,
+                   action="create",
+                   base_url=repository.base_url,
+                   mirror_list=repository.mirrors_list,
+                   repo_file_name=self.command_repository.repo_filename,
+                   repo_template=self.template,
+                   components=repository.ubuntu_components,
+                   append_to_file=append_to_file)
+        append_to_file = True
+        repo_files[repository.repo_id] = self.command_repository.repo_filename
+
+    return repo_files
 
 
 def _find_value(dictionary, key, default=None):
@@ -146,6 +161,7 @@ class CommandRepositoryItem(object):
     self.components = _find_value(json_dict, 'components')
     self.base_url = _find_value(json_dict, 'baseUrl')
     self.mirrors_list = _find_value(json_dict, 'mirrorsList')
+    self.tags = set(_find_value(json_dict, 'tags', default=[]))
     self.ambari_managed = _find_value(json_dict, 'ambariManaged', default=True)
 
     self.ubuntu_components = [self.distribution if self.distribution else self.repo_name]
+ \

http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac717a4/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index ccb8a54..fbb90a0 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -52,7 +52,7 @@ from resource_management.libraries.functions.version import format_stack_version
 from resource_management.libraries.functions import stack_tools
 from resource_management.libraries.functions.constants import Direction
 from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration
-from resource_management.libraries.functions.repository_util import CommandRepository
+from resource_management.libraries.functions.repository_util import CommandRepository, RepositoryUtil
 from resource_management.core.resources.system import Execute
 from contextlib import closing
 from resource_management.libraries.functions.stack_features import check_stack_feature
@@ -355,6 +355,14 @@ class Script(object):
       Logger.logger.exception("Can not read json file with command parameters: ")
       sys.exit(1)
 
+    from resource_management.libraries.functions import lzo_utils
+
+    repo_tags_to_skip = set()
+    if not lzo_utils.is_gpl_license_accepted():
+      repo_tags_to_skip.add("GPL")
+
+    Script.repository_util = RepositoryUtil(Script.config, repo_tags_to_skip)
+
     # Run class method depending on a command type
     try:
       method = self.choose_method_to_execute(self.command_name)

http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac717a4/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
index e207ac5..0777edd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
@@ -20,11 +20,13 @@ package org.apache.ambari.server.agent;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.ambari.annotations.Experimental;
 import org.apache.ambari.annotations.ExperimentalFeature;
 import org.apache.ambari.server.orm.entities.RepositoryEntity;
 import org.apache.ambari.server.state.RepositoryInfo;
+import org.apache.ambari.server.state.stack.RepoTag;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.builder.ToStringBuilder;
 
@@ -256,6 +258,9 @@ public class CommandRepository {
     @SerializedName("mirrorsList")
     private String m_mirrorsList;
 
+    @SerializedName("tags")
+    private Set<RepoTag> m_tags;
+
     @SerializedName("applicableServices")
     @Experimental(feature = ExperimentalFeature.CUSTOM_SERVICE_REPOS,
       comment = "Remove logic for handling custom service repos after enabling multi-mpack
cluster deployment")
@@ -283,6 +288,7 @@ public class CommandRepository {
       m_mirrorsList = entity.getMirrorsList();
       m_osType = osType;
       m_applicableServices = entity.getApplicableServices();
+      m_tags = entity.getTags();
     }
 
     public void setRepoId(String repoId){

http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac717a4/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
index b819dca..9542b74 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
@@ -37,7 +37,7 @@ from resource_management.libraries.functions.repo_version_history \
 from resource_management.core.providers import get_provider
 from resource_management.core.resources.system import Link
 from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository
+from resource_management.libraries.functions.repository_util import CommandRepository
 from resource_management.libraries.functions.stack_features import check_stack_feature
 from resource_management.libraries.resources.repository import Repository
 from resource_management.libraries.script.script import Script
@@ -72,10 +72,6 @@ class InstallPackages(Script):
     except KeyError:
       raise Fail("The command repository indicated by 'repositoryFile' was not found")
 
-    repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
-    repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
-    template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else
repo_ubuntu
-
     # Handle a SIGTERM and SIGINT gracefully
     signal.signal(signal.SIGTERM, self.abort_handler)
     signal.signal(signal.SIGINT, self.abort_handler)
@@ -112,7 +108,7 @@ class InstallPackages(Script):
       else:
         Logger.info(
           "Will install packages for repository version {0}".format(self.repository_version))
-        new_repo_files = create_repo_files(template, command_repository)
+        new_repo_files = Script.repository_util.create_repo_files()
         self.repo_files.update(new_repo_files)
     except Exception, err:
       Logger.logger.exception("Cannot install repository files. Error: {0}".format(str(err)))

http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac717a4/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py
index 82e57aa..a329167 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py
@@ -19,7 +19,8 @@ limitations under the License.
 
 from ambari_commons.os_check import OSCheck
 from resource_management.libraries.resources.repository import Repository
-from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository,
UBUNTU_REPO_COMPONENTS_POSTFIX
+from resource_management.libraries.functions.repository_util import CommandRepository, UBUNTU_REPO_COMPONENTS_POSTFIX
+from resource_management.libraries.script.script import Script
 from resource_management.core.logger import Logger
 import ambari_simplejson as json
 
@@ -62,13 +63,13 @@ def install_repos():
   if params.host_sys_prepped:
     return
 
-  template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family()
else params.repo_ubuntu
-
   # use this newer way of specifying repositories, if available
   if params.repo_file is not None:
-    create_repo_files(template, CommandRepository(params.repo_file))
+    Script.repository_util.create_repo_files()
     return
 
+  template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family()
else params.repo_ubuntu
+
   _alter_repo("create", params.repo_info, template)
 
   if params.service_repo_info:

http://git-wip-us.apache.org/repos/asf/ambari/blob/3ac717a4/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
index 80712c5..5e7f41e 100644
--- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
+++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
@@ -38,6 +38,7 @@ with patch("platform.linux_distribution", return_value = ('Suse','11','Final')):
     from resource_management.libraries.script.config_dictionary import ConfigDictionary
     from resource_management.libraries.script.script import Script
     from resource_management.libraries.script.config_dictionary import UnknownConfiguration
+    from resource_management.libraries.functions.repository_util import RepositoryUtil
 
 PATH_TO_STACKS = "main/resources/stacks/HDP"
 PATH_TO_STACK_TESTS = "test/python/stacks/"
@@ -123,6 +124,7 @@ class RMFTestCase(TestCase):
         script_class_inst = RMFTestCase._get_attr(script_module, classname)()
         script_class_inst.log_out_files = log_out_files
         script_class_inst.available_packages_in_repos = available_packages_in_repos
+        Script.repository_util = RepositoryUtil(self.config_dict, set())
         method = RMFTestCase._get_attr(script_class_inst, command)
     except IOError, err:
       raise RuntimeError("Cannot load class %s from %s: %s" % (classname, norm_path, err.message))


Mime
View raw message