ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dmitriu...@apache.org
Subject [2/2] ambari git commit: AMBARI-18435. Provide script to delete an old HDP stack version (dlysnichenko)
Date Mon, 26 Sep 2016 11:09:56 GMT
AMBARI-18435. Provide script to delete an old HDP stack version (dlysnichenko)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a6c12bfc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a6c12bfc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a6c12bfc

Branch: refs/heads/branch-2.5
Commit: a6c12bfc3df56d4b8a4f178edfc60bed48a449d4
Parents: f5f1272
Author: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Authored: Mon Sep 26 14:05:11 2016 +0300
Committer: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Committed: Mon Sep 26 14:09:09 2016 +0300

----------------------------------------------------------------------
 .../test/python/ambari_agent/TestHostInfo.py    |  10 +-
 .../libraries/functions/packages_analyzer.py    |   2 +
 .../system_action_definitions.xml               |  10 ++
 .../scripts/remove_previous_stacks.py           | 118 ++++++++++++++++
 .../custom_actions/TestRemoveStackVersion.py    | 140 +++++++++++++++++++
 .../configs/remove_previous_stacks.json         |  90 ++++++++++++
 6 files changed, 365 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
index 616fd5b..0c0a3b3 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
@@ -105,9 +105,9 @@ class TestHostInfo(TestCase):
     packages_analyzer.allInstalledPackages(installedPackages)
     self.assertEqual(9, len(installedPackages))
     for package in installedPackages:
-      self.assertTrue(package[0] in ["AMBARI.dev.noarch", "PyXML.x86_64", "oracle-server-db.x86",
-                                 "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch",
-                                 "hcatalog.noarch", "hesiod.x86_64", "hive.noarch", "ambari-log4j.noarch",
"libconfuse.x86_64"])
+      self.assertTrue(package[0] in ["AMBARI.dev", "PyXML", "oracle-server-db",
+                                 "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US",
+                                 "hcatalog", "hesiod", "hive", "ambari-log4j", "libconfuse"])
       self.assertTrue(package[1] in ["1.x-1.el6", "0.8.4-19.el6", "3-7.el6", "3.1.0-19.el6",
                                  "0.11.0.1.3.0.0-107.el6", "1.2.5.9-1", "1.3.17-2", "1.2.5.9-1",
"2.7-4.el6"])
       self.assertTrue(package[2] in ["installed", "koji-override-0", "HDP-1.3.0",
@@ -116,8 +116,8 @@ class TestHostInfo(TestCase):
     packages = packages_analyzer.getInstalledPkgsByNames(["AMBARI", "Red_Hat_Enterprise",
"hesiod", "hive"],
                                                        installedPackages)
     self.assertEqual(4, len(packages))
-    expected = ["AMBARI.dev.noarch", "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch",
-                                "hesiod.x86_64", "hive.noarch"]
+    expected = ["AMBARI.dev", "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US",
+                                "hesiod", "hive"]
     for package in expected:
       self.assertTrue(package in packages)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
index 7bbbd3d..52c46ed 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
@@ -155,6 +155,8 @@ def _lookUpYumPackages(command, skipTill, allPackages):
         items = items + line.strip(' \t\n\r').split()
 
       for i in range(0, len(items), 3):
+        if '.' in items[i]:
+          items[i] = items[i][:items[i].rindex('.')]
         if items[i + 2].find('@') == 0:
           items[i + 2] = items[i + 2][1:]
         allPackages.append(items[i:i + 3])

http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
b/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
index fc17584..0f50256 100644
--- a/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
+++ b/ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml
@@ -84,4 +84,14 @@
     <targetType>ANY</targetType>
     <permissions>CLUSTER.UPGRADE_DOWNGRADE_STACK</permissions>
   </actionDefinition>
+  <actionDefinition>
+    <actionName>remove_previous_stacks</actionName>
+    <actionType>SYSTEM</actionType>
+    <inputs>version</inputs>
+    <targetService/>
+    <targetComponent/>
+    <defaultTimeout>600</defaultTimeout>
+    <description>Perform remove old stack version action</description>
+    <targetType>ANY</targetType>
+  </actionDefinition>
 </actionDefinitions>

http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
new file mode 100644
index 0000000..958b800
--- /dev/null
+++ b/ambari-server/src/main/resources/custom_actions/scripts/remove_previous_stacks.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import re
+
+import os
+from resource_management import Script, format, Package, Execute, Fail
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions import stack_tools
+from resource_management.libraries.functions.packages_analyzer import allInstalledPackages
+from resource_management.libraries.functions.stack_select import get_stack_versions
+
+CURRENT_ = "/current/"
+stack_root = Script.get_stack_root()
+stack_root_current = stack_root + CURRENT_
+
+
+class RemovePreviousStacks(Script):
+
+
+  def actionexecute(self, env):
+    config = Script.get_config()
+    structured_output = {}
+    version = config['commandParams']['version']
+    self.stack_tool_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
+
+    versions_to_remove = self.get_lower_versions(version)
+
+    for low_version in versions_to_remove:
+      self.remove_stack_version(structured_output, low_version)
+
+  def remove_stack_version(self, structured_output, version):
+    # check simlinks not refer to version for remove
+    self.check_no_symlink_to_version(structured_output, version)
+    packages_to_remove = self.get_packages_to_remove(version)
+    for package in packages_to_remove:
+      Package(package, action="remove")
+    self.remove_stack_folder(structured_output, version)
+    structured_output["remove_previous_stacks"] = {"exit_code": 0,
+                                       "message": format("Stack version {0} successfully
removed!".format(version))}
+    self.put_structured_out(structured_output)
+
+  def remove_stack_folder(self, structured_output, version):
+    if version and version != '' and stack_root and stack_root != '':
+
+      Logger.info("Removing {0}/{1}".format(stack_root, version))
+      try:
+        Execute(('rm', '-f', stack_root + version),
+                sudo=True)
+      finally:
+        structured_output["remove_previous_stacks"] = {"exit_code": -1,
+                                           "message": "Failed to remove version {0}{1}".format(stack_root,
version)}
+        self.put_structured_out(structured_output)
+
+  def get_packages_to_remove(self, version):
+    packages = []
+    formated_version = version.replace('.', '_').replace('-', '_')
+    all_installed_packages = []
+    allInstalledPackages(all_installed_packages)
+    all_installed_packages = [package[0] for package in all_installed_packages]
+    for package in all_installed_packages:
+      if formated_version in package and self.stack_tool_package not in package:
+        packages.append(package)
+        Logger.info("%s added to remove" % (package))
+    return packages
+
+  def check_no_symlink_to_version(self, structured_output, version):
+    files = os.listdir(stack_root_current)
+    for file in files:
+      if version in os.path.realpath(stack_root_current + file):
+        structured_output["remove_previous_stacks"] = {"exit_code": -1,
+                                           "message": "{0} contains symlink to version for
remove! {1}".format(
+                                             stack_root_current, version)}
+        self.put_structured_out(structured_output)
+        raise Fail("{0} contains symlink to version for remove! {1}".format(stack_root_current,
version))
+
+  def get_lower_versions(self, current_version):
+    versions = get_stack_versions(stack_root)
+    Logger.info("available versions: {0}".format(str(versions)))
+
+    lover_versions = []
+    for version in versions:
+      if self.compare(version, current_version) < 0 :
+        lover_versions.append(version)
+        Logger.info("version %s added to remove" % (version))
+    return lover_versions
+
+  def compare(self, version1, version2):
+    """
+    Compare version1 and version2
+    :param version1:
+    :param version2:
+    :return: Return negative if version1<version2, zero if version1==version2, positive
if version1>version2
+    """
+    vesion1_sections = re.findall(r"[\w']+", version1)
+    vesion2_sections = re.findall(r"[\w']+", version2)
+    return cmp(vesion1_sections, vesion2_sections)
+
+if __name__ == "__main__":
+  RemovePreviousStacks().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py b/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py
new file mode 100644
index 0000000..5a05cc2
--- /dev/null
+++ b/ambari-server/src/test/python/custom_actions/TestRemoveStackVersion.py
@@ -0,0 +1,140 @@
+# !/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+from mock.mock import patch
+from mock.mock import MagicMock
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
+from stacks.utils.RMFTestCase import *
+
+OLD_VERSION_STUB = '2.1.0.0-400'
+VERSION_STUB = '2.2.0.1-885'
+
+@patch.object(Logger, 'logger', new=MagicMock())
+class TestRemoveStackVersion(RMFTestCase):
+
+  @staticmethod
+  def _add_packages(arg):
+    arg.append(["pkg12_1_0_0_400", "1.0", "repo"])
+    arg.append(["pkg22_1_0_1_885", "2.0", "repo2"])
+    arg.append(["hdp-select2_1_0_1_885", "2.0", "repo2"])
+
+  @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
+  @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+  @patch("resource_management.libraries.script.Script.put_structured_out")
+  @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
+  @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
+  @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
+  @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new
= MagicMock(return_value="hdp-select"))
+  @patch("os.listdir", new = MagicMock(return_value=["somefile"]))
+  def test_normal_flow(self,
+                       write_actual_version_to_history_file_mock,
+                       read_actual_version_from_history_file_mock,
+                       stack_versions_mock,
+                       put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock,
):
+
+    stack_versions_mock.return_value = [VERSION_STUB, OLD_VERSION_STUB]
+    allInstalledPackages_mock.side_effect = TestRemoveStackVersion._add_packages
+    list_ambari_managed_repos_mock.return_value = []
+
+    self.executeScript("scripts/remove_previous_stacks.py",
+                       classname="RemovePreviousStacks",
+                       command="actionexecute",
+                       config_file="remove_previous_stacks.json",
+                       target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+                       os_type=('Redhat', '6.4', 'Final')
+                       )
+    self.assertTrue(stack_versions_mock.called)
+    self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
+
+    self.assertResourceCalled('Package', "pkg12_1_0_0_400", action=["remove"])
+    self.assertTrue(put_structured_out_mock.called)
+    self.assertEquals(put_structured_out_mock.call_args[0][0],
+                      {'remove_previous_stacks': {'exit_code': 0,
+                       'message': 'Stack version 2.1.0.0-400 successfully removed!'}})
+    self.assertResourceCalled('Execute', ('rm', '-f', '/usr/hdp2.1.0.0-400'),
+                              sudo = True,
+                              )
+    self.assertNoMoreResources()
+
+  @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
+  @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+  @patch("resource_management.libraries.script.Script.put_structured_out")
+  @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
+  @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
+  @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
+  @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new
= MagicMock(return_value="hdp-select"))
+  @patch("os.listdir", new = MagicMock(return_value=["somefile"]))
+  def test_without_versions(self,
+                       write_actual_version_to_history_file_mock,
+                       read_actual_version_from_history_file_mock,
+                       stack_versions_mock,
+                       put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock
):
+
+    stack_versions_mock.return_value = [VERSION_STUB]
+    allInstalledPackages_mock.side_effect = TestRemoveStackVersion._add_packages
+    list_ambari_managed_repos_mock.return_value = []
+
+    self.executeScript("scripts/remove_previous_stacks.py",
+                       classname="RemovePreviousStacks",
+                       command="actionexecute",
+                       config_file="remove_previous_stacks.json",
+                       target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+                       os_type=('Redhat', '6.4', 'Final')
+                       )
+    self.assertTrue(stack_versions_mock.called)
+    self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
+    self.assertNoMoreResources()
+
+  @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos")
+  @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages")
+  @patch("resource_management.libraries.script.Script.put_structured_out")
+  @patch("resource_management.libraries.functions.stack_select.get_stack_versions")
+  @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file")
+  @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file")
+  @patch("resource_management.libraries.functions.stack_tools.get_stack_tool_package", new
= MagicMock(return_value="hdp-select"))
+  @patch("os.listdir", new = MagicMock(return_value=["somefile" + OLD_VERSION_STUB]))
+  def test_symlink_exist(self,
+                       write_actual_version_to_history_file_mock,
+                       read_actual_version_from_history_file_mock,
+                       stack_versions_mock,
+                       put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock,
):
+
+    stack_versions_mock.return_value = [VERSION_STUB, OLD_VERSION_STUB]
+    allInstalledPackages_mock.side_effect = TestRemoveStackVersion._add_packages
+    list_ambari_managed_repos_mock.return_value = []
+
+    try:
+      self.executeScript("scripts/remove_previous_stacks.py",
+                       classname="RemovePreviousStacks",
+                       command="actionexecute",
+                       config_file="remove_previous_stacks.json",
+                       target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+                       os_type=('Redhat', '6.4', 'Final')
+                       )
+      self.fail("Should throw exception")
+    except Fail, e:
+      self.assertEquals(str(e), '/usr/hdp/current/ contains symlink to version for remove!
2.1.0.0-400')
+      pass  # Expected
+
+    self.assertTrue(stack_versions_mock.called)
+    self.assertEquals(stack_versions_mock.call_args[0][0], '/usr/hdp')
+    self.assertNoMoreResources()
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/a6c12bfc/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
b/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
new file mode 100644
index 0000000..cc4a626
--- /dev/null
+++ b/ambari-server/src/test/python/custom_actions/configs/remove_previous_stacks.json
@@ -0,0 +1,90 @@
+{
+  "configuration_attributes": {},
+  "roleCommand": "ACTIONEXECUTE",
+  "clusterName": "cc",
+  "hostname": "0b3.vm",
+  "passiveInfo": [],
+  "hostLevelParams": {
+    "agent_stack_retry_count": "5",
+    "agent_stack_retry_on_unavailability": "false",
+    "jdk_location": "http://0b3.vm:8080/resources/",
+    "ambari_db_rca_password": "mapred",
+    "java_home": "/usr/jdk64/jdk1.7.0_67",
+    "java_version": "8",
+    "ambari_db_rca_url": "jdbc:postgresql://0b3.vm/ambarirca",
+    "jce_name": "UnlimitedJCEPolicyJDK7.zip",
+    "oracle_jdbc_url": "http://0b3.vm:8080/resources//ojdbc6.jar",
+    "stack_version": "2.1",
+    "stack_name": "HDP",
+    "db_name": "ambari",
+    "ambari_db_rca_driver": "org.postgresql.Driver",
+    "jdk_name": "jdk-7u67-linux-x64.tar.gz",
+    "ambari_db_rca_username": "mapred",
+    "db_driver_filename": "mysql-connector-java.jar",
+    "mysql_jdbc_url": "http://0b3.vm:8080/resources//mysql-connector-java.jar"
+  },
+  "commandType": "SYSTEM",
+  "serviceName": "null",
+  "role": "remove_previous_stacks",
+  "forceRefreshConfigTags": [],
+  "taskId": 61,
+  "public_hostname": "0b3.vm",
+  "configurations": {
+    "cluster-env": {
+      "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{%
else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
+      "repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}"
+    }
+  },
+  "commandParams": {
+    "command_timeout": "60",
+    "script_type": "PYTHON",
+    "repository_version": "2.2.0.1-885",
+    "script": "remove_previous_stacks.py",
+    "version": "2.2.0.1-885"
+  },
+  "commandId": "14-1",
+  "clusterHostInfo": {
+    "snamenode_host": [
+      "0b3.vm"
+    ],
+    "nm_hosts": [
+      "0b3.vm"
+    ],
+    "app_timeline_server_hosts": [
+      "0b3.vm"
+    ],
+    "all_ping_ports": [
+      "8670"
+    ],
+    "rm_host": [
+      "0b3.vm"
+    ],
+    "all_hosts": [
+      "0b3.vm"
+    ],
+    "slave_hosts": [
+      "0b3.vm"
+    ],
+    "namenode_host": [
+      "0b3.vm"
+    ],
+    "ambari_server_host": [
+      "0b3.vm"
+    ],
+    "zookeeper_hosts": [
+      "0b3.vm"
+    ],
+    "hs_host": [
+      "0b3.vm"
+    ]
+  },
+  "configurations": {
+    "cluster-env": {
+      "repo_suse_rhel_template": "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{%
else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
+      "repo_ubuntu_template": "{{package_type}} {{base_url}} {{components}}"
+    },
+    "core-site": {
+      "io.compression.codecs": "com.hadoop.compression.lzo"
+    }
+  }
+}


Mime
View raw message