ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-13518 Host cleanup should remove only stack directories from /usr/hdp (dsen)
Date Thu, 22 Oct 2015 14:18:28 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk b9bb0d351 -> a8445c9aa


AMBARI-13518 Host cleanup should remove only stack directories from /usr/hdp (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a8445c9a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a8445c9a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a8445c9a

Branch: refs/heads/trunk
Commit: a8445c9aa699622be486695fcfc6a1c9cf10a518
Parents: b9bb0d3
Author: Dmytro Sen <dsen@apache.org>
Authored: Thu Oct 22 17:18:18 2015 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Thu Oct 22 17:18:18 2015 +0300

----------------------------------------------------------------------
 .../ambari_agent/HostCheckReportFileHandler.py  | 44 ++++++++++--
 .../TestHostCheckReportFileHandler.py           | 72 ++++++++++++++------
 2 files changed, 89 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a8445c9a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
index 794e427..1f87a73 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -16,15 +16,19 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""
 
 import datetime
 import os.path
 import logging
+import re
 import traceback
 from AmbariConfig import AmbariConfig
-import ConfigParser;
+import ConfigParser
 
+HADOOP_ROOT_DIR = "/usr/hdp"
+HADOOP_PERM_REMOVE_LIST = ["current"]
+HADOOP_ITEMDIR_REGEX = "(\d\.){3}\d-\d{4}"
 logger = logging.getLogger(__name__)
 
 class HostCheckReportFileHandler:
@@ -83,6 +87,37 @@ class HostCheckReportFileHandler:
       logger.error("Can't write host check file at %s :%s " % (self.hostCheckCustomActionsFilePath,
err.message))
       traceback.print_exc()
 
+  def _hdp_list_directory(self):
+    """
+    Return filtered list of /usr/hdp directory allowed to be removed
+    :rtype list
+    """
+
+    if not os.path.exists(HADOOP_ROOT_DIR):
+      return []
+
+    matcher = re.compile(HADOOP_ITEMDIR_REGEX)  # pre-compile regexp
+    folder_content = os.listdir(HADOOP_ROOT_DIR)
+    remove_list = []
+
+    remlist_items_count = 0
+
+    for item in folder_content:
+      full_path = "%s%s%s" % (HADOOP_ROOT_DIR, os.path.sep, item)
+      if item in HADOOP_PERM_REMOVE_LIST:
+        remove_list.append(full_path)
+        remlist_items_count += 1
+
+      if matcher.match(item) is not None:
+        remove_list.append(full_path)
+        remlist_items_count += 1
+
+    # if remove list have same length as target folder, assume that they are identical
+    if remlist_items_count == len(folder_content):
+      remove_list.append(HADOOP_ROOT_DIR)
+
+    return remove_list
+
   def writeHostCheckFile(self, hostInfo):
     if self.hostCheckFilePath is None:
       return
@@ -117,8 +152,7 @@ class HostCheckReportFileHandler:
         items = []
         for itemDetail in hostInfo['stackFoldersAndFiles']:
           items.append(itemDetail['name'])
-        if os.path.exists('/usr/hdp'):
-          items.append('/usr/hdp')
+        items += self._hdp_list_directory()
         config.add_section('directories')
         config.set('directories', 'dir_list', ','.join(items))
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/a8445c9a/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py b/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py
index d56ad8f..c595082 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py
@@ -20,6 +20,7 @@ limitations under the License.
 
 from unittest import TestCase
 import unittest
+from mock.mock import patch
 import os
 import tempfile
 from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler
@@ -38,8 +39,8 @@ class TestHostCheckReportFileHandler(TestCase):
     config.set('agent', 'prefix', os.path.dirname(tmpfile))
 
     handler = HostCheckReportFileHandler(config)
-    dict = {}
-    handler.writeHostCheckFile(dict)
+    mydict = {}
+    handler.writeHostCheckFile(mydict)
 
     configValidator = ConfigParser.RawConfigParser()
     configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
@@ -56,16 +57,16 @@ class TestHostCheckReportFileHandler(TestCase):
     config.set('agent', 'prefix', os.path.dirname(tmpfile))
 
     handler = HostCheckReportFileHandler(config)
-    dict = {}
-    dict['hostHealth'] = {}
-    dict['existingUsers'] = []
-    dict['alternatives'] = []
-    dict['stackFoldersAndFiles'] = []
-    dict['hostHealth']['activeJavaProcs'] = []
-    dict['installedPackages'] = []
-    dict['existingRepos'] = []
+    mydict = {}
+    mydict['hostHealth'] = {}
+    mydict['existingUsers'] = []
+    mydict['alternatives'] = []
+    mydict['stackFoldersAndFiles'] = []
+    mydict['hostHealth']['activeJavaProcs'] = []
+    mydict['installedPackages'] = []
+    mydict['existingRepos'] = []
 
-    handler.writeHostCheckFile(dict)
+    handler.writeHostCheckFile(mydict)
 
     configValidator = ConfigParser.RawConfigParser()
     configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
@@ -96,19 +97,19 @@ class TestHostCheckReportFileHandler(TestCase):
 
     handler = HostCheckReportFileHandler(config)
 
-    dict = {}
-    dict['hostHealth'] = {}
-    dict['existingUsers'] = [{'name':'user1', 'homeDir':'/var/log', 'status':'Exists'}]
-    dict['alternatives'] = [
+    mydict = {}
+    mydict['hostHealth'] = {}
+    mydict['existingUsers'] = [{'name':'user1', 'homeDir':'/var/log', 'status':'Exists'}]
+    mydict['alternatives'] = [
       {'name':'/etc/alternatives/hadoop-conf', 'target':'/etc/hadoop/conf.dist'},
       {'name':'/etc/alternatives/hbase-conf', 'target':'/etc/hbase/conf.1'}
     ]
-    dict['stackFoldersAndFiles'] = [{'name':'/a/b', 'type':'directory'},{'name':'/a/b.txt',
'type':'file'}]
-    dict['hostHealth']['activeJavaProcs'] = [
+    mydict['stackFoldersAndFiles'] = [{'name':'/a/b', 'type':'directory'},{'name':'/a/b.txt',
'type':'file'}]
+    mydict['hostHealth']['activeJavaProcs'] = [
       {'pid':355,'hadoop':True,'command':'some command','user':'root'},
       {'pid':455,'hadoop':True,'command':'some command','user':'hdfs'}
     ]
-    handler.writeHostCheckFile(dict)
+    handler.writeHostCheckFile(mydict)
 
     configValidator = ConfigParser.RawConfigParser()
     configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
@@ -130,13 +131,13 @@ class TestHostCheckReportFileHandler(TestCase):
     self.chkItemsEqual(procs, ['455', '355'])
 
 
-    dict['installed_packages'] = [
+    mydict['installed_packages'] = [
       {'name':'hadoop','version':'3.2.3','repoName':'HDP'},
       {'name':'hadoop-lib','version':'3.2.3','repoName':'HDP'}
     ]
-    dict['existing_repos'] = ['HDP', 'HDP-epel']
+    mydict['existing_repos'] = ['HDP', 'HDP-epel']
     
-    handler.writeHostChecksCustomActionsFile(dict)
+    handler.writeHostChecksCustomActionsFile(mydict)
     configValidator = ConfigParser.RawConfigParser()
     configPath_ca = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_CUSTOM_ACTIONS_FILE)
     configValidator.read(configPath_ca)
@@ -150,13 +151,40 @@ class TestHostCheckReportFileHandler(TestCase):
     time = configValidator.get('metadata', 'created')
     self.assertTrue(time != None)
 
+  @patch("os.path.exists")
+  @patch("os.listdir")
+  def test_write_host_stack_list(self, list_mock, exists_mock):
+    exists_mock.return_value = True
+    list_mock.return_value = ["1.1.1.1-1234", "current", "test"]
+
+    tmpfile = tempfile.mktemp()
+
+    config = ConfigParser.RawConfigParser()
+    config.add_section('agent')
+    config.set('agent', 'prefix', os.path.dirname(tmpfile))
+
+    handler = HostCheckReportFileHandler(config)
+
+    mydict = {}
+    mydict['hostHealth'] = {}
+    mydict['stackFoldersAndFiles'] = [{'name':'/a/b', 'type':'directory'},{'name':'/a/b.txt',
'type':'file'}]
+
+    handler.writeHostCheckFile(mydict)
+
+    configValidator = ConfigParser.RawConfigParser()
+    configPath = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_FILE)
+    configValidator.read(configPath)
+
+    paths = configValidator.get('directories', 'dir_list')
+    self.chkItemsEqual(paths, ['/a/b', '/a/b.txt', '/usr/hdp/1.1.1.1-1234', '/usr/hdp/current'])
+
   def chkItemsEqual(self, commaDelimited, items):
     items1 = commaDelimited.split(',')
     items1.sort()
     items.sort()
     items1Str = ','.join(items1)
     items2Str = ','.join(items)
-    self.assertEquals(items1Str, items2Str)
+    self.assertEquals(items2Str, items1Str)
 
 if __name__ == "__main__":
   unittest.main(verbosity=2)


Mime
View raw message