ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vbrodets...@apache.org
Subject git commit: AMBARI-7762. Confirm Hosts page is lagging.(vbrodetskyi)
Date Mon, 13 Oct 2014 18:08:58 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk f489f0b2f -> 62542465b


AMBARI-7762. Confirm Hosts page is lagging.(vbrodetskyi)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/62542465
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/62542465
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/62542465

Branch: refs/heads/trunk
Commit: 62542465b1ee3c3e2051984695826abf841dcc41
Parents: f489f0b
Author: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Authored: Mon Oct 13 19:53:58 2014 +0300
Committer: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Committed: Mon Oct 13 19:53:58 2014 +0300

----------------------------------------------------------------------
 .../src/main/python/ambari_agent/HostCleanup.py | 51 +++++++++++++++++---
 .../test/python/ambari_agent/TestHostCleanup.py | 35 +++++++++++---
 2 files changed, 73 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/62542465/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
index 0840378..570c320 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
@@ -27,16 +27,16 @@ import subprocess
 import logging
 import shutil
 import platform
+import fnmatch
 import ConfigParser
 import optparse
 import shlex
 import datetime
-import AmbariConfig
+from AmbariConfig import AmbariConfig
 from pwd import getpwnam
 from ambari_commons import OSCheck
 
 logger = logging.getLogger()
-configFile = "/etc/ambari-agent/conf/ambari-agent.ini"
 
 PACKAGE_ERASE_CMD = {
   "redhat": "yum erase -y {0}",
@@ -67,6 +67,9 @@ REPOS_KEY = "repo_list"
 DIR_SECTION = "directories"
 ADDITIONAL_DIRS = "additional_directories"
 DIR_KEY = "dir_list"
+CACHE_FILES_PATTERN = {
+  'alerts': ['*.json']
+}
 PROCESS_SECTION = "processes"
 PROCESS_KEY = "proc_list"
 ALT_SECTION = "alternatives"
@@ -86,9 +89,9 @@ PACKAGES_BLACK_LIST = ["ambari-server", "ambari-agent"]
 class HostCleanup:
   def resolve_ambari_config(self):
     try:
-      config = AmbariConfig.AmbariConfig()
-      if os.path.exists(configFile):
-        config.read(configFile)
+      config = AmbariConfig()
+      if os.path.exists(AmbariConfig.CONFIG_FILE):
+        config.read(AmbariConfig.CONFIG_FILE)
       else:
         raise Exception("No config found, use default")
 
@@ -229,6 +232,33 @@ class HostCleanup:
 
     return out
 
+
+  def do_clear_cache(self, cache_root, dir_map=None):
+    """
+     Clear cache dir according to provided root directory
+
+     cache_root - root dir for cache directory
+     dir_map - should be used only for recursive calls
+    """
+    global CACHE_FILES_PATTERN
+    file_map = CACHE_FILES_PATTERN if dir_map is None else dir_map
+    remList = []
+
+    # Build remove list according to masks
+    for folder in file_map:
+      if isinstance(file_map[folder], list):  # here is list of file masks/files
+        for mask in file_map[folder]:
+          remList += self.get_files_in_dir("%s/%s" % (cache_root, folder), mask)
+      elif isinstance(file_map[folder], dict):  # here described sub-folder
+        remList += self.do_clear_cache("%s/%s" % (cache_root, folder), file_map[folder])
+
+    if dir_map is not None:  # push result list back as this is call from stack
+      return remList
+    else:  # root call, so we have final list
+      self.do_erase_files_silent(remList)
+
+
+
   # Alternatives exist as a stack of symlinks under /var/lib/alternatives/$name
   # Script expects names of the alternatives as input
   # We find all the symlinks using command, #] alternatives --display $name
@@ -276,7 +306,7 @@ class HostCleanup:
             logger.error("Unable to kill process with pid: " + pid + ", " + stderrdata)
     return 0
 
-  def get_files_in_dir(self, dirPath):
+  def get_files_in_dir(self, dirPath, filemask = None):
     fileList = []
     if dirPath:
       if os.path.exists(dirPath):
@@ -285,7 +315,11 @@ class HostCleanup:
           for link in listdir:
             path = dirPath + os.sep + link
             if not os.path.islink(path) and not os.path.isdir(path):
-              fileList.append(path)
+              if filemask is not None:
+                if fnmatch.fnmatch(path, filemask):
+                  fileList.append(path)
+              else:
+                fileList.append(path)
 
     return fileList
 
@@ -532,6 +566,9 @@ def main():
   if propMap:
     h.do_cleanup(propMap)
 
+  if os.path.exists(config.get('agent', 'cache_dir')):
+    h.do_clear_cache(config.get('agent', 'cache_dir'))
+
   logger.info('Clean-up completed. The output is at %s' % (str(options.outputfile)))
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/62542465/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
index cdd5c36..c958412 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
@@ -117,6 +117,7 @@ class TestHostCleanup(TestCase):
       self.silent = silent
       self.java_home = java_home
 
+  @patch.object(HostCleanup.HostCleanup, 'do_clear_cache')
   @patch.object(HostCleanup, 'get_YN_input')
   @patch.object(HostCleanup.HostCleanup, 'do_cleanup')
   @patch.object(HostCleanup.HostCleanup, 'is_current_user_root')
@@ -126,7 +127,7 @@ class TestHostCleanup(TestCase):
   @patch.object(logging, 'FileHandler')
   @patch.object(optparse.OptionParser, 'parse_args')
   def test_options(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock,
-                   set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock):
+                   set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock,
clear_cache_mock):
     parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile',
'', False,
                                                                    False, 'java_home'), [])
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
@@ -144,7 +145,27 @@ class TestHostCleanup(TestCase):
     read_host_check_file_mock.assert_called_with('/someinputfile')
     self.assertTrue(get_yn_input_mock.called)
 
+  @patch.object(HostCleanup.HostCleanup, 'get_files_in_dir')
+  @patch.object(HostCleanup.HostCleanup, 'do_erase_files_silent')
+  def test_clear_cache(self, erase_files_mock, get_files_mock):
+    old_data = HostCleanup.CACHE_FILES_PATTERN
+
+    HostCleanup.CACHE_FILES_PATTERN = {
+      'somedir': ['*.txt']
+    }
+
+    files_list = ['/tmp/somedir/test.txt']
+    get_files_mock.return_value = files_list
+
+    self.hostcleanup.do_clear_cache('/tmp')
 
+
+    get_files_mock.assert_called_with('/tmp/somedir', '*.txt')
+    erase_files_mock.assert_called_with(files_list)
+
+    HostCleanup.CACHE_FILES_PATTERN = old_data
+
+  @patch.object(HostCleanup.HostCleanup, 'do_clear_cache')
   @patch.object(HostCleanup, 'get_YN_input')
   @patch.object(HostCleanup.HostCleanup, 'do_cleanup')
   @patch.object(HostCleanup.HostCleanup, 'is_current_user_root')
@@ -154,7 +175,7 @@ class TestHostCleanup(TestCase):
   @patch.object(logging, 'FileHandler')
   @patch.object(optparse.OptionParser, 'parse_args')
   def test_options_silent(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock,
-                   set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock):
+                   set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock,
clear_cache_mock):
     parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile',
'', False,
                                                                    True, 'java_home'), [])
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
@@ -172,6 +193,7 @@ class TestHostCleanup(TestCase):
     read_host_check_file_mock.assert_called_with('/someinputfile')
     self.assertFalse(get_yn_input_mock.called)
 
+  @patch.object(HostCleanup.HostCleanup, 'do_clear_cache')
   @patch.object(HostCleanup.HostCleanup, 'get_additional_dirs')
   @patch.object(HostCleanup.HostCleanup, 'do_erase_alternatives')
   @patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos')
@@ -185,7 +207,7 @@ class TestHostCleanup(TestCase):
                       do_erase_dir_silent_method,
                       do_erase_files_silent_method, do_kill_processes_method,
                       get_os_type_method, find_repo_files_for_repos_method,
-                      do_erase_alternatives_method, get_additional_dirs_method):
+                      do_erase_alternatives_method, get_additional_dirs_method, clear_cache_mock):
     out = StringIO.StringIO()
     sys.stdout = out
     get_additional_dirs_method.return_value = ['/tmp/hadoop-nagios','/tmp/hsperfdata_007']
@@ -216,7 +238,7 @@ class TestHostCleanup(TestCase):
 
     sys.stdout = sys.__stdout__
 
-
+  @patch.object(HostCleanup.HostCleanup, 'do_clear_cache')
   @patch.object(HostCleanup.HostCleanup, 'do_delete_by_owner')
   @patch.object(HostCleanup.HostCleanup, 'get_user_ids')
   @patch.object(HostCleanup.HostCleanup, 'do_erase_alternatives')
@@ -232,7 +254,7 @@ class TestHostCleanup(TestCase):
                       do_erase_files_silent_method, do_kill_processes_method,
                       get_os_type_method, find_repo_files_for_repos_method,
                       do_erase_alternatives_method, get_user_ids_method,
-                      do_delete_by_owner_method):
+                      do_delete_by_owner_method, clear_cache_mock):
 
     global SKIP_LIST
     oldSkipList = HostCleanup.SKIP_LIST
@@ -260,6 +282,7 @@ class TestHostCleanup(TestCase):
     HostCleanup.SKIP_LIST = oldSkipList
     sys.stdout = sys.__stdout__
 
+  @patch.object(HostCleanup.HostCleanup, 'do_clear_cache')
   @patch.object(HostCleanup.HostCleanup, 'find_repo_files_for_repos')
   @patch.object(OSCheck, "get_os_type")
   @patch.object(HostCleanup.HostCleanup, 'do_kill_processes')
@@ -271,7 +294,7 @@ class TestHostCleanup(TestCase):
                       do_delete_users_method,
                       do_erase_dir_silent_method,
                       do_erase_files_silent_method, do_kill_processes_method,
-                      get_os_type_method, find_repo_files_for_repos_method):
+                      get_os_type_method, find_repo_files_for_repos_method, clear_cache_mock):
 
     out = StringIO.StringIO()
     sys.stdout = out


Mime
View raw message