Repository: ambari
Updated Branches:
refs/heads/trunk 58fb4135e -> 0c57d030a
AMBARI-9795. [Monarch] Cluster create failed with timeout on the client side (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0c57d030
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0c57d030
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0c57d030
Branch: refs/heads/trunk
Commit: 0c57d030a8079c8a0b1f0623d42f5e65f2aec931
Parents: 58fb413
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Wed Feb 25 18:36:48 2015 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Wed Feb 25 18:36:48 2015 +0200
----------------------------------------------------------------------
.../ambari_agent/HostCheckReportFileHandler.py | 86 ++++++++++++++------
.../src/main/python/ambari_agent/HostCleanup.py | 19 ++++-
.../src/main/python/ambari_agent/HostInfo.py | 62 --------------
.../TestHostCheckReportFileHandler.py | 23 +++---
.../test/python/ambari_agent/TestHostCleanup.py | 16 ++--
.../test/python/ambari_agent/TestHostInfo.py | 30 -------
.../libraries/functions/packages_analyzer.py | 12 +++
.../custom_actions/scripts/check_host.py | 73 ++++++++++++++++-
.../test/python/custom_actions/TestCheckHost.py | 7 +-
9 files changed, 185 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0c57d030/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
index bf24730..9c0f1d0 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCheckReportFileHandler.py
@@ -22,6 +22,7 @@ import datetime
import os.path
import logging
import traceback
+from AmbariConfig import AmbariConfig
import ConfigParser;
logger = logging.getLogger()
@@ -29,12 +30,58 @@ logger = logging.getLogger()
class HostCheckReportFileHandler:
HOST_CHECK_FILE = "hostcheck.result"
+ HOST_CHECK_CUSTOM_ACTIONS_FILE = "hostcheck_custom_actions.result"
- def __init__(self, config):
+ def __init__(self, config=None):
self.hostCheckFilePath = None
- if config is not None:
- hostCheckFileDir = config.get('agent', 'prefix')
- self.hostCheckFilePath = os.path.join(hostCheckFileDir, self.HOST_CHECK_FILE)
+
+ if config is None:
+ config = self.resolve_ambari_config()
+
+ hostCheckFileDir = config.get('agent', 'prefix')
+ self.hostCheckFilePath = os.path.join(hostCheckFileDir, self.HOST_CHECK_FILE)
+ self.hostCheckCustomActionsFilePath = os.path.join(hostCheckFileDir, self.HOST_CHECK_CUSTOM_ACTIONS_FILE)
+
+ def resolve_ambari_config(self):
+ try:
+ config = AmbariConfig()
+ if os.path.exists(AmbariConfig.getConfigFile()):
+ config.read(AmbariConfig.getConfigFile())
+ else:
+ raise Exception("No config found, use default")
+
+ except Exception, err:
+ logger.warn(err)
+ return config
+
+ def writeHostChecksCustomActionsFile(self, structuredOutput):
+ if self.hostCheckCustomActionsFilePath is None:
+ return
+
+ try:
+ logger.info("Host check custom action report at " + self.hostCheckCustomActionsFilePath)
+ config = ConfigParser.RawConfigParser()
+ config.add_section('metadata')
+ config.set('metadata', 'created', str(datetime.datetime.now()))
+
+ if 'installed_packages' in structuredOutput.keys():
+ items = []
+ for itemDetail in structuredOutput['installed_packages']:
+ items.append(itemDetail['name'])
+ config.add_section('packages')
+ config.set('packages', 'pkg_list', ','.join(map(str, items)))
+
+ if 'existing_repos' in structuredOutput.keys():
+ config.add_section('repositories')
+ config.set('repositories', 'repo_list', ','.join(structuredOutput['existing_repos']))
+
+ self.removeFile(self.hostCheckCustomActionsFilePath)
+ self.touchFile(self.hostCheckCustomActionsFilePath)
+ with open(self.hostCheckCustomActionsFilePath, 'wb') as configfile:
+ config.write(configfile)
+ except Exception, err:
+ logger.error("Can't write host check file at %s :%s " % (self.hostCheckFilePath, err.message))
+ traceback.print_exc()
def writeHostCheckFile(self, hostInfo):
if self.hostCheckFilePath is None:
@@ -81,33 +128,22 @@ class HostCheckReportFileHandler:
config.add_section('processes')
config.set('processes', 'proc_list', ','.join(map(str, items)))
- if 'installedPackages' in hostInfo.keys():
- items = []
- for itemDetail in hostInfo['installedPackages']:
- items.append(itemDetail['name'])
- config.add_section('packages')
- config.set('packages', 'pkg_list', ','.join(map(str, items)))
-
- if 'existingRepos' in hostInfo.keys():
- config.add_section('repositories')
- config.set('repositories', 'repo_list', ','.join(hostInfo['existingRepos']))
-
- self.removeFile()
- self.touchFile()
+ self.removeFile(self.hostCheckFilePath)
+ self.touchFile(self.hostCheckFilePath)
with open(self.hostCheckFilePath, 'wb') as configfile:
config.write(configfile)
except Exception, err:
logger.error("Can't write host check file at %s :%s " % (self.hostCheckFilePath, err.message))
traceback.print_exc()
- def removeFile(self):
- if os.path.isfile(self.hostCheckFilePath):
- logger.info("Removing old host check file at %s" % self.hostCheckFilePath)
- os.remove(self.hostCheckFilePath)
+ def removeFile(self, path):
+ if os.path.isfile(path):
+ logger.info("Removing old host check file at %s" % path)
+ os.remove(path)
- def touchFile(self):
- if not os.path.isfile(self.hostCheckFilePath):
- logger.info("Creating host check file at %s" % self.hostCheckFilePath)
- open(self.hostCheckFilePath, 'w').close()
+ def touchFile(self, path):
+ if not os.path.isfile(path):
+ logger.info("Creating host check file at %s" % path)
+ open(path, 'w').close()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0c57d030/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
index 398502e..e4a4754 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
@@ -55,7 +55,9 @@ ALT_ERASE_CMD = "alternatives --remove {0} {1}"
REPO_PATH_RHEL = "/etc/yum.repos.d"
REPO_PATH_SUSE = "/etc/zypp/repos.d/"
SKIP_LIST = []
+TMP_HOST_CHECK_FILE_NAME = "tmp_hostcheck.result"
HOST_CHECK_FILE_NAME = "hostcheck.result"
+HOST_CHECK_CUSTOM_ACTIONS_FILE = "hostcheck_custom_actions.result"
OUTPUT_FILE_NAME = "hostcleanup.result"
PACKAGE_SECTION = "packages"
@@ -523,13 +525,15 @@ def main():
config = h.resolve_ambari_config()
hostCheckFileDir = config.get('agent', 'prefix')
hostCheckFilePath = os.path.join(hostCheckFileDir, HOST_CHECK_FILE_NAME)
+ hostCheckCustomActionsFilePath = os.path.join(hostCheckFileDir, HOST_CHECK_CUSTOM_ACTIONS_FILE)
+ hostCheckFilesPaths = hostCheckFilePath + "," + hostCheckCustomActionsFilePath
hostCheckResultPath = os.path.join(hostCheckFileDir, OUTPUT_FILE_NAME)
parser = optparse.OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", action="store_false",
default=False, help="output verbosity.")
- parser.add_option("-f", "--file", dest="inputfile",
- default=hostCheckFilePath,
+ parser.add_option("-f", "--file", dest="inputfiles",
+ default=hostCheckFilesPaths,
help="host check result file to read.", metavar="FILE")
parser.add_option("-o", "--out", dest="outputfile",
default=hostCheckResultPath,
@@ -575,8 +579,15 @@ def main():
print 'Exiting. Use option --skip="users" to skip deleting users'
sys.exit(1)
- hostcheckfile = options.inputfile
- propMap = h.read_host_check_file(hostcheckfile)
+ hostcheckfile, hostcheckfileca = options.inputfiles.split(",")
+
+ with open(TMP_HOST_CHECK_FILE_NAME, "wb") as tmp_f:
+ with open(hostcheckfile, "rb") as f1:
+ with open(hostcheckfileca, "rb") as f2:
+ tmp_f.write(f1.read())
+ tmp_f.write(f2.read())
+
+ propMap = h.read_host_check_file(TMP_HOST_CHECK_FILE_NAME)
if propMap:
h.do_cleanup(propMap)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0c57d030/ambari-agent/src/main/python/ambari_agent/HostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
index dc0834a..516221d 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
@@ -147,33 +147,6 @@ class HostInfoLinux(HostInfo):
"/hadoop", "/usr/hdp"
]
- # Packages that are used to find repos (then repos are used to find other packages)
- PACKAGES = [
- "hadoop_2_2_*", "hadoop-2-2-.*", "zookeeper_2_2_*", "zookeeper-2-2-.*",
- "hadoop", "zookeeper", "webhcat", "*-manager-server-db", "*-manager-daemons"
- ]
-
- # Additional packages to look for (search packages that start with these)
- ADDITIONAL_PACKAGES = [
- "rrdtool", "rrdtool-python", "ganglia", "gmond", "gweb", "libconfuse",
- "ambari-log4j", "hadoop", "zookeeper", "oozie", "webhcat"
- ]
-
- # ignore packages from repos whose names start with these strings
- IGNORE_PACKAGES_FROM_REPOS = [
- "ambari", "installed"
- ]
-
- # ignore required packages
- IGNORE_PACKAGES = [
- "epel-release"
- ]
-
- # ignore repos from the list of repos to be cleaned
- IGNORE_REPOS = [
- "ambari", "HDP-UTILS"
- ]
-
DEFAULT_SERVICE_NAME = "ntpd"
SERVICE_STATUS_CMD = "%s %s status" % (SERVICE_CMD, DEFAULT_SERVICE_NAME)
@@ -202,8 +175,6 @@ class HostInfoLinux(HostInfo):
result['name'] = fields[0]
result['homeDir'] = fields[5]
result['status'] = "Available"
- if not os.path.exists(homeDir):
- result['status'] = "Invalid home directory"
results.append(result)
def checkFolders(self, basePaths, projectNames, existingUsers, dirs):
@@ -248,18 +219,6 @@ class HostInfoLinux(HostInfo):
pass
pass
- def getReposToRemove(self, repos, ignoreList):
- reposToRemove = []
- for repo in repos:
- addToRemoveList = True
- for ignoreRepo in ignoreList:
- if packages_analyzer.nameMatch(ignoreRepo, repo):
- addToRemoveList = False
- continue
- if addToRemoveList:
- reposToRemove.append(repo)
- return reposToRemove
-
def getTransparentHugePage(self):
# This file exist only on redhat 6
thp_regex = "\[(.+)\]"
@@ -323,8 +282,6 @@ class HostInfoLinux(HostInfo):
# If commands are in progress or components are already mapped to this host
# Then do not perform certain expensive host checks
if componentsMapped or commandsInProgress:
- dict['existingRepos'] = [self.RESULT_UNAVAILABLE]
- dict['installedPackages'] = []
dict['alternatives'] = []
dict['stackFoldersAndFiles'] = []
dict['existingUsers'] = []
@@ -342,23 +299,6 @@ class HostInfoLinux(HostInfo):
self.checkFolders(self.DEFAULT_DIRS, self.DEFAULT_PROJECT_NAMES, existingUsers, dirs)
dict['stackFoldersAndFiles'] = dirs
- installedPackages = []
- availablePackages = []
- packages_analyzer.allInstalledPackages(installedPackages)
- packages_analyzer.allAvailablePackages(availablePackages)
-
- repos = []
- packages_analyzer.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages,
- self.IGNORE_PACKAGES_FROM_REPOS, repos)
- packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES,
installedPackages)
- additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames(
- self.ADDITIONAL_PACKAGES, installedPackages)
- allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
- dict['installedPackages'] = packages_analyzer.getPackageDetails(installedPackages,
allPackages)
-
- repos = self.getReposToRemove(repos, self.IGNORE_REPOS)
- dict['existingRepos'] = repos
-
self.reportFileHandler.writeHostCheckFile(dict)
pass
@@ -469,8 +409,6 @@ class HostInfoWindows(HostInfo):
# If commands are in progress or components are already mapped to this host
# Then do not perform certain expensive host checks
if componentsMapped or commandsInProgress:
- dict['existingRepos'] = [self.RESULT_UNAVAILABLE]
- dict['installedPackages'] = []
dict['alternatives'] = []
dict['stackFoldersAndFiles'] = []
dict['existingUsers'] = []
http://git-wip-us.apache.org/repos/asf/ambari/blob/0c57d030/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py b/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py
index eb7efb2..d56ad8f 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostCheckReportFileHandler.py
@@ -84,12 +84,6 @@ class TestHostCheckReportFileHandler(TestCase):
procs = configValidator.get('processes', 'proc_list')
self.assertEquals(procs, '')
- pkgs = configValidator.get('packages', 'pkg_list')
- self.assertEquals(pkgs, '')
-
- repos = configValidator.get('repositories', 'repo_list')
- self.assertEquals(repos, '')
-
time = configValidator.get('metadata', 'created')
self.assertTrue(time != None)
@@ -114,11 +108,6 @@ class TestHostCheckReportFileHandler(TestCase):
{'pid':355,'hadoop':True,'command':'some command','user':'root'},
{'pid':455,'hadoop':True,'command':'some command','user':'hdfs'}
]
- dict['installedPackages'] = [
- {'name':'hadoop','version':'3.2.3','repoName':'HDP'},
- {'name':'hadoop-lib','version':'3.2.3','repoName':'HDP'}
- ]
- dict['existingRepos'] = ['HDP', 'HDP-epel']
handler.writeHostCheckFile(dict)
configValidator = ConfigParser.RawConfigParser()
@@ -140,6 +129,18 @@ class TestHostCheckReportFileHandler(TestCase):
procs = configValidator.get('processes', 'proc_list')
self.chkItemsEqual(procs, ['455', '355'])
+
+ dict['installed_packages'] = [
+ {'name':'hadoop','version':'3.2.3','repoName':'HDP'},
+ {'name':'hadoop-lib','version':'3.2.3','repoName':'HDP'}
+ ]
+ dict['existing_repos'] = ['HDP', 'HDP-epel']
+
+ handler.writeHostChecksCustomActionsFile(dict)
+ configValidator = ConfigParser.RawConfigParser()
+ configPath_ca = os.path.join(os.path.dirname(tmpfile), HostCheckReportFileHandler.HOST_CHECK_CUSTOM_ACTIONS_FILE)
+ configValidator.read(configPath_ca)
+
pkgs = configValidator.get('packages', 'pkg_list')
self.chkItemsEqual(pkgs, ['hadoop', 'hadoop-lib'])
http://git-wip-us.apache.org/repos/asf/ambari/blob/0c57d030/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
index f43784c..734db5d 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostCleanup.py
@@ -112,9 +112,9 @@ class TestHostCleanup(TestCase):
sys.stdout = sys.__stdout__
class HostCleanupOptions:
- def __init__(self, outputfile, inputfile, skip, verbose, silent, java_home):
+ def __init__(self, outputfile, inputfiles, skip, verbose, silent, java_home):
self.outputfile = outputfile
- self.inputfile = inputfile
+ self.inputfiles = inputfiles
self.skip = skip
self.verbose = verbose
self.silent = silent
@@ -131,7 +131,9 @@ class TestHostCleanup(TestCase):
@patch.object(optparse.OptionParser, 'parse_args')
def test_options(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock,
set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock,
clear_cache_mock):
- parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile',
'', False,
+ open('/tmp/someinputfile1', 'a').close()
+ open('/tmp/someinputfile2', 'a').close()
+ parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/tmp/someinputfile1,/tmp/someinputfile2',
'', False,
False, 'java_home'), [])
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
user_root_mock.return_value = True
@@ -145,7 +147,7 @@ class TestHostCleanup(TestCase):
#test --verbose
logging_mock.assert_called_with(level=logging.INFO)
# test --in
- read_host_check_file_mock.assert_called_with('/someinputfile')
+ read_host_check_file_mock.assert_called_with('tmp_hostcheck.result')
self.assertTrue(get_yn_input_mock.called)
@patch.object(HostCleanup.HostCleanup, 'get_files_in_dir')
@@ -179,7 +181,9 @@ class TestHostCleanup(TestCase):
@patch.object(optparse.OptionParser, 'parse_args')
def test_options_silent(self, parser_mock, file_handler_mock, logging_mock, read_host_check_file_mock,
set_formatter_mock, user_root_mock, do_cleanup_mock, get_yn_input_mock,
clear_cache_mock):
- parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/someinputfile',
'', False,
+ open('/tmp/someinputfile1', 'a').close()
+ open('/tmp/someinputfile2', 'a').close()
+ parser_mock.return_value = (TestHostCleanup.HostCleanupOptions('/someoutputfile', '/tmp/someinputfile1,/tmp/someinputfile2',
'', False,
True, 'java_home'), [])
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
user_root_mock.return_value = True
@@ -193,7 +197,7 @@ class TestHostCleanup(TestCase):
#test --verbose
logging_mock.assert_called_with(level=logging.INFO)
# test --in
- read_host_check_file_mock.assert_called_with('/someinputfile')
+ read_host_check_file_mock.assert_called_with('tmp_hostcheck.result')
self.assertFalse(get_yn_input_mock.called)
@patch.object(HostCleanup.HostCleanup, 'do_clear_cache')
http://git-wip-us.apache.org/repos/asf/ambari/blob/0c57d030/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
index c1b93d6..fcab6b8 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
@@ -83,20 +83,6 @@ class TestHostInfo(TestCase):
self.assertTrue(installedPackages[6][1], "11-38.13.9")
@not_for_platform(PLATFORM_WINDOWS)
- def test_getReposToRemove(self):
- l1 = ["Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.15", "Ambari 1.x", "HDP"]
- l2 = ["Ambari", "HDP-UTIL"]
- hostInfo = HostInfoLinux()
- l3 = hostInfo.getReposToRemove(l1, l2)
- self.assertTrue(1, len(l3))
- self.assertEqual(l3[0], "HDP")
-
- l1 = ["AMBARI.dev-1.x", "HDP-1.3.0"]
- l3 = hostInfo.getReposToRemove(l1, l2)
- self.assertTrue(1, len(l3))
- self.assertEqual(l3[0], "HDP-1.3.0")
-
- @not_for_platform(PLATFORM_WINDOWS)
def test_perform_package_analysis(self):
installedPackages = [
["hadoop-a", "2.3", "HDP"], ["zk", "3.1", "HDP"], ["webhcat", "3.1", "HDP"],
@@ -286,14 +272,9 @@ class TestHostInfo(TestCase):
dict = {}
hostInfo.register(dict, False, False)
self.assertTrue(cit_mock.called)
- self.assertTrue(gir_mock.called)
- self.assertTrue(gpd_mock.called)
- self.assertTrue(aip_mock.called)
- self.assertTrue(aap_mock.called)
self.assertTrue(os_umask_mock.called)
self.assertTrue(whcf_mock.called)
- self.assertTrue(0 < len(dict['installedPackages']))
self.assertTrue('agentTimeStampAtReporting' in dict['hostHealth'])
@not_for_platform(PLATFORM_WINDOWS)
@@ -342,25 +323,14 @@ class TestHostInfo(TestCase):
hostInfo = HostInfoLinux()
dict = {}
hostInfo.register(dict, False, False)
- self.assertTrue(gir_mock.called)
- self.assertTrue(gpd_mock.called)
- self.assertTrue(aip_mock.called)
self.assertTrue(cit_mock.called)
self.assertEqual(1, cit_mock.call_count)
- for existingPkg in ["pkg1", "pkg2"]:
- self.assertTrue(existingPkg in dict['installedPackages'])
- args, kwargs = gpd_mock.call_args_list[0]
- for existingPkg in ["pkg1", "pkg2"]:
- self.assertTrue(existingPkg in args[1])
-
def verifyReturnedValues(self, dict):
hostInfo = HostInfoLinux()
self.assertEqual(dict['alternatives'], [])
self.assertEqual(dict['stackFoldersAndFiles'], [])
self.assertEqual(dict['existingUsers'], [])
- self.assertEqual(dict['existingRepos'][0], hostInfo.RESULT_UNAVAILABLE)
- self.assertEqual(dict['installedPackages'], [])
self.assertTrue(dict['iptablesIsRunning'])
@patch("os.path.exists")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0c57d030/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
index 888387c..55c1eb8 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/packages_analyzer.py
@@ -261,3 +261,15 @@ def getPackageDetails(installedPackages, foundPackages):
pkgDetail['repoName'] = installedPackage[2]
packageDetails.append(pkgDetail)
return packageDetails
+
+def getReposToRemove(repos, ignoreList):
+ reposToRemove = []
+ for repo in repos:
+ addToRemoveList = True
+ for ignoreRepo in ignoreList:
+ if nameMatch(ignoreRepo, repo):
+ addToRemoveList = False
+ continue
+ if addToRemoveList:
+ reposToRemove.append(repo)
+ return reposToRemove
http://git-wip-us.apache.org/repos/asf/ambari/blob/0c57d030/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
index ff2eabd..b0fcfb6 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
@@ -24,16 +24,20 @@ import os
import subprocess
import socket
+from resource_management.libraries.functions import packages_analyzer
from ambari_commons import os_utils
from ambari_commons.os_check import OSCheck, OSConst
from ambari_commons.inet_utils import download_file
from resource_management import Script, Execute, format
from ambari_agent.HostInfo import HostInfo
+from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler
CHECK_JAVA_HOME = "java_home_check"
CHECK_DB_CONNECTION = "db_connection_check"
CHECK_HOST_RESOLUTION = "host_resolution_check"
CHECK_LAST_AGENT_ENV = "last_agent_env_check"
+CHECK_INSTALLED_PACKAGES = "installed_packages"
+CHECK_EXISTING_REPOS = "existing_repos"
DB_MYSQL = "mysql"
DB_ORACLE = "oracle"
@@ -52,9 +56,42 @@ JDBC_DRIVER_SYMLINK_MSSQL = "sqljdbc4.jar"
JDBC_AUTH_SYMLINK_MSSQL = "sqljdbc_auth.dll"
class CheckHost(Script):
+ # Packages that are used to find repos (then repos are used to find other packages)
+ PACKAGES = [
+ "hadoop_2_2_*", "hadoop-2-2-.*", "zookeeper_2_2_*", "zookeeper-2-2-.*",
+ "hadoop", "zookeeper", "webhcat", "*-manager-server-db", "*-manager-daemons"
+ ]
+
+
+ # ignore packages from repos whose names start with these strings
+ IGNORE_PACKAGES_FROM_REPOS = [
+ "ambari", "installed"
+ ]
+
+
+ # ignore required packages
+ IGNORE_PACKAGES = [
+ "epel-release"
+ ]
+
+ # Additional packages to look for (search packages that start with these)
+ ADDITIONAL_PACKAGES = [
+ "rrdtool", "rrdtool-python", "ganglia", "gmond", "gweb", "libconfuse",
+ "ambari-log4j", "hadoop", "zookeeper", "oozie", "webhcat"
+ ]
+
+ # ignore repos from the list of repos to be cleaned
+ IGNORE_REPOS = [
+ "ambari", "HDP-UTILS"
+ ]
+
+ def __init__(self):
+ self.reportFileHandler = HostCheckReportFileHandler()
+
def actionexecute(self, env):
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
+ report_file_handler_dict = {}
#print "CONFIG: " + str(config)
@@ -93,9 +130,43 @@ class CheckHost(Script):
except Exception, exception :
print "There was an unknown error while checking last host environment details: "
+ str(exception)
structured_output[CHECK_LAST_AGENT_ENV] = {"exit_code" : 1, "message": str(exception)}
-
+
+ # CHECK_INSTALLED_PACKAGES and CHECK_EXISTING_REPOS required to run together for
+ # reasons of not doing the same common work twice for them as it takes some time, especially
on Ubuntu.
+ if CHECK_INSTALLED_PACKAGES in check_execute_list and CHECK_EXISTING_REPOS in check_execute_list:
+ try :
+ installed_packages, repos = self.execute_existing_repos_and_installed_packages_check(config)
+ structured_output[CHECK_INSTALLED_PACKAGES] = installed_packages
+ structured_output[CHECK_EXISTING_REPOS] = repos
+ except Exception, exception :
+ print "There was an unknown error while checking installed packages and existing
repositories: " + str(exception)
+ structured_output[CHECK_INSTALLED_PACKAGES] = {"exit_code" : 1, "message": str(exception)}
+ structured_output[CHECK_EXISTING_REPOS] = {"exit_code" : 1, "message": str(exception)}
+
+ # this is necessary for HostCleanup to know later what were the results.
+ self.reportFileHandler.writeHostChecksCustomActionsFile(structured_output)
+
self.put_structured_out(structured_output)
+ def execute_existing_repos_and_installed_packages_check(self, config):
+ installedPackages = []
+ availablePackages = []
+ packages_analyzer.allInstalledPackages(installedPackages)
+ packages_analyzer.allAvailablePackages(availablePackages)
+
+ repos = []
+ packages_analyzer.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages,
+ self.IGNORE_PACKAGES_FROM_REPOS, repos)
+ packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES,
installedPackages)
+ additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames(
+ self.ADDITIONAL_PACKAGES, installedPackages)
+ allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
+
+ installedPackages = packages_analyzer.getPackageDetails(installedPackages, allPackages)
+ repos = packages_analyzer.getReposToRemove(repos, self.IGNORE_REPOS)
+
+ return installedPackages, repos
+
def execute_java_home_available_check(self, config):
print "Java home check started."
http://git-wip-us.apache.org/repos/asf/ambari/blob/0c57d030/ambari-server/src/test/python/custom_actions/TestCheckHost.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/TestCheckHost.py b/ambari-server/src/test/python/custom_actions/TestCheckHost.py
index 105a0b0..2cd5c7a 100644
--- a/ambari-server/src/test/python/custom_actions/TestCheckHost.py
+++ b/ambari-server/src/test/python/custom_actions/TestCheckHost.py
@@ -55,7 +55,8 @@ class TestCheckHost(TestCase):
checkHost = CheckHost()
checkHost.actionexecute(None)
- self.assertEquals(os_isfile_mock.call_args[0][0], 'test_java_home/bin/java')
+ print os_isfile_mock.call_args
+ self.assertEquals(os_isfile_mock.call_args[0][0], '/tmp/ambari-agent/hostcheck_custom_actions.result')
self.assertEquals(structured_out_mock.call_args[0][0], {'java_home_check': {'message':
'Java home exists!',
'exit_code':
0}})
# test, java home doesn't exist
@@ -64,7 +65,7 @@ class TestCheckHost(TestCase):
checkHost.actionexecute(None)
- self.assertEquals(os_isfile_mock.call_args[0][0], 'test_java_home/bin/java')
+ self.assertEquals(os_isfile_mock.call_args[0][0], '/tmp/ambari-agent/hostcheck_custom_actions.result')
self.assertEquals(structured_out_mock.call_args[0][0], {'java_home_check': {"message":
"Java home doesn't exist!",
"exit_code"
: 1}})
@@ -282,8 +283,6 @@ class TestCheckHost(TestCase):
self.assertTrue('alternatives' in last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('umask' in last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('stackFoldersAndFiles' in last_agent_env_check_result['last_agent_env_check'])
- self.assertTrue('existingRepos' in last_agent_env_check_result['last_agent_env_check'])
- self.assertTrue('installedPackages' in last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('existingUsers' in last_agent_env_check_result['last_agent_env_check'])
# try it now with errors
|