ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rle...@apache.org
Subject ambari git commit: AMBARI-14690. Configurable system resources for ambari agents (Oliver Szabo via rlevas)
Date Wed, 02 Mar 2016 17:28:46 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.2 938adb7c2 -> 8ccbf8fb1


AMBARI-14690. Configurable system resources for ambari agents (Oliver Szabo via rlevas)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8ccbf8fb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8ccbf8fb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8ccbf8fb

Branch: refs/heads/branch-2.2
Commit: 8ccbf8fb1d069b193b27401260b6dc4e26936793
Parents: 938adb7
Author: Oliver Szabo <oszabo@hortonworks.com>
Authored: Wed Mar 2 12:28:38 2016 -0500
Committer: Robert Levas <rlevas@hortonworks.com>
Committed: Wed Mar 2 12:28:38 2016 -0500

----------------------------------------------------------------------
 ambari-agent/conf/unix/ambari-agent.ini         |  1 +
 ambari-agent/conf/windows/ambari-agent.ini      |  1 +
 .../main/python/ambari_agent/AmbariConfig.py    |  1 +
 .../src/main/python/ambari_agent/Facter.py      | 60 ++++++++++++++++++--
 .../test/python/ambari_agent/TestHardware.py    | 51 +++++++++++++++++
 5 files changed, 110 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8ccbf8fb/ambari-agent/conf/unix/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini
index 80afdb2..b46bfe5 100644
--- a/ambari-agent/conf/unix/ambari-agent.ini
+++ b/ambari-agent/conf/unix/ambari-agent.ini
@@ -30,6 +30,7 @@ tolerate_download_failures=true
 run_as_user=root
 parallel_execution=0
 alert_grace_period=5
+system_resource_overrides=/etc/resource_overrides
 
 [security]
 keysdir=/var/lib/ambari-agent/keys

http://git-wip-us.apache.org/repos/asf/ambari/blob/8ccbf8fb/ambari-agent/conf/windows/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/windows/ambari-agent.ini b/ambari-agent/conf/windows/ambari-agent.ini
index 1b24f25..e490f7c 100644
--- a/ambari-agent/conf/windows/ambari-agent.ini
+++ b/ambari-agent/conf/windows/ambari-agent.ini
@@ -30,6 +30,7 @@ cache_dir=cache
 tolerate_download_failures=true
 parallel_execution=0
 alert_grace_period=5
+system_resource_overrides=\\etc\\resource_overrides
 
 [security]
 keysdir=keys

http://git-wip-us.apache.org/repos/asf/ambari/blob/8ccbf8fb/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
index efb95f9..55b2f8f 100644
--- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
+++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
@@ -46,6 +46,7 @@ data_cleanup_max_size_MB = 100
 ping_port=8670
 cache_dir={ps}var{ps}lib{ps}ambari-agent{ps}cache
 parallel_execution=0
+system_resource_overrides={ps}etc{ps}resource_overrides
 
 [services]
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8ccbf8fb/ambari-agent/src/main/python/ambari_agent/Facter.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/Facter.py b/ambari-agent/src/main/python/ambari_agent/Facter.py
index 301fdc9..9879080 100644
--- a/ambari-agent/src/main/python/ambari_agent/Facter.py
+++ b/ambari-agent/src/main/python/ambari_agent/Facter.py
@@ -31,6 +31,8 @@ import subprocess
 from ambari_commons.shell import shellRunner
 import time
 import uuid
+import json
+import glob
 from AmbariConfig import AmbariConfig
 from ambari_commons import OSCheck, OSConst
 from ambari_commons.os_family_impl import OsFamilyImpl
@@ -150,6 +152,47 @@ class Facter(object):
   def getUptimeDays(self):
     return self.getUptimeSeconds() / (60 * 60 * 24)
 
+  def getSystemResourceIfExists(self, systemResources, key, default):
+    if key in systemResources:
+      return systemResources[key]
+    else:
+      return default
+
+  def replaceFacterInfoWithSystemResources(self, systemResources, facterInfo):
+    """
+    Replace facter info with fake system resource data (if there are any).
+    """
+    for key in facterInfo:
+      facterInfo[key] = self.getSystemResourceIfExists(systemResources, key, facterInfo[key])
+    return facterInfo
+
+  def getSystemResourceOverrides(self):
+    """
+    Read all json files from 'system_resource_overrides' directory, and later these values
are used as
+    fake system data for hosts. In case of the key-value pairs cannot be loaded use default
behaviour.
+    """
+    systemResources = {}
+    if self.config.has_option('agent', 'system_resource_overrides'):
+      systemResourceDir = self.config.get('agent', 'system_resource_overrides', '').strip()
+      if systemResourceDir:
+        if os.path.isdir(systemResourceDir) and os.path.exists(systemResourceDir):
+          try:
+            for filename in glob.glob('%s/*.json' % systemResourceDir):
+              with open(filename) as fp:
+                data = json.loads(fp.read())
+                for (key, value) in data.items():
+                  systemResources[key] = data[key]
+          except:
+            log.warn(
+              "Cannot read values from json files in %s. it won't be used for gathering system
resources." % systemResourceDir)
+        else:
+          log.info(
+            "Directory: '%s' does not exist - it won't be used for gathering system resources."
% systemResourceDir)
+      else:
+        log.info("'system_resource_dir' is not set - it won't be used for gathering system
resources.")
+    return systemResources
+
+
   def facterInfo(self):
     facterInfo = {}
     facterInfo['id'] = self.getId()
@@ -290,8 +333,12 @@ class FacterWindows(Facter):
 
   def facterInfo(self):
     facterInfo = super(FacterWindows, self).facterInfo()
-    facterInfo['swapsize'] = Facter.convertSizeMbToGb(self.getSwapSize())
-    facterInfo['swapfree'] = Facter.convertSizeMbToGb(self.getSwapFree())
+    systemResourceOverrides = self.getSystemResourceOverrides()
+    facterInfo = self.replaceFacterInfoWithSystemResources(systemResourceOverrides, facterInfo)
+    facterInfo['swapsize'] = Facter.convertSizeMbToGb(
+      self.getSystemResourceIfExists(systemResourceOverrides, 'swapsize', self.getSwapSize()))
+    facterInfo['swapfree'] = Facter.convertSizeMbToGb(
+      self.getSystemResourceIfExists(systemResourceOverrides, 'swapfree', self.getSwapFree()))
     return facterInfo
 
 
@@ -473,9 +520,14 @@ class FacterLinux(Facter):
 
   def facterInfo(self):
     facterInfo = super(FacterLinux, self).facterInfo()
+    systemResourceOverrides = self.getSystemResourceOverrides()
+    facterInfo = self.replaceFacterInfoWithSystemResources(systemResourceOverrides, facterInfo)
+
     facterInfo['selinux'] = self.isSeLinux()
-    facterInfo['swapsize'] = Facter.convertSizeKbToGb(self.getSwapSize())
-    facterInfo['swapfree'] = Facter.convertSizeKbToGb(self.getSwapFree())
+    facterInfo['swapsize'] = Facter.convertSizeKbToGb(
+      self.getSystemResourceIfExists(systemResourceOverrides, 'swapsize', self.getSwapSize()))
+    facterInfo['swapfree'] = Facter.convertSizeKbToGb(
+      self.getSystemResourceIfExists(systemResourceOverrides, 'swapfree', self.getSwapFree()))
     return facterInfo
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8ccbf8fb/ambari-agent/src/test/python/ambari_agent/TestHardware.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHardware.py b/ambari-agent/src/test/python/ambari_agent/TestHardware.py
index e096b95..876fff3 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHardware.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHardware.py
@@ -29,6 +29,7 @@ from ambari_agent.Hardware import Hardware
 from ambari_agent.AmbariConfig import AmbariConfig
 from ambari_agent.Facter import Facter, FacterLinux
 from ambari_commons import OSCheck
+from glob import glob
 
 @not_for_platform(PLATFORM_WINDOWS)
 @patch.object(platform,"linux_distribution", new = ('Suse','11','Final'))
@@ -329,6 +330,56 @@ lo        Link encap:Local Loopback
     self.assertEquals(result['operatingsystem'], 'some_type_of_os')
     self.assertEquals(result['osfamily'], 'My_new_family')
 
+
+  @patch("os.path.exists")
+  @patch("os.path.isdir")
+  @patch("json.loads")
+  @patch("glob.glob")
+  @patch("__builtin__.open")
+  @patch.object(OSCheck, "get_os_type")
+  @patch.object(OSCheck, "get_os_version")
+  @patch.object(FacterLinux, "resolve_ambari_config")
+  def test_system_resource_overrides(self, resolve_ambari_config, get_os_version_mock, get_os_type_mock,
+                                     open_mock, glob_mock, json_mock, isdir, exists):
+    get_os_type_mock.return_value = "suse"
+    get_os_version_mock.return_value = "11"
+    config = MagicMock()
+    config.get.return_value = '/etc/custom_resource_overrides'
+    config.has_option.return_value = True
+    resolve_ambari_config.return_value = config
+    isdir.return_value = True
+    exists.return_value = True
+    open_mock.return_value.read = "1"
+    file_handle = open_mock.return_value.__enter__.return_value
+    file_handle.read.return_value = '1'
+    glob_mock.side_effect = \
+      [
+        [
+          "/etc/custom_resource_overrides/1.json",
+          "/etc/custom_resource_overrides/2.json"
+          ]
+      ]
+    json_data = json_mock.return_value
+    json_data.items.return_value = {('key', 'value')}
+    json_data.__getitem__.return_value = 'value'
+
+    facter = Facter()
+    facter.config = config
+    result = facter.getSystemResourceOverrides()
+
+    isdir.assert_called_with('/etc/custom_resource_overrides')
+    exists.assert_called_with('/etc/custom_resource_overrides')
+    glob_mock.assert_called_with('/etc/custom_resource_overrides/*.json')
+    self.assertTrue(config.has_option.called)
+    self.assertTrue(config.get.called)
+    self.assertTrue(glob_mock.called)
+    self.assertEquals(2, file_handle.read.call_count)
+    self.assertEquals(2, open_mock.call_count)
+    self.assertEquals(2, json_mock.call_count)
+    self.assertEquals('value', result['key'])
+
+
+
 if __name__ == "__main__":
   unittest.main()
 


Mime
View raw message