ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smn...@apache.org
Subject ambari git commit: AMBARI-18299: Support open files ulimit for ambari-agent
Date Tue, 06 Sep 2016 18:52:23 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 2d27f547f -> 1005843fe


AMBARI-18299: Support open files ulimit for ambari-agent


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1005843f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1005843f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1005843f

Branch: refs/heads/trunk
Commit: 1005843fea5c8dffecfde29c43c45031897d24af
Parents: 2d27f54
Author: Nahappan Somasundaram <nsomasundaram@hortonworks.com>
Authored: Thu Sep 1 16:20:03 2016 -0700
Committer: Nahappan Somasundaram <nsomasundaram@hortonworks.com>
Committed: Tue Sep 6 10:24:34 2016 -0700

----------------------------------------------------------------------
 .../main/python/ambari_agent/AmbariConfig.py    | 12 ++++++++++++
 .../src/main/python/ambari_agent/main.py        | 20 +++++++++++++++++++-
 .../python/ambari_agent/TestAmbariConfig.py     |  8 ++++++++
 .../src/test/python/ambari_agent/TestMain.py    | 10 ++++++++++
 4 files changed, 49 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/1005843f/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
index ae938dc..78b34e5 100644
--- a/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
+++ b/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py
@@ -154,6 +154,9 @@ class AmbariConfig:
   SERVER_CONNECTION_INFO = "{0}/connection_info"
   CONNECTION_PROTOCOL = "https"
 
+  # linux open-file limit
+  ULIMIT_OPEN_FILES_KEY = 'ulimit.open.files'
+
   config = None
   net = None
 
@@ -264,6 +267,15 @@ class AmbariConfig:
   def get_parallel_exec_option(self):
     return int(self.get('agent', 'parallel_execution', 0))
 
+  def get_ulimit_open_files(self):
+    open_files_config_val =  int(self.get('agent', self.ULIMIT_OPEN_FILES_KEY, 0))
+    open_files_ulimit = int(open_files_config_val) if (open_files_config_val and int(open_files_config_val)
> 0) else 0
+    return open_files_ulimit
+
+  def set_ulimit_open_files(self, value):
+    self.set('agent', self.ULIMIT_OPEN_FILES_KEY, value)
+
+
   def update_configuration_from_registration(self, reg_resp):
     if reg_resp and AmbariConfig.AMBARI_PROPERTIES_CATEGORY in reg_resp:
       if not self.has_section(AmbariConfig.AMBARI_PROPERTIES_CATEGORY):

http://git-wip-us.apache.org/repos/asf/ambari/blob/1005843f/ambari-agent/src/main/python/ambari_agent/main.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/main.py b/ambari-agent/src/main/python/ambari_agent/main.py
index d7864b3..76a6a5b 100644
--- a/ambari-agent/src/main/python/ambari_agent/main.py
+++ b/ambari-agent/src/main/python/ambari_agent/main.py
@@ -50,6 +50,7 @@ import locale
 import platform
 import ConfigParser
 import ProcessHelper
+import resource
 from logging.handlers import SysLogHandler
 from Controller import Controller
 import AmbariConfig
@@ -165,7 +166,22 @@ def check_sudo():
   if run_time > 2:
     logger.warn(("Sudo commands on this host are running slowly ('{0}' took {1} seconds).\n"
+
                 "This will create a significant slow down for ambari-agent service tasks.").format(test_command_str,
run_time))
-    
+
+# Updates the hard limit for open file handles
+def update_open_files_ulimit(config):
+  global logger
+  # get the current soft and hard limits
+  # if the specified value is greater than or equal to the soft limit
+  # we can update the hard limit
+  (soft_limit, hard_limit) = resource.getrlimit(resource.RLIMIT_NOFILE)
+  open_files_ulimit = config.get_ulimit_open_files()
+  if open_files_ulimit >= soft_limit:
+    try:
+      resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, open_files_ulimit))
+      logger.info('open files ulimit = {0}'.format(open_files_ulimit))
+    except ValueError, err:
+      logger.error('Unable to set open files ulimit to {0}: {1}'.format(open_files_ulimit,
str(err)))
+      logger.info('open files ulimit = {0}'.format(hard_limit))
 
 def perform_prestart_checks(expected_hostname):
   # Check if current hostname is equal to expected one (got from the server
@@ -329,6 +345,8 @@ def main(heartbeat_stop_callback=None):
 
   update_log_level(config)
 
+  update_open_files_ulimit(config)
+
   if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
     daemonize()
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/1005843f/ambari-agent/src/test/python/ambari_agent/TestAmbariConfig.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestAmbariConfig.py b/ambari-agent/src/test/python/ambari_agent/TestAmbariConfig.py
index 78f29fe..c89644f 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestAmbariConfig.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestAmbariConfig.py
@@ -45,3 +45,11 @@ class TestAmbariConfig(TestCase):
     config.set("security", "keysdir", " /tmp/non-stripped")
     self.assertEqual(config.get("security", "keysdir"), "/tmp/non-stripped")
 
+    # test default value
+    open_files_ulimit = config.get_ulimit_open_files()
+    self.assertEqual(open_files_ulimit, 0)
+
+    # set a value
+    open_files_ulimit = 128000
+    config.set_ulimit_open_files(open_files_ulimit)
+    self.assertEqual(config.get_ulimit_open_files(), open_files_ulimit)

http://git-wip-us.apache.org/repos/asf/ambari/blob/1005843f/ambari-agent/src/test/python/ambari_agent/TestMain.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestMain.py b/ambari-agent/src/test/python/ambari_agent/TestMain.py
index 23cdb13..aa62f29 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestMain.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestMain.py
@@ -27,6 +27,7 @@ import socket
 import tempfile
 import ConfigParser
 import ambari_agent.hostname as hostname
+import resource
 
 from ambari_commons import OSCheck
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -126,6 +127,15 @@ class TestMain(unittest.TestCase):
     main.update_log_level(config)
     setLevel_mock.assert_called_with(logging.INFO)
 
+  # Set open files ulimit hard limit
+  def test_update_open_files_ulimit(self):
+    config = AmbariConfig()
+    open_files_ulimit = 10240
+    config.set_ulimit_open_files(open_files_ulimit)
+    main.update_open_files_ulimit(config)
+    (soft_limit, hard_limit) = resource.getrlimit(resource.RLIMIT_NOFILE)
+    self.assertEquals(hard_limit, open_files_ulimit)
+
   @not_for_platform(PLATFORM_WINDOWS)
   @patch("signal.signal")
   def test_bind_signal_handlers(self, signal_mock):


Mime
View raw message