ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-12813 Increase OS kernel parameters for hbase (Juanjo Marron via dsen)
Date Wed, 30 Sep 2015 14:42:01 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk b3d36fb5c -> bff61b83e


AMBARI-12813 Increase OS kernel parameters for hbase (Juanjo Marron via dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bff61b83
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bff61b83
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bff61b83

Branch: refs/heads/trunk
Commit: bff61b83e962dfc7a8c7c9f8198bd038f16b6ec7
Parents: b3d36fb
Author: Dmytro Sen <dsen@apache.org>
Authored: Wed Sep 30 17:41:48 2015 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Wed Sep 30 17:41:48 2015 +0300

----------------------------------------------------------------------
 .../0.96.0.2.0/configuration/hbase-env.xml      | 12 ++++++-
 .../HBASE/0.96.0.2.0/package/scripts/hbase.py   | 20 +++++++++--
 .../0.96.0.2.0/package/scripts/params_linux.py  |  4 +++
 .../0.96.0.2.0/package/scripts/status_params.py |  1 +
 .../0.96.0.2.0/package/templates/hbase.conf.j2  | 35 ++++++++++++++++++++
 .../stacks/2.0.6/HBASE/test_hbase_client.py     | 28 ++++++++++++++--
 .../stacks/2.0.6/HBASE/test_hbase_master.py     | 34 ++++++++++++++++++-
 .../2.0.6/HBASE/test_hbase_regionserver.py      | 33 ++++++++++++++++++
 .../2.0.6/HBASE/test_phoenix_queryserver.py     | 24 +++++++++++++-
 9 files changed, 182 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
index 242e9a7..f1dd64f 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
@@ -86,12 +86,22 @@ and the -Xmn ratio (hbase_regionserver_xmn_ratio) exceeds this value.
       <increment-step>256</increment-step>
     </value-attributes>
   </property>
-   <property>
+  <property>
     <name>hbase_user</name>
     <value>hbase</value>
     <property-type>USER</property-type>
     <description>HBase User Name.</description>
   </property>
+  <property>
+    <name>hbase_user_nofile_limit</name>
+    <value>32000</value>
+    <description>Max open files limit setting for HBASE user.</description>
+  </property>
+  <property>
+    <name>hbase_user_nproc_limit</name>
+    <value>16000</value>
+    <description>Max number of processes limit setting for HBASE user.</description>
+  </property>
 
   <!-- hbase-env.sh -->
   <property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
index 991f4e7..6b5369e 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
@@ -55,7 +55,7 @@ def hbase(name=None):
       group = params.user_group,
       recursive = True
   )
-  
+ 
   parent_dir = os.path.dirname(params.tmp_dir)
   # In case if we have several placeholders in path
   while ("${" in parent_dir):
@@ -119,8 +119,22 @@ def hbase(name=None):
        owner = params.hbase_user,
        content=InlineTemplate(params.hbase_env_sh_template),
        group = params.user_group,
-  )     
-       
+  )
+  
+  # On some OS this folder could be not exists, so we will create it before pushing there
files
+  Directory(params.limits_conf_dir,
+            recursive=True,
+            owner='root',
+            group='root'
+            )
+  
+  File(os.path.join(params.limits_conf_dir, 'hbase.conf'),
+       owner='root',
+       group='root',
+       mode=0644,
+       content=Template("hbase.conf.j2")
+       )
+    
   hbase_TemplateConfig( params.metric_prop_file_name,
     tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS'
   )

http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
index f22b035..635be5e 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
@@ -75,6 +75,10 @@ if Script.is_hdp_stack_greater_or_equal("2.2"):
 
 
 hbase_conf_dir = status_params.hbase_conf_dir
+limits_conf_dir = status_params.limits_conf_dir
+
+hbase_user_nofile_limit = default("/configurations/hbase-env/hbase_user_nofile_limit", "32000")
+hbase_user_nproc_limit = default("/configurations/hbase-env/hbase_user_nproc_limit", "16000")
 
 # no symlink for phoenix-server at this point
 phx_daemon_script = '/usr/hdp/current/phoenix-server/bin/queryserver.py'

http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
index 084ee06..014e8d7 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/status_params.py
@@ -50,5 +50,6 @@ else:
   tmp_dir = Script.get_tmp_dir()
 
   hbase_conf_dir = "/etc/hbase/conf"
+  limits_conf_dir = "/etc/security/limits.d"
   if Script.is_hdp_stack_greater_or_equal("2.2"):
     hbase_conf_dir = format("/usr/hdp/current/{component_directory}/conf")
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2
b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2
new file mode 100644
index 0000000..3580db0
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/templates/hbase.conf.j2
@@ -0,0 +1,35 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{hbase_user}}   - nofile   {{hbase_user_nofile_limit}}
+{{hbase_user}}   - nproc    {{hbase_user_nproc_limit}}

http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
index 9959874..8d96849 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
@@ -90,6 +90,17 @@ class TestHBaseClient(RMFTestCase):
         owner = 'hbase',
         group = 'hadoop'
     )
+    self.assertResourceCalled('Directory', '/etc/security/limits.d',
+      owner = 'root',
+      group = 'root',
+      recursive = True,
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+      content = Template('hbase.conf.j2'),
+      owner = 'root',
+      group = 'root',
+      mode = 0644,
+    )
     self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
       owner = 'hbase',
       template_tag = 'GANGLIA-RS',
@@ -167,9 +178,20 @@ class TestHBaseClient(RMFTestCase):
       group = 'hadoop',
     )
     self.assertResourceCalled('File', '/etc/hbase/conf/hbase-env.sh',
-        content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
-        owner = 'hbase',
-        group = 'hadoop',
+      content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
+      owner = 'hbase',
+      group = 'hadoop',
+    )
+    self.assertResourceCalled('Directory', '/etc/security/limits.d',
+      owner = 'root',
+      group = 'root',
+      recursive = True,
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+      content = Template('hbase.conf.j2'),
+      owner = 'root',
+      group = 'root',
+      mode = 0644,
     )
     self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
       owner = 'hbase',

http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
index 74b4aa6..4ca74a8 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
@@ -277,6 +277,17 @@ class TestHBaseMaster(RMFTestCase):
       content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
       group = 'hadoop',
     )
+    self.assertResourceCalled('Directory', '/etc/security/limits.d',
+      owner = 'root',
+      group = 'root',
+      recursive = True,
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+      content = Template('hbase.conf.j2'),
+      owner = 'root',
+      group = 'root',
+      mode = 0644,
+    )
     self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
       owner = 'hbase',
       template_tag = 'GANGLIA-MASTER',
@@ -393,6 +404,17 @@ class TestHBaseMaster(RMFTestCase):
       content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
       group = 'hadoop',
     )
+    self.assertResourceCalled('Directory', '/etc/security/limits.d',
+      owner = 'root',
+      group = 'root',
+      recursive = True,
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+      content = Template('hbase.conf.j2'),
+      owner = 'root',
+      group = 'root',
+      mode = 0644,
+    )
     self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
       owner = 'hbase',
       template_tag = 'GANGLIA-MASTER',
@@ -524,7 +546,17 @@ class TestHBaseMaster(RMFTestCase):
       content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
       group = 'hadoop'
     )
-
+    self.assertResourceCalled('Directory', '/etc/security/limits.d',
+      owner = 'root',
+      group = 'root',
+      recursive = True,
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+      content = Template('hbase.conf.j2'),
+      owner = 'root',
+      group = 'root',
+      mode = 0644,
+    )
     self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-master/conf/hadoop-metrics2-hbase.properties',
       owner = 'hbase',
       template_tag = 'GANGLIA-MASTER')

http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
index fa134b4..e22584a 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
@@ -177,6 +177,17 @@ class TestHbaseRegionServer(RMFTestCase):
       content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
       group = 'hadoop',
     )
+    self.assertResourceCalled('Directory', '/etc/security/limits.d',
+      owner = 'root',
+      group = 'root',
+      recursive = True,
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+      content = Template('hbase.conf.j2'),
+      owner = 'root',
+      group = 'root',
+      mode = 0644,
+    )
     self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
       owner = 'hbase',
       template_tag = 'GANGLIA-RS',
@@ -254,6 +265,17 @@ class TestHbaseRegionServer(RMFTestCase):
       content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
       group = 'hadoop',
     )
+    self.assertResourceCalled('Directory', '/etc/security/limits.d',
+        owner = 'root',
+        group = 'root',
+        recursive = True,
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+        content = Template('hbase.conf.j2'),
+        owner = 'root',
+        group = 'root',
+        mode = 0644,
+    )
     self.assertResourceCalled('TemplateConfig', '/etc/hbase/conf/hadoop-metrics2-hbase.properties',
       owner = 'hbase',
       template_tag = 'GANGLIA-RS',
@@ -346,6 +368,17 @@ class TestHbaseRegionServer(RMFTestCase):
       content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
       group = 'hadoop'
     )
+    self.assertResourceCalled('Directory', '/etc/security/limits.d',
+      owner = 'root',
+      group = 'root',
+      recursive = True,
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+      content = Template('hbase.conf.j2'),
+      owner = 'root',
+      group = 'root',
+      mode = 0644,
+    )
 
     self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties',
       owner = 'hbase',

http://git-wip-us.apache.org/repos/asf/ambari/blob/bff61b83/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
index 51a9edc..2cb535c 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
@@ -271,6 +271,17 @@ class TestPhoenixQueryServer(RMFTestCase):
         self.getConfig()['configurations']['hbase-env']['content']),
       group = 'hadoop',
     )
+    self.assertResourceCalled('Directory', '/etc/security/limits.d',
+      owner = 'root',
+      group = 'root',
+      recursive = True,
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+      content = Template('hbase.conf.j2'),
+      owner = 'root',
+      group = 'root',
+      mode = 0644,
+    )    
     self.assertResourceCalled('TemplateConfig',
       '/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties',
       owner = 'hbase',
@@ -361,6 +372,17 @@ class TestPhoenixQueryServer(RMFTestCase):
         self.getConfig()['configurations']['hbase-env']['content']),
       group = 'hadoop',
     )
+    self.assertResourceCalled('Directory', '/etc/security/limits.d',
+      owner = 'root',
+      group = 'root',
+      recursive = True,
+    )
+    self.assertResourceCalled('File', '/etc/security/limits.d/hbase.conf',
+      content = Template('hbase.conf.j2'),
+      owner = 'root',
+      group = 'root',
+      mode = 0644,
+    )
     self.assertResourceCalled('TemplateConfig',
       '/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties',
       owner = 'hbase',
@@ -415,4 +437,4 @@ class TestPhoenixQueryServer(RMFTestCase):
     )
     self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set',
'phoenix-server', '2.3.0.0-1234'), sudo=True)
 
-    self.assertNoMoreResources()
\ No newline at end of file
+    self.assertNoMoreResources()


Mime
View raw message