ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject git commit: AMBARI-4914 HBase Service Check failed after adding HBase service (Ivan Kozlov via dsen)
Date Mon, 03 Mar 2014 16:06:25 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk b1ad9ea73 -> 9ed9f129b


AMBARI-4914 HBase Service Check failed after adding HBase service (Ivan Kozlov via dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9ed9f129
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9ed9f129
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9ed9f129

Branch: refs/heads/trunk
Commit: 9ed9f129bdc9858efae0cf19c2ec2911398050e8
Parents: b1ad9ea
Author: Dmitry Sen <dsen@hortonworks.com>
Authored: Mon Mar 3 18:06:38 2014 +0200
Committer: Dmitry Sen <dsen@hortonworks.com>
Committed: Mon Mar 3 18:06:38 2014 +0200

----------------------------------------------------------------------
 .../libraries/functions/__init__.py             |  1 +
 .../libraries/functions/substitute_vars.py      | 53 ++++++++++++++
 .../resource_management/TestSubstituteVars.py   | 75 ++++++++++++++++++++
 .../services/HBASE/configuration/hbase-site.xml |  6 ++
 .../services/HBASE/configuration/hbase-site.xml |  6 ++
 .../services/HBASE/configuration/hbase-site.xml |  6 ++
 .../services/HBASE/package/scripts/hbase.py     |  7 ++
 .../services/HBASE/package/scripts/params.py    |  1 +
 .../stacks/2.0.6/HBASE/test_hbase_client.py     | 12 ++++
 .../stacks/2.0.6/HBASE/test_hbase_master.py     | 12 ++++
 .../2.0.6/HBASE/test_hbase_regionserver.py      | 12 ++++
 .../2.0.6/configs/default.hbasedecom.json       |  3 +-
 .../python/stacks/2.0.6/configs/default.json    |  3 +-
 .../python/stacks/2.0.6/configs/secured.json    |  3 +-
 14 files changed, 197 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-agent/src/main/python/resource_management/libraries/functions/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/functions/__init__.py
b/ambari-agent/src/main/python/resource_management/libraries/functions/__init__.py
index dcde79e..32e886b 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/functions/__init__.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/functions/__init__.py
@@ -26,3 +26,4 @@ from resource_management.libraries.functions.get_kinit_path import *
 from resource_management.libraries.functions.get_unique_id_and_date import *
 from resource_management.libraries.functions.check_process_status import *
 from resource_management.libraries.functions.is_empty import *
+from resource_management.libraries.functions.substitute_vars import *

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py
b/ambari-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py
new file mode 100644
index 0000000..2036208
--- /dev/null
+++ b/ambari-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import re
+
+_MAX_SUBST = 20
+
+def substitute_vars(raw, config):
+  """
+  @param raw: str (e.g '${hbase.tmp.dir}/local')
+  @param config: dict (e.g {'hbase.tmp.dir': '/hadoop/hbase'})
+  """
+  result = raw
+
+  pattern = re.compile("\$\{[^\}\$\x0020]+\}")
+
+  for depth in range(0, _MAX_SUBST - 1):
+    match = pattern.search(result)
+
+    if match:
+      start = match.start()
+      end = match.end()
+
+      name = result[start + 2 : end - 1]
+
+      try:
+        value = config[name]
+      except KeyError:
+        return result
+
+      result = result[:start] + value + result[end:]
+    else:
+      break
+
+  return result

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-agent/src/test/python/resource_management/TestSubstituteVars.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestSubstituteVars.py b/ambari-agent/src/test/python/resource_management/TestSubstituteVars.py
new file mode 100644
index 0000000..b3623cd
--- /dev/null
+++ b/ambari-agent/src/test/python/resource_management/TestSubstituteVars.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from unittest import TestCase, main
+from resource_management.libraries.functions.substitute_vars import substitute_vars
+
+import StringIO, sys
+
+class TestSubstituteVars(TestCase):
+  def setUp(self):
+    # disable stdout
+    out = StringIO.StringIO()
+    sys.stdout = out
+
+  def test_substitute_vars(self):
+    raw_config = {
+      'val.intvar' : '42',
+      'pass.intvar' : '${val.intvar}',
+      'fail.unknown' : 'a${unknown}b',
+      'fail.empty' : '${}',
+      'fail.space' : '${ my.int}',
+      'val.0' : 'will_fail',
+      'fail.digit' : '${val.0}',
+      'val.file' : 'hello',
+      'val.suffix' : '.txt',
+      'pass.seq.depth' : '${val.file}${val.suffix}${pass.intvar}',
+      'fail.seq.depth.a' : '${val.file}${unknown}${pass.intvar}',
+      'fail.seq.depth.b' : '${val.file}${fail.seq.depth.a}${pass.intvar}',
+      'val.name' : 'val.intvar',
+      'pass.name.as.param' : '${${val.name}}',
+      'fail.inf.loop' : '${fail.inf.loop}'
+    }
+    expected_config = {
+      'val.intvar' : '42',
+      'pass.intvar' : '42',
+      'fail.unknown' : 'a${unknown}b',
+      'fail.empty' : '${}',
+      'fail.space' : '${ my.int}',
+      'val.0' : 'will_fail',
+      'fail.digit' : '${val.0}',
+      'val.file' : 'hello',
+      'val.suffix' : '.txt',
+      'pass.seq.depth' : 'hello.txt42',
+      'fail.seq.depth.a' : 'hello${unknown}${pass.intvar}',
+      'fail.seq.depth.b' : 'hellohello${unknown}${pass.intvar}${pass.intvar}',
+      'val.name' : 'val.intvar',
+      'pass.name.as.param' : '42',
+      'fail.inf.loop' : '${fail.inf.loop}'
+    }
+
+    for key in raw_config.keys():
+      actual_value = substitute_vars(raw_config[key], raw_config)
+      expected_value = expected_config[key]
+
+      self.assertEqual(actual_value, expected_value)
+
+  def tearDown(self):
+    # enable stdout
+    sys.stdout = sys.__stdout__

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HBASE/configuration/hbase-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HBASE/configuration/hbase-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HBASE/configuration/hbase-site.xml
index 4244bbc..c1f1b29 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HBASE/configuration/hbase-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HBASE/configuration/hbase-site.xml
@@ -52,6 +52,12 @@
     </description>
   </property>
   <property>
+    <name>hbase.local.dir</name>
+    <value>${hbase.tmp.dir}/local</value>
+    <description>Directory on the local filesystem to be used as a local storage
+    </description>
+  </property>
+  <property>
     <name>hbase.master.info.bindAddress</name>
     <value></value>
     <description>The bind address for the HBase Master web UI

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HBASE/configuration/hbase-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HBASE/configuration/hbase-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HBASE/configuration/hbase-site.xml
index 303a74c..104ed0d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HBASE/configuration/hbase-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HBASE/configuration/hbase-site.xml
@@ -52,6 +52,12 @@
     </description>
   </property>
   <property>
+    <name>hbase.local.dir</name>
+    <value>${hbase.tmp.dir}/local</value>
+    <description>Directory on the local filesystem to be used as a local storage
+    </description>
+  </property>
+  <property>
     <name>hbase.master.info.bindAddress</name>
     <value></value>
     <description>The bind address for the HBase Master web UI

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-site.xml
index d1e933d..cf9416e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-site.xml
@@ -52,6 +52,12 @@
     </description>
   </property>
   <property>
+    <name>hbase.local.dir</name>
+    <value>${hbase.tmp.dir}/local</value>
+    <description>Directory on the local filesystem to be used as a local storage
+    </description>
+  </property>
+  <property>
     <name>hbase.master.info.bindAddress</name>
     <value></value>
     <description>The bind address for the HBase Master web UI

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
index d890339..0646b57 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
@@ -48,6 +48,13 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
              recursive = True
   )
 
+  Directory (os.path.join(params.local_dir, "jars"),
+             owner = params.hbase_user,
+             group = params.user_group,
+             mode=0775,
+             recursive = True
+  )
+
   XmlConfig( "hbase-site.xml",
             conf_dir = params.conf_dir,
             configurations = params.config['configurations']['hbase-site'],

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py
index 3b1316c..1d95320 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py
@@ -53,6 +53,7 @@ regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, 0.2, 512)
 
 pid_dir = status_params.pid_dir
 tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
+local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'], config['configurations']['hbase-site'])
 
 client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf_dir}/hbase_client_jaas.conf"))
 master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf"))

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
index b497745..50dda17 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
@@ -39,6 +39,12 @@ class TestHBaseClient(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',
@@ -96,6 +102,12 @@ class TestHBaseClient(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
index f413857..1084ed3 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
@@ -200,6 +200,12 @@ class TestHBaseMaster(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',
@@ -281,6 +287,12 @@ class TestHBaseMaster(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
index d0eaa90..4ced781 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
@@ -133,6 +133,12 @@ class TestHbaseRegionServer(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',
@@ -214,6 +220,12 @@ class TestHbaseRegionServer(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json
index 16d223e..f99821b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json
@@ -372,6 +372,7 @@
             "hbase.regionserver.global.memstore.upperLimit": "0.4", 
             "zookeeper.session.timeout": "30000", 
             "hbase.tmp.dir": "/hadoop/hbase", 
+            "hbase.local.dir": "${hbase.tmp.dir}/local", 
             "hbase.hregion.max.filesize": "10737418240", 
             "hfile.block.cache.size": "0.40", 
             "hbase.security.authentication": "simple", 
@@ -609,4 +610,4 @@
             "c6402.ambari.apache.org"
         ]
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index fea2268..db34e87 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -372,6 +372,7 @@
             "hbase.regionserver.global.memstore.upperLimit": "0.4", 
             "zookeeper.session.timeout": "30000", 
             "hbase.tmp.dir": "/hadoop/hbase", 
+            "hbase.local.dir": "${hbase.tmp.dir}/local", 
             "hbase.hregion.max.filesize": "10737418240", 
             "hfile.block.cache.size": "0.40", 
             "hbase.security.authentication": "simple", 
@@ -657,4 +658,4 @@
             "c6402.ambari.apache.org"
         ]
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/9ed9f129/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
index a8c9315..8bebaf7 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
@@ -487,6 +487,7 @@
             "hbase.regionserver.global.memstore.upperLimit": "0.4", 
             "zookeeper.session.timeout": "30000", 
             "hbase.tmp.dir": "/hadoop/hbase", 
+            "hbase.local.dir": "${hbase.tmp.dir}/local", 
             "hfile.block.cache.size": "0.40", 
             "hbase.regionserver.kerberos.principal": "hbase/_HOST@EXAMPLE.COM", 
             "hbase.security.authentication": "kerberos", 
@@ -784,4 +785,4 @@
             "c6402.ambari.apache.org"
         ]
     }
-}
\ No newline at end of file
+}


Mime
View raw message