ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-13851 AMS should not assume hdfs as the protocol (dsen)
Date Mon, 16 Nov 2015 12:42:15 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 26a6f5f8d -> 4d15afe5e


AMBARI-13851 AMS should not assume hdfs as the protocol (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4d15afe5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4d15afe5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4d15afe5

Branch: refs/heads/branch-2.1
Commit: 4d15afe5ef62494100ed6ce86b0fd63440194a4c
Parents: 26a6f5f
Author: Dmytro Sen <dsen@apache.org>
Authored: Mon Nov 16 14:33:35 2015 +0200
Committer: Dmytro Sen <dsen@apache.org>
Committed: Mon Nov 16 14:40:42 2015 +0200

----------------------------------------------------------------------
 .../0.1.0/configuration/ams-hbase-env.xml         | 12 ++++++++++++
 .../AMBARI_METRICS/0.1.0/package/scripts/ams.py   |  4 ++--
 .../AMBARI_METRICS/0.1.0/package/scripts/hbase.py |  3 ++-
 .../0.1.0/package/scripts/metrics_collector.py    |  2 +-
 .../0.1.0/package/scripts/params.py               |  3 ++-
 .../stacks/HDP/2.0.6/services/stack_advisor.py    | 18 ++++++++++--------
 .../AMBARI_METRICS/test_metrics_collector.py      |  1 +
 .../stacks/2.0.6/common/test_stack_advisor.py     |  2 +-
 .../stacks/2.2/common/test_stack_advisor.py       |  1 +
 9 files changed, 32 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4d15afe5/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml
index 8a5a0f1..b3f8dd1 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml
@@ -61,6 +61,12 @@
       <type>int</type>
       <unit>MB</unit>
     </value-attributes>
+    <depends-on>
+      <property>
+        <type>ams-hbase-site</type>
+        <name>hbase.cluster.distributed</name>
+      </property>
+    </depends-on>
   </property>
   <property>
     <name>hbase_master_xmn_size</name>
@@ -72,6 +78,12 @@
       <type>int</type>
       <unit>MB</unit>
     </value-attributes>
+    <depends-on>
+      <property>
+        <type>ams-hbase-site</type>
+        <name>hbase.cluster.distributed</name>
+      </property>
+    </depends-on>
   </property>
   <property>
     <name>hbase_master_maxperm_size</name>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4d15afe5/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
index 0bdb45f..84aaa0e 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
@@ -77,7 +77,7 @@ def ams(name=None):
                   username = params.ams_user,
                   password = Script.get_password(params.ams_user))
 
-    if params.is_hbase_distributed:
+    if params.is_hdfs_rootdir:
       # Configuration needed to support NN HA
       XmlConfig("hdfs-site.xml",
             conf_dir=params.ams_collector_conf_dir,
@@ -285,7 +285,7 @@ def ams(name=None):
       )
     pass
 
-    if params.is_hbase_distributed:
+    if params.is_hdfs_rootdir:
       # Configuration needed to support NN HA
       XmlConfig("hdfs-site.xml",
             conf_dir=params.ams_collector_conf_dir,

http://git-wip-us.apache.org/repos/asf/ambari/blob/4d15afe5/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py
index cfe1710..cc86e27 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py
@@ -190,7 +190,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
 
   if name == "master":
 
-    if params.is_hbase_distributed:
+    if params.is_hdfs_rootdir:
       # If executing Stop All, HDFS is probably down
       if action != 'stop':
 
@@ -210,6 +210,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
 
         params.HdfsResource(None, action="execute")
 
+      if params.is_hbase_distributed:
         #Workaround for status commands not aware of operating mode
         File(format("{params.hbase_pid_dir}/distributed_mode"), action="create", mode=0644,
owner=params.hbase_user)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4d15afe5/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py
index 00e4123..c6e556b 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py
@@ -83,7 +83,7 @@ class AmsCollectorDefault(AmsCollector):
     security_params = get_params_from_filesystem(status_params.ams_hbase_conf_dir,
                                                  {'hbase-site.xml': FILE_TYPE_XML})
 
-    is_hbase_distributed = security_params['hbase-site']['hbase.rootdir'].startswith('hdfs://')
+    is_hbase_distributed = security_params['hbase-site']['hbase.cluster.distributed']
     # for embedded mode, when HBase is backed by file, security state is SECURED_KERBEROS
by definition when cluster is secured
     if status_params.security_enabled and not is_hbase_distributed:
       self.put_structured_out({"securityState": "SECURED_KERBEROS"})

http://git-wip-us.apache.org/repos/asf/ambari/blob/4d15afe5/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 1c6a49f..568b11d 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -73,7 +73,8 @@ smokeuser = config['configurations']['cluster-env']['smokeuser']
 hbase_root_dir = config['configurations']['ams-hbase-site']['hbase.rootdir']
 hbase_pid_dir = status_params.hbase_pid_dir
 
-is_hbase_distributed = hbase_root_dir.startswith('hdfs://')
+is_hbase_distributed = config['configurations']['ams-hbase-site']['hbase.cluster.distributed']
+is_hdfs_rootdir = hbase_root_dir.startswith('hdfs://')
 
 # security is disabled for embedded mode, when HBase is backed by file
 security_enabled = False if not is_hbase_distributed else config['configurations']['cluster-env']['security_enabled']

http://git-wip-us.apache.org/repos/asf/ambari/blob/4d15afe5/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index c8114a4..2b55ffb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -437,11 +437,14 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
     rootDir = "file:///var/lib/ambari-metrics-collector/hbase"
     tmpDir = "/var/lib/ambari-metrics-collector/hbase-tmp"
+    hbaseClusterDistributed = False
     if "ams-hbase-site" in services["configurations"]:
       if "hbase.rootdir" in services["configurations"]["ams-hbase-site"]["properties"]:
         rootDir = services["configurations"]["ams-hbase-site"]["properties"]["hbase.rootdir"]
       if "hbase.tmp.dir" in services["configurations"]["ams-hbase-site"]["properties"]:
         tmpDir = services["configurations"]["ams-hbase-site"]["properties"]["hbase.tmp.dir"]
+      if "hbase.cluster.distributed" in services["configurations"]["ams-hbase-site"]["properties"]:
+        hbaseClusterDistributed = services["configurations"]["ams-hbase-site"]["properties"]["hbase.cluster.distributed"].lower()
== 'true'
 
     mountpoints = ["/"]
     for collectorHostName in amsCollectorHosts:
@@ -496,7 +499,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       pass
 
     # Distributed mode heap size
-    if rootDir.startswith("hdfs://"):
+    if hbaseClusterDistributed:
       putHbaseEnvProperty("hbase_master_heapsize", "512")
       putHbaseEnvProperty("hbase_master_xmn_size", "102") #20% of 512 heap size
       putHbaseEnvProperty("hbase_regionserver_heapsize", hbase_heapsize)
@@ -521,7 +524,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     metricsDir = os.path.join(scriptDir, '../../../../common-services/AMBARI_METRICS/0.1.0/package')
     serviceMetricsDir = os.path.join(metricsDir, 'files', 'service-metrics')
     sys.path.append(os.path.join(metricsDir, 'scripts'))
-    mode = 'distributed' if rootDir.startswith("hdfs://") else 'embedded'
+    mode = 'distributed' if hbaseClusterDistributed else 'embedded'
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
 
     from split_points import FindSplitPointsForAMSRegions
@@ -813,7 +816,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     hbase_rootdir = properties.get("hbase.rootdir")
     hbase_tmpdir = properties.get("hbase.tmp.dir")
     if op_mode == "distributed" and not hbase_rootdir.startswith("hdfs://"):
-      rootdir_item = self.getWarnItem("In distributed mode hbase.rootdir should point to
HDFS. Collector will operate in embedded mode otherwise.")
+      rootdir_item = self.getWarnItem("In distributed mode hbase.rootdir should point to
HDFS.")
       pass
 
     distributed_item = None
@@ -890,8 +893,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
     masterXmnItem = None
     regionServerXmnItem = None
-    hbase_rootdir = amsHbaseSite.get("hbase.rootdir")
-    is_hbase_distributed = hbase_rootdir.startswith('hdfs://')
+    is_hbase_distributed = amsHbaseSite.get("hbase.cluster.distributed").lower() == 'true'
 
     if is_hbase_distributed:
       minMasterXmn = 0.12 *  hbase_master_heapsize
@@ -967,8 +969,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
           requiredMemory = getMemorySizeRequired(hostComponents, configurations)
           unusedMemory = host["Hosts"]["total_mem"] * 1024 - requiredMemory # in bytes
           if unusedMemory > 4294967296:  # warn user, if more than 4GB RAM is unused
-            heapPropertyToIncrease = "hbase_regionserver_heapsize" if hbase_rootdir.startswith("hdfs://")
else "hbase_master_heapsize"
-            xmnPropertyToIncrease = "regionserver_xmn_size" if hbase_rootdir.startswith("hdfs://")
else "hbase_master_xmn_size"
+            heapPropertyToIncrease = "hbase_regionserver_heapsize" if is_hbase_distributed
else "hbase_master_heapsize"
+            xmnPropertyToIncrease = "regionserver_xmn_size" if is_hbase_distributed else
"hbase_master_xmn_size"
             collector_heapsize = int((unusedMemory - 4294967296)/5) + to_number(ams_env.get("metrics_collector_heapsize"))*1048576
             hbase_heapsize = int((unusedMemory - 4294967296)*4/5) + to_number(properties.get(heapPropertyToIncrease))*1048576
             hbase_heapsize = min(32*1024*1024*1024, hbase_heapsize) #Make sure heapsize <
32GB
@@ -1018,7 +1020,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
     # '/etc/resolv.conf', '/etc/hostname', '/etc/hosts' are docker specific mount points
     undesirableMountPoints = ["/", "/home", "/etc/resolv.conf", "/etc/hosts",
-                              "/etc/hostname"]
+                              "/etc/hostname", "/tmp"]
     undesirableFsTypes = ["devtmpfs", "tmpfs", "vboxsf", "CDFS"]
     mountPoints = []
     if hostInfo and "disk_info" in hostInfo:

http://git-wip-us.apache.org/repos/asf/ambari/blob/4d15afe5/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
index 9395614..7840b05 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
@@ -17,6 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
+
 from mock.mock import MagicMock, patch
 from stacks.utils.RMFTestCase import *
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4d15afe5/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 99ea217..41da879 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -1352,7 +1352,7 @@ class TestHDP206StackAdvisor(TestCase):
         'config-name': 'hbase.rootdir',
         'config-type': 'ams-hbase-site',
         'level': 'WARN',
-        'message': 'In distributed mode hbase.rootdir should point to HDFS. Collector will
operate in embedded mode otherwise.',
+        'message': 'In distributed mode hbase.rootdir should point to HDFS.',
         'type': 'configuration'
       }
     ]

http://git-wip-us.apache.org/repos/asf/ambari/blob/4d15afe5/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index d3d2c3a..2a30119 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -2173,6 +2173,7 @@ class TestHDP22StackAdvisor(TestCase):
       }
     ]
     services["configurations"]['ams-hbase-site']['properties']['hbase.rootdir'] = 'hdfs://host1/amshbase'
+    services["configurations"]['ams-hbase-site']['properties']['hbase.cluster.distributed']
= 'true'
     expected['ams-hbase-site']['properties']['hbase.rootdir'] = 'hdfs://host1/amshbase'
     expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512'
     # services["configurations"]['ams-hbase-site']['properties']['dfs.client.read.shortcircuit']
= 'true'


Mime
View raw message