ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-14234 If a non-hdfs path is specified (such as wasb) as the hbase.rootdir then stack advisor throws error (dsen)
Date Sun, 06 Dec 2015 15:37:00 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.2 d13cd352d -> 16307d4d2


AMBARI-14234 If a non-hdfs path is specified (such as wasb) as the hbase.rootdir then stack
advisor throws error (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/16307d4d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/16307d4d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/16307d4d

Branch: refs/heads/branch-2.2
Commit: 16307d4d2fe74a97ffd43fe4b6c957122c843d39
Parents: d13cd35
Author: Dmytro Sen <dsen@apache.org>
Authored: Sun Dec 6 17:34:23 2015 +0200
Committer: Dmytro Sen <dsen@apache.org>
Committed: Sun Dec 6 17:35:35 2015 +0200

----------------------------------------------------------------------
 .../AMBARI_METRICS/0.1.0/package/scripts/ams.py |  2 +-
 .../0.1.0/package/scripts/params.py             |  1 +
 .../stacks/HDP/2.0.6/services/stack_advisor.py  | 30 +++++++-------
 .../stacks/2.0.6/common/test_stack_advisor.py   | 42 ++++++++++++++++++--
 4 files changed, 57 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/16307d4d/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
index 615e93a..733fcc0 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
@@ -285,7 +285,7 @@ def ams(name=None):
       )
     pass
 
-    if not params.is_local_fs_rootdir:
+    if not params.is_local_fs_rootdir and params.is_ams_distributed:
       # Configuration needed to support NN HA
       XmlConfig("hdfs-site.xml",
             conf_dir=params.ams_collector_conf_dir,

http://git-wip-us.apache.org/repos/asf/ambari/blob/16307d4d/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index ec5e848..9125a01 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -90,6 +90,7 @@ hbase_pid_dir = status_params.hbase_pid_dir
 
 is_hbase_distributed = config['configurations']['ams-hbase-site']['hbase.cluster.distributed']
 is_local_fs_rootdir = hbase_root_dir.startswith('file://')
+is_ams_distributed = config['configurations']['ams-site']['timeline.metrics.service.operation.mode']
== 'distributed'
 
 # security is disabled for embedded mode, when HBase is backed by file
 security_enabled = False if not is_hbase_distributed else config['configurations']['cluster-env']['security_enabled']

http://git-wip-us.apache.org/repos/asf/ambari/blob/16307d4d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 9d9e0dd..f3e6f8b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -476,11 +476,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
         if host["Hosts"]["host_name"] == collectorHostName:
           mountpoints = self.getPreferredMountPoints(host["Hosts"])
           break
-    if not rootDir.startswith("hdfs://"):
+    isLocalRootDir = rootDir.startswith("file://")
+    if isLocalRootDir:
       rootDir = re.sub("^file:///|/", "", rootDir, count=1)
       rootDir = "file://" + os.path.join(mountpoints[0], rootDir)
     tmpDir = re.sub("^file:///|/", "", tmpDir, count=1)
-    if len(mountpoints) > 1 and not rootDir.startswith("hdfs://"):
+    if len(mountpoints) > 1 and isLocalRootDir:
       tmpDir = os.path.join(mountpoints[1], tmpDir)
     else:
       tmpDir = os.path.join(mountpoints[0], tmpDir)
@@ -840,7 +841,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     op_mode = ams_site.get("timeline.metrics.service.operation.mode")
     hbase_rootdir = properties.get("hbase.rootdir")
     hbase_tmpdir = properties.get("hbase.tmp.dir")
-    if op_mode == "distributed" and not hbase_rootdir.startswith("hdfs://"):
+    if op_mode == "distributed" and hbase_rootdir.startswith("file://"):
       rootdir_item = self.getWarnItem("In distributed mode hbase.rootdir should point to
HDFS.")
       pass
 
@@ -855,9 +856,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     for collectorHostName in amsCollectorHosts:
       for host in hosts["items"]:
         if host["Hosts"]["host_name"] == collectorHostName:
-          validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorEnoughDiskSpace(properties,
'hbase.rootdir', host["Hosts"], recommendedDiskSpace)}])
-          validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorNotRootFs(properties,
recommendedDefaults, 'hbase.rootdir', host["Hosts"])}])
-          validationItems.extend([{"config-name": 'hbase.tmp.dir', "item": self.validatorNotRootFs(properties,
recommendedDefaults, 'hbase.tmp.dir', host["Hosts"])}])
+          if op_mode == 'embedded':
+            validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorEnoughDiskSpace(properties,
'hbase.rootdir', host["Hosts"], recommendedDiskSpace)}])
+            validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorNotRootFs(properties,
recommendedDefaults, 'hbase.rootdir', host["Hosts"])}])
+            validationItems.extend([{"config-name": 'hbase.tmp.dir', "item": self.validatorNotRootFs(properties,
recommendedDefaults, 'hbase.tmp.dir', host["Hosts"])}])
 
           dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE")
           if not hbase_rootdir.startswith("hdfs"):
@@ -951,35 +953,35 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       maxMasterXmn = 0.2 * hbase_master_heapsize
       if hbase_master_xmn_size < minMasterXmn:
         masterXmnItem = self.getWarnItem("Value is lesser than the recommended minimum Xmn
size of {0} "
-                                         "(12% of hbase_master_heapsize)".format(int(math.ceil(minMasterXmn))))
+                                         "(12% of hbase_master_heapsize)".format(int(ceil(minMasterXmn))))
 
       if hbase_master_xmn_size > maxMasterXmn:
         masterXmnItem = self.getWarnItem("Value is greater than the recommended maximum Xmn
size of {0} "
-                                         "(20% of hbase_master_heapsize)".format(int(math.floor(maxMasterXmn))))
+                                         "(20% of hbase_master_heapsize)".format(int(floor(maxMasterXmn))))
 
       minRegionServerXmn = 0.12 * hbase_regionserver_heapsize
       maxRegionServerXmn = 0.2 * hbase_regionserver_heapsize
       if hbase_regionserver_xmn_size < minRegionServerXmn:
         regionServerXmnItem = self.getWarnItem("Value is lesser than the recommended minimum
Xmn size of {0} "
                                                "(12% of hbase_regionserver_heapsize)"
-                                               .format(int(math.ceil(minRegionServerXmn))))
+                                               .format(int(ceil(minRegionServerXmn))))
 
       if hbase_regionserver_xmn_size > maxRegionServerXmn:
         regionServerXmnItem = self.getWarnItem("Value is greater than the recommended maximum
Xmn size of {0} "
                                                "(20% of hbase_regionserver_heapsize)"
-                                               .format(int(math.floor(maxRegionServerXmn))))
+                                               .format(int(floor(maxRegionServerXmn))))
     else:
       minMasterXmn = 0.12 * (hbase_master_heapsize + hbase_regionserver_heapsize)
       maxMasterXmn = 0.2 *  (hbase_master_heapsize + hbase_regionserver_heapsize)
       if hbase_master_xmn_size < minMasterXmn:
         masterXmnItem = self.getWarnItem("Value is lesser than the recommended minimum Xmn
size of {0} "
                                          "(12% of hbase_master_heapsize + hbase_regionserver_heapsize)"
-                                         .format(int(math.ceil(minMasterXmn))))
+                                         .format(int(ceil(minMasterXmn))))
 
       if hbase_master_xmn_size > maxMasterXmn:
         masterXmnItem = self.getWarnItem("Value is greater than the recommended maximum Xmn
size of {0} "
                                          "(20% of hbase_master_heapsize + hbase_regionserver_heapsize)"
-                                         .format(int(math.floor(maxMasterXmn))))
+                                         .format(int(floor(maxMasterXmn))))
     if masterXmnItem:
       validationItems.extend([{"config-name": "hbase_master_xmn_size", "item": masterXmnItem}])
 
@@ -1098,7 +1100,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     if not propertyName in properties:
       return self.getErrorItem("Value should be set")
     dir = properties[propertyName]
-    if dir.startswith("hdfs://") or dir == recommendedDefaults.get(propertyName):
+    if not dir.startswith("file://") or dir == recommendedDefaults.get(propertyName):
       return None
 
     dir = re.sub("^file://", "", dir, count=1)
@@ -1116,7 +1118,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     if not propertyName in properties:
       return self.getErrorItem("Value should be set")
     dir = properties[propertyName]
-    if dir.startswith("hdfs://"):
+    if not dir.startswith("file://"):
       return None
 
     dir = re.sub("^file://", "", dir, count=1)

http://git-wip-us.apache.org/repos/asf/ambari/blob/16307d4d/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index c946d42..46e51f8 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -1726,8 +1726,8 @@ class TestHDP206StackAdvisor(TestCase):
         "mountpoint" : "/"
       }
     ]}
-    properties = {"property1": "/var/dir"}
-    recommendedDefaults = {"property1": "/var/dir"}
+    properties = {"property1": "file:///var/dir"}
+    recommendedDefaults = {"property1": "file:///var/dir"}
     # only / mountpoint - no warning
     self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults,
'property1', hostInfo) == None)
     # More preferable /grid/0 mountpoint - warning
@@ -1738,7 +1738,7 @@ class TestHDP206StackAdvisor(TestCase):
         "mountpoint" : "/grid/0"
       }
     )
-    recommendedDefaults = {"property1": "/grid/0/var/dir"}
+    recommendedDefaults = {"property1": "file:///grid/0/var/dir"}
     warn = self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults, 'property1',
hostInfo)
     self.assertTrue(warn != None)
     self.assertEquals({'message': 'It is not recommended to use root partition for property1',
'level': 'WARN'}, warn)
@@ -1753,6 +1753,42 @@ class TestHDP206StackAdvisor(TestCase):
     )
     self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults,
'property1', hostInfo) == None)
 
+  def test_validatorEnoughDiskSpace(self):
+    reqiuredDiskSpace = 1048576
+    errorMsg = "Ambari Metrics disk space requirements not met. \n" \
+               "Recommended disk space for partition / is 1G"
+
+    # local FS, enough space
+    hostInfo = {"disk_info": [
+      {
+        "available" : "1048578",
+        "type" : "ext4",
+        "mountpoint" : "/"
+      }
+    ]}
+    properties = {"property1": "file:///var/dir"}
+    self.assertTrue(self.stackAdvisor.validatorEnoughDiskSpace(properties, 'property1', hostInfo,
reqiuredDiskSpace) == None)
+
+    # local FS, no enough space
+    hostInfo = {"disk_info": [
+      {
+        "available" : "1",
+        "type" : "ext4",
+        "mountpoint" : "/"
+      }
+    ]}
+    warn = self.stackAdvisor.validatorEnoughDiskSpace(properties, 'property1', hostInfo,
reqiuredDiskSpace)
+    self.assertTrue(warn != None)
+    self.assertEquals({'message': errorMsg, 'level': 'WARN'}, warn)
+
+    # non-local FS, HDFS
+    properties = {"property1": "hdfs://h1"}
+    self.assertTrue(self.stackAdvisor.validatorEnoughDiskSpace(properties, 'property1', hostInfo,
reqiuredDiskSpace) == None)
+
+    # non-local FS, WASB
+    properties = {"property1": "wasb://h1"}
+    self.assertTrue(self.stackAdvisor.validatorEnoughDiskSpace(properties, 'property1', hostInfo,
reqiuredDiskSpace) == None)
+
   def test_round_to_n(self):
     self.assertEquals(self.stack_advisor_impl.round_to_n(0), 0)
     self.assertEquals(self.stack_advisor_impl.round_to_n(1000), 1024)


Mime
View raw message