ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-13105 Minor optimizations for AMS memory settings (dsen)
Date Thu, 17 Sep 2015 16:51:05 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 bd155764a -> 87e9ded96


AMBARI-13105 Minor optimizations for AMS memory settings (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/87e9ded9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/87e9ded9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/87e9ded9

Branch: refs/heads/branch-2.1
Commit: 87e9ded96c6e07acd97f9fd06fff7d623c4c3e1f
Parents: bd15576
Author: Dmytro Sen <dsen@apache.org>
Authored: Thu Sep 17 19:39:14 2015 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Thu Sep 17 19:50:40 2015 +0300

----------------------------------------------------------------------
 .../stacks/HDP/2.0.6/services/stack_advisor.py  |  95 +++++---
 .../stacks/2.0.6/common/test_stack_advisor.py   | 220 ++++++++++++++++++-
 .../stacks/2.2/common/test_stack_advisor.py     |  24 +-
 3 files changed, 287 insertions(+), 52 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/87e9ded9/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 4d1ab23..b2332b0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -288,7 +288,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
         total_sinks_count += schCount
     collector_heapsize = int(hbase_heapsize/4 if hbase_heapsize > 2048 else 512)
 
-    return collector_heapsize, hbase_heapsize, total_sinks_count
+    return round_to_n(collector_heapsize), round_to_n(hbase_heapsize), total_sinks_count
 
   def recommendAmsConfigurations(self, configurations, clusterData, services, hosts):
     putAmsEnvProperty = self.putProperty(configurations, "ams-env", services)
@@ -334,13 +334,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.3)
     putTimelineServiceProperty("timeline.metrics.host.aggregator.ttl", 86400)
 
-    # Embedded mode heap size : master + regionserver
-    if rootDir.startswith("hdfs://"):
-      putHbaseEnvProperty("hbase_master_heapsize", "512m")
-      putHbaseEnvProperty("hbase_regionserver_heapsize", str(hbase_heapsize) + "m")
-    else:
-      putHbaseEnvProperty("hbase_master_heapsize", str(hbase_heapsize) + "m")
-
     if len(amsCollectorHosts) > 1:
       pass
     else:
@@ -354,20 +347,28 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
         putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.25)
         putAmsHbaseSiteProperty("phoenix.query.maxGlobalMemoryPercentage", 20)
         putTimelineServiceProperty("phoenix.query.maxGlobalMemoryPercentage", 30)
-        putAmsHbaseSiteProperty("hbase_master_xmn_size", "512m")
-        putAmsHbaseSiteProperty("regionserver_xmn_size", "512m")
+        hbase_xmn_size = '512m'
       elif total_sinks_count >= 500:
         putAmsHbaseSiteProperty("hbase.regionserver.handler.count", 60)
         putAmsHbaseSiteProperty("hbase.regionserver.hlog.blocksize", 134217728)
         putAmsHbaseSiteProperty("hbase.regionserver.maxlogs", 64)
         putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 268435456)
-        putAmsHbaseSiteProperty("hbase_master_xmn_size", "512m")
+        hbase_xmn_size = '512m'
       elif total_sinks_count >= 250:
-        putAmsHbaseSiteProperty("hbase_master_xmn_size", "256m")
+        hbase_xmn_size = '256m'
       else:
-        putAmsHbaseSiteProperty("hbase_master_xmn_size", "128m")
+        hbase_xmn_size = '128m'
       pass
 
+    # Embedded mode heap size : master + regionserver
+    if rootDir.startswith("hdfs://"):
+      putHbaseEnvProperty("hbase_master_heapsize", "512m")
+      putHbaseEnvProperty("hbase_regionserver_heapsize", str(hbase_heapsize) + "m")
+      putHbaseEnvProperty("regionserver_xmn_size", hbase_xmn_size)
+    else:
+      putHbaseEnvProperty("hbase_master_heapsize", str(hbase_heapsize) + "m")
+      putHbaseEnvProperty("hbase_master_xmn_size", hbase_xmn_size)
+
     #split points
     scriptDir = os.path.dirname(os.path.abspath(__file__))
     metricsDir = os.path.join(scriptDir, '../../../../common-services/AMBARI_METRICS/0.1.0/package')
@@ -630,6 +631,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     rootdir_item = None
     op_mode = ams_site.get("timeline.metrics.service.operation.mode")
     hbase_rootdir = properties.get("hbase.rootdir")
+    hbase_tmpdir = properties.get("hbase.tmp.dir")
     if op_mode == "distributed" and not hbase_rootdir.startswith("hdfs://"):
       rootdir_item = self.getWarnItem("In distributed mode hbase.rootdir should point to
HDFS. Collector will operate in embedded mode otherwise.")
       pass
@@ -646,25 +648,33 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       for host in hosts["items"]:
         if host["Hosts"]["host_name"] == collectorHostName:
           validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorEnoughDiskSpace(properties,
'hbase.rootdir', host["Hosts"], recommendedDiskSpace)}])
-          validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorNotRootFs(properties,
'hbase.rootdir', host["Hosts"])}])
-          validationItems.extend([{"config-name": 'hbase.tmp.dir', "item": self.validatorNotRootFs(properties,
'hbase.tmp.dir', host["Hosts"])}])
+          validationItems.extend([{"config-name": 'hbase.rootdir', "item": self.validatorNotRootFs(properties,
recommendedDefaults, 'hbase.rootdir', host["Hosts"])}])
+          validationItems.extend([{"config-name": 'hbase.tmp.dir', "item": self.validatorNotRootFs(properties,
recommendedDefaults, 'hbase.tmp.dir', host["Hosts"])}])
 
-          dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE")
-          # if METRICS_COLLECTOR is co-hosted with DATANODE
-          if not hbase_rootdir.startswith("hdfs") and \
-            dn_hosts and collectorHostName in dn_hosts:
-            # cross-check dfs.datanode.data.dir and hbase.rootdir
-            # they shouldn't share same disk partition IO
-            hdfs_site = getSiteProperties(configurations, "hdfs-site")
+          if not hbase_rootdir.startswith("hdfs"):
             mountPoints = []
             for mountPoint in host["Hosts"]["disk_info"]:
               mountPoints.append(mountPoint["mountpoint"])
             hbase_rootdir_mountpoint = getMountPointForDir(hbase_rootdir, mountPoints)
-            if ams_site and hdfs_site and "dfs.datanode.data.dir" in hdfs_site:
-              for dfs_datadir in hdfs_site.get("dfs.datanode.data.dir").split(","):
-                mountPoints = []
-                for mountPoint in host["Hosts"]["disk_info"]:
-                  mountPoints.append(mountPoint["mountpoint"])
+            hbase_tmpdir_mountpoint = getMountPointForDir(hbase_tmpdir, mountPoints)
+            preferred_mountpoints = self.getPreferredMountPoints(host['Hosts'])
+            # hbase.rootdir and hbase.tmp.dir shouldn't point to the same partition
+            # if multiple preferred_mountpoints exist
+            if hbase_rootdir_mountpoint == hbase_tmpdir_mountpoint and \
+              len(preferred_mountpoints) > 1:
+              item = self.getWarnItem("Consider not using {0} partition for storing metrics
temporary data. "
+                                      "{0} partition is already used as hbase.rootdir to
store metrics data".format(hbase_tmpdir_mountpoint))
+              validationItems.extend([{"config-name":'hbase.tmp.dir', "item": item}])
+
+            # if METRICS_COLLECTOR is co-hosted with DATANODE
+            # cross-check dfs.datanode.data.dir and hbase.rootdir
+            # they shouldn't share same disk partition IO
+            dn_hosts = self.getComponentHostNames(services, "HDFS", "DATANODE")
+            hdfs_site = getSiteProperties(configurations, "hdfs-site")
+            dfs_datadirs = hdfs_site.get("dfs.datanode.data.dir").split(",") if hdfs_site
and "dfs.datanode.data.dir" in hdfs_site else []
+            if dn_hosts and collectorHostName in dn_hosts and ams_site and \
+              dfs_datadirs and len(preferred_mountpoints) > len(dfs_datadirs):
+              for dfs_datadir in dfs_datadirs:
                 dfs_datadir_mountpoint = getMountPointForDir(dfs_datadir, mountPoints)
                 if dfs_datadir_mountpoint == hbase_rootdir_mountpoint:
                   item = self.getWarnItem("Consider not using {0} partition for storing metrics
data. "
@@ -725,14 +735,26 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
           unusedMemory = host["Hosts"]["total_mem"] * 1024 - requiredMemory # in bytes
           hbase_rootdir = amsHbaseSite.get("hbase.rootdir")
           if unusedMemory > 4294967296:  # warn user, if more than 4GB RAM is unused
-            propertyToIncrease = "hbase_regionserver_heapsize" if hbase_rootdir.startswith("hdfs://")
else "hbase_master_heapsize"
+            heapPropertyToIncrease = "hbase_regionserver_heapsize" if hbase_rootdir.startswith("hdfs://")
else "hbase_master_heapsize"
+            xmnPropertyToIncrease = "regionserver_xmn_size" if hbase_rootdir.startswith("hdfs://")
else "hbase_master_xmn_size"
             collector_heapsize = int((unusedMemory - 4294967296)/5) + to_number(ams_env.get("metrics_collector_heapsize"))*1048576
-            hbase_heapsize = int((unusedMemory - 4294967296)*4/5) + to_number(properties.get(propertyToIncrease))*1048576
+            hbase_heapsize = int((unusedMemory - 4294967296)*4/5) + to_number(properties.get(heapPropertyToIncrease))*1048576
+
+            if hbase_heapsize/1048576 > 2048:
+              xmn_size = '512'
+            elif hbase_heapsize/1048576 > 1024:
+              xmn_size = '256'
+            else:
+              xmn_size = '128'
+
             msg = "{0} MB RAM is unused on the host {1} based on components " \
-                  "assigned. Consider allocating  {2} MB to metrics_collector_heapsize in
ams-env and {3} MB to {4} in ams-hbase-env"
+                  "assigned. Consider allocating  {2} MB to " \
+                  "metrics_collector_heapsize in ams-env, " \
+                  "{3} MB to {4} in ams-hbase-env and " \
+                  "{5} MB to {6} in ams-hbase-env"
 
-            unusedMemoryHbaseItem = self.getWarnItem(msg.format(unusedMemory/1048576, collectorHostName,
collector_heapsize/1048576, hbase_heapsize/1048576, propertyToIncrease))
-            validationItems.extend([{"config-name": propertyToIncrease, "item": unusedMemoryHbaseItem}])
+            unusedMemoryHbaseItem = self.getWarnItem(msg.format(unusedMemory/1048576, collectorHostName,
collector_heapsize/1048576, hbase_heapsize/1048576, heapPropertyToIncrease, xmn_size, xmnPropertyToIncrease))
+            validationItems.extend([{"config-name": heapPropertyToIncrease, "item": unusedMemoryHbaseItem}])
       pass
 
     # Check RS memory in distributed mode since we set default as 512m
@@ -791,11 +813,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     mountPoints.append("/")
     return mountPoints
 
-  def validatorNotRootFs(self, properties, propertyName, hostInfo):
+  def validatorNotRootFs(self, properties, recommendedDefaults, propertyName, hostInfo):
     if not propertyName in properties:
       return self.getErrorItem("Value should be set")
     dir = properties[propertyName]
-    if dir.startswith("hdfs://"):
+    if dir.startswith("hdfs://") or dir == recommendedDefaults.get(propertyName):
       return None
 
     dir = re.sub("^file://", "", dir, count=1)
@@ -1095,7 +1117,7 @@ def getMountPointForDir(dir, mountPoints):
   """
   bestMountFound = None
   if dir:
-    dir = re.sub("^file:///", "", dir, count=1).strip().lower()
+    dir = re.sub("^file://", "", dir, count=1).strip().lower()
 
     # If the path is "/hadoop/hdfs/data", then possible matches for mounts could be
     # "/", "/hadoop/hdfs", and "/hadoop/hdfs/data".
@@ -1170,3 +1192,6 @@ def getMemorySizeRequired(components, configurations):
         totalMemoryRequired += formatXmxSizeToBytes(heapsize)
 
   return totalMemoryRequired
+
+def round_to_n(mem_size, n=128):
+  return int(round(mem_size / float(n))) * int(n)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/87e9ded9/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 2ffa6b8..16ba278 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -33,8 +33,8 @@ class TestHDP206StackAdvisor(TestCase):
     with open(stackAdvisorPath, 'rb') as fp:
       stack_advisor = imp.load_module( 'stack_advisor', fp, stackAdvisorPath, ('.py', 'rb',
imp.PY_SOURCE) )
     with open(hdp206StackAdvisorPath, 'rb') as fp:
-      stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath,
('.py', 'rb', imp.PY_SOURCE))
-    clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName)
+      self.stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath,
('.py', 'rb', imp.PY_SOURCE))
+    clazz = getattr(self.stack_advisor_impl, hdp206StackAdvisorClassName)
     self.stackAdvisor = clazz()
     self.maxDiff = None
     # substitute method in the instance
@@ -894,6 +894,201 @@ class TestHDP206StackAdvisor(TestCase):
     res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults,
configurations, '', '')
     self.assertEquals(res, res_expected)
 
+  def test_validateAmsHbaseSiteConfigurations(self):
+    configurations = {
+      "hdfs-site": {
+        "properties": {
+          'dfs.datanode.data.dir': "/hadoop/data"
+        }
+      },
+      "ams-site": {
+        "properties": {
+          "timeline.metrics.service.operation.mode": "embedded"
+        }
+      }
+    }
+
+    recommendedDefaults = {
+      'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+      'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase',
+      'hbase.cluster.distributed': 'false'
+    }
+    properties = {
+      'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+      'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+      'hbase.cluster.distributed': 'false'
+    }
+    host = {
+      "href" : "/api/v1/hosts/host1",
+      "Hosts" : {
+        "cpu_count" : 1,
+        "host_name" : "host1",
+        "os_arch" : "x86_64",
+        "os_type" : "centos6",
+        "ph_cpu_count" : 1,
+        "public_host_name" : "host1",
+        "rack_info" : "/default-rack",
+        "total_mem" : 2097152,
+        "disk_info": [
+          {
+            "available": str(15<<30), # 15 GB
+            "type": "ext4",
+            "mountpoint": "/"
+          }
+        ]
+      }
+    }
+
+    hosts = {
+      "items" : [
+        host
+      ]
+    }
+
+    services = {
+      "services":  [
+        {
+          "StackServices": {
+            "service_name": "AMBARI_METRICS"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "METRICS_COLLECTOR",
+                "hostnames": ["host1"]
+              }
+            }, {
+              "StackServiceComponents": {
+                "component_name": "METRICS_MONITOR",
+                "hostnames": ["host1"]
+              }
+            }
+          ]
+        },
+        {
+          "StackServices": {
+            "service_name": "HDFS"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "DATANODE",
+                "hostnames": ["host1"]
+              }
+            }
+          ]
+        }
+      ],
+      "configurations": configurations
+    }
+
+    # only 1 partition, enough disk space, no warnings
+    res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults,
configurations, services, hosts)
+    expected = []
+    self.assertEquals(res, expected)
+
+
+    # 1 partition, no enough disk space
+    host['Hosts']['disk_info'] = [
+      {
+        "available" : '1',
+        "type" : "ext4",
+        "mountpoint" : "/"
+      }
+    ]
+    res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults,
configurations, services, hosts)
+    expected = [
+      {'config-name': 'hbase.rootdir',
+       'config-type': 'ams-hbase-site',
+       'level': 'WARN',
+       'message': 'Ambari Metrics disk space requirements not met. '
+                  '\nRecommended disk space for partition / is 10G',
+       'type': 'configuration'
+      }
+    ]
+    self.assertEquals(res, expected)
+
+    # 2 partitions
+    host['Hosts']['disk_info'] = [
+      {
+        "available": str(15<<30), # 15 GB
+        "type" : "ext4",
+        "mountpoint" : "/grid/0"
+      },
+      {
+        "available" : str(15<<30), # 15 GB
+        "type" : "ext4",
+        "mountpoint" : "/"
+      }
+    ]
+    recommendedDefaults = {
+      'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
+      'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase',
+      'hbase.cluster.distributed': 'false'
+    }
+    properties = {
+      'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
+      'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+      'hbase.cluster.distributed': 'false'
+    }
+    res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults,
configurations, services, hosts)
+    expected = []
+    self.assertEquals(res, expected)
+
+    # dfs.dir & hbase.rootdir crosscheck + root partition + hbase.rootdir == hbase.tmp.dir
warnings
+    properties = {
+      'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+      'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+      'hbase.cluster.distributed': 'false'
+    }
+
+    res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults,
configurations, services, hosts)
+    expected = [
+      {
+        'config-name': 'hbase.rootdir',
+        'config-type': 'ams-hbase-site',
+        'level': 'WARN',
+        'message': 'It is not recommended to use root partition for hbase.rootdir',
+        'type': 'configuration'
+      },
+      {
+        'config-name': 'hbase.tmp.dir',
+        'config-type': 'ams-hbase-site',
+        'level': 'WARN',
+        'message': 'Consider not using / partition for storing metrics temporary data. '
+                   '/ partition is already used as hbase.rootdir to store metrics data',
+        'type': 'configuration'
+      },
+      {
+        'config-name': 'hbase.rootdir',
+        'config-type': 'ams-hbase-site',
+        'level': 'WARN',
+        'message': 'Consider not using / partition for storing metrics data. '
+                   '/ is already used by datanode to store HDFS data',
+        'type': 'configuration'
+      }
+    ]
+    self.assertEquals(res, expected)
+
+    # incorrect hbase.rootdir in distributed mode
+    properties = {
+      'hbase.rootdir': 'file:///grid/0/var/lib/ambari-metrics-collector/hbase',
+      'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+      'hbase.cluster.distributed': 'false'
+    }
+    configurations['ams-site']['properties']['timeline.metrics.service.operation.mode'] =
'distributed'
+    res = self.stackAdvisor.validateAmsHbaseSiteConfigurations(properties, recommendedDefaults,
configurations, services, hosts)
+    expected = [
+      {
+        'config-name': 'hbase.rootdir',
+        'config-type': 'ams-hbase-site',
+        'level': 'WARN',
+        'message': 'In distributed mode hbase.rootdir should point to HDFS. Collector will
operate in embedded mode otherwise.',
+        'type': 'configuration'
+      }
+    ]
+    self.assertEquals(res, expected)
+
   def test_getHostsWithComponent(self):
     services = {"services":
                   [{"StackServices":
@@ -1161,8 +1356,9 @@ class TestHDP206StackAdvisor(TestCase):
       }
     ]}
     properties = {"property1": "/var/dir"}
+    recommendedDefaults = {"property1": "/var/dir"}
     # only / mountpoint - no warning
-    self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, 'property1', hostInfo)
== None)
+    self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults,
'property1', hostInfo) == None)
     # More preferable /grid/0 mountpoint - warning
     hostInfo["disk_info"].append(
       {
@@ -1171,7 +1367,8 @@ class TestHDP206StackAdvisor(TestCase):
         "mountpoint" : "/grid/0"
       }
     )
-    warn = self.stackAdvisor.validatorNotRootFs(properties, 'property1', hostInfo)
+    recommendedDefaults = {"property1": "/grid/0/var/dir"}
+    warn = self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults, 'property1',
hostInfo)
     self.assertTrue(warn != None)
     self.assertEquals({'message': 'It is not recommended to use root partition for property1',
'level': 'WARN'}, warn)
 
@@ -1183,4 +1380,17 @@ class TestHDP206StackAdvisor(TestCase):
         "mountpoint" : "/var"
       }
     )
-    self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, 'property1', hostInfo)
== None)
+    self.assertTrue(self.stackAdvisor.validatorNotRootFs(properties, recommendedDefaults,
'property1', hostInfo) == None)
+
+  def test_round_to_n(self):
+    self.assertEquals(self.stack_advisor_impl.round_to_n(0), 0)
+    self.assertEquals(self.stack_advisor_impl.round_to_n(1000), 1024)
+    self.assertEquals(self.stack_advisor_impl.round_to_n(2000), 2048)
+    self.assertEquals(self.stack_advisor_impl.round_to_n(4097), 4096)
+
+  def test_getMountPointForDir(self):
+    self.assertEquals(self.stack_advisor_impl.getMountPointForDir("/var/log", ["/"]), "/")
+    self.assertEquals(self.stack_advisor_impl.getMountPointForDir("/var/log", ["/var", "/"]),
"/var")
+    self.assertEquals(self.stack_advisor_impl.getMountPointForDir("file:///var/log", ["/var",
"/"]), "/var")
+    self.assertEquals(self.stack_advisor_impl.getMountPointForDir("hdfs:///hdfs_path", ["/var",
"/"]), None)
+    self.assertEquals(self.stack_advisor_impl.getMountPointForDir("relative/path", ["/var",
"/"]), None)

http://git-wip-us.apache.org/repos/asf/ambari/blob/87e9ded9/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 4c29dd8..2908d8f 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1977,8 +1977,9 @@ class TestHDP22StackAdvisor(TestCase):
     expected = {
       "ams-hbase-env": {
         "properties": {
-          "hbase_master_heapsize": "540m"
-          }
+          "hbase_master_xmn_size": "128m",
+          "hbase_master_heapsize": "512m"
+        }
       },
       "ams-env": {
         "properties": {
@@ -1993,7 +1994,6 @@ class TestHDP22StackAdvisor(TestCase):
           "hfile.block.cache.size": "0.3",
           "hbase.rootdir": "file:///var/lib/ambari-metrics-collector/hbase",
           "hbase.tmp.dir": "/var/lib/ambari-metrics-collector/hbase-tmp",
-          "hbase_master_xmn_size" : "128m"
         }
       },
       "ams-site": {
@@ -2008,7 +2008,7 @@ class TestHDP22StackAdvisor(TestCase):
     self.assertEquals(configurations, expected)
 
     # 100-nodes cluster, but still only 1 sink (METRICS_COLLECTOR)
-    for i in range(2, 101):
+    for i in range(2, 201):
       hosts['items'].extend([{
         "Hosts": {
           "host_name": "host" + str(i)
@@ -2030,13 +2030,13 @@ class TestHDP22StackAdvisor(TestCase):
           {
             "StackServiceComponents": {
               "component_name": "METRICS_MONITOR",
-              "hostnames": ["host" + str(i) for i in range(1, 101)]
+              "hostnames": ["host" + str(i) for i in range(1, 201)]
             }
           }
         ]
       }
     ]
-    expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '1034m'
+    expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '1408m'
     expected["ams-env"]['properties']['metrics_collector_heapsize'] = '512m'
 
     self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
@@ -2058,7 +2058,7 @@ class TestHDP22StackAdvisor(TestCase):
           {
             "StackServiceComponents": {
               "component_name": "DATANODE",
-              "hostnames": ["host" + str(i) for i in range(1, 101)]
+              "hostnames": ["host" + str(i) for i in range(1, 201)]
             }
           }
         ]
@@ -2077,7 +2077,7 @@ class TestHDP22StackAdvisor(TestCase):
           {
             "StackServiceComponents": {
               "component_name": "NODEMANAGER",
-              "hostnames": ["host" + str(i) for i in range(1, 101)]
+              "hostnames": ["host" + str(i) for i in range(1, 201)]
             }
           }
         ]
@@ -2096,16 +2096,16 @@ class TestHDP22StackAdvisor(TestCase):
           {
             "StackServiceComponents": {
               "component_name": "METRICS_MONITOR",
-              "hostnames": ["host" + str(i) for i in range(1, 101)]
+              "hostnames": ["host" + str(i) for i in range(1, 201)]
             }
           }
         ]
       }
 
     ]
-    expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '1601m'
-    expected["ams-env"]['properties']['metrics_collector_heapsize'] = '512m'
-    # expected["ams-hbase-site"]['properties']['hbase_master_xmn_size'] = '256m'
+    expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '2432m'
+    expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '256m'
+    expected["ams-env"]['properties']['metrics_collector_heapsize'] = '640m'
 
     self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)


Mime
View raw message