ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [44/50] [abbrv] ambari git commit: AMBARI-19309 Stack advisor doesn't recommend distributed mode for AMS HA clusters (dsen)
Date Thu, 05 Jan 2017 00:05:25 GMT
AMBARI-19309 Stack advisor doesn't recommend distributed mode for AMS HA clusters (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/30b27b1c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/30b27b1c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/30b27b1c

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 30b27b1c2744b8095970c59235ef31a4cea900c0
Parents: 8f16d64
Author: Dmytro Sen <dsen@apache.org>
Authored: Wed Jan 4 18:04:34 2017 +0200
Committer: Dmytro Sen <dsen@apache.org>
Committed: Wed Jan 4 18:04:34 2017 +0200

----------------------------------------------------------------------
 .../stacks/HDP/2.0.6/services/stack_advisor.py  |   7 +-
 .../stacks/2.0.6/common/test_stack_advisor.py   | 346 +++++++++++++++++++
 2 files changed, 352 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/30b27b1c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 7736301..b85ddf8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -736,6 +736,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       if "timeline.metrics.service.operation.mode" in services["configurations"]["ams-site"]["properties"]:
         operatingMode = services["configurations"]["ams-site"]["properties"]["timeline.metrics.service.operation.mode"]
 
+    if len(amsCollectorHosts) > 1 :
+      operatingMode = "distributed"
+      putAmsSiteProperty("timeline.metrics.service.operation.mode", operatingMode)
+
     if operatingMode == "distributed":
       putAmsSiteProperty("timeline.metrics.service.watcher.disabled", 'true')
       putAmsHbaseSiteProperty("hbase.cluster.distributed", 'true')
@@ -1126,7 +1130,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     if op_mode not in ("embedded", "distributed"):
       correct_op_mode_item = self.getErrorItem("Correct value should be set.")
       pass
-
+    elif len(self.getComponentHostNames(services, "AMBARI_METRICS", "METRICS_COLLECTOR"))
> 1 and op_mode != 'distributed':
+      correct_op_mode_item = self.getErrorItem("Correct value should be 'distributed' for
clusters with more then 1 Metrics collector")
     validationItems.extend([{"config-name":'timeline.metrics.service.operation.mode', "item":
correct_op_mode_item }])
     return self.toConfigurationValidationProblems(validationItems, "ams-site")
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/30b27b1c/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index ac3028e..ca697c0 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -1514,6 +1514,190 @@ class TestHDP206StackAdvisor(TestCase):
     self.assertEquals(configurations, expected)
 
 
+  def test_recommendAmsConfigurations(self):
+    configurations = {
+      "hadoop-env": {
+        "properties": {
+          "hdfs_user": "hdfs",
+          "proxyuser_group": "users"
+        }
+      }
+    }
+
+    hosts = {
+      "items": [
+        {
+          "href": "/api/v1/hosts/host1",
+          "Hosts": {
+            "cpu_count": 1,
+            "host_name": "c6401.ambari.apache.org",
+            "os_arch": "x86_64",
+            "os_type": "centos6",
+            "ph_cpu_count": 1,
+            "public_host_name": "public.c6401.ambari.apache.org",
+            "rack_info": "/default-rack",
+            "total_mem": 2097152,
+            "disk_info": [{
+              "size": '80000000',
+              "mountpoint": "/"
+            }]
+          }
+        },
+        {
+          "href": "/api/v1/hosts/host2",
+          "Hosts": {
+            "cpu_count": 1,
+            "host_name": "c6402.ambari.apache.org",
+            "os_arch": "x86_64",
+            "os_type": "centos6",
+            "ph_cpu_count": 1,
+            "public_host_name": "public.c6402.ambari.apache.org",
+            "rack_info": "/default-rack",
+            "total_mem": 1048576,
+            "disk_info": [{
+              "size": '800000000',
+              "mountpoint": "/"
+            }]
+          }
+        }
+      ]}
+
+
+    services1 = {
+      "services": [
+        {
+          "StackServices": {
+            "service_name": "HDFS"
+          }, "components": [
+          {
+            "StackServiceComponents": {
+              "component_name": "NAMENODE",
+              "hostnames": ["c6401.ambari.apache.org"]
+            }
+          }
+        ]
+        },
+        {
+          "StackServices": {
+            "service_name": "AMBARI_METRICS"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "METRICS_COLLECTOR",
+                "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
+              }
+            }, {
+              "StackServiceComponents": {
+                "component_name": "METRICS_MONITOR",
+                "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
+              }
+            }
+          ]
+        }],
+      "configurations": configurations,
+      "ambari-server-properties": {"ambari-server.user":"ambari_user"}
+    }
+
+    clusterData = {
+      "totalAvailableRam": 2048
+    }
+
+    expected = {'ams-env': {'properties': {'metrics_collector_heapsize': '512'}},
+                  'ams-grafana-env': {'properties': {},
+                                                             'property_attributes': {'metrics_grafana_password':
{'visible': 'false'}}},
+                  'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector',
+                                                                                       'hbase_master_heapsize':
'512',
+                                                                                       'hbase_master_xmn_size':
'102',
+                                                                                       'hbase_regionserver_heapsize':
'768',
+                                                                                       'regionserver_xmn_size':
'128'}},
+                  'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true',
+                                                                                        
'hbase.hregion.memstore.flush.size': '134217728',
+                                                                                        
'hbase.regionserver.global.memstore.lowerLimit': '0.3',
+                                                                                        
'hbase.regionserver.global.memstore.upperLimit': '0.35',
+                                                                                        
'hbase.rootdir': '/user/ams/hbase',
+                                                                                        
'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase-tmp',
+                                                                                        
'hbase.zookeeper.property.clientPort': '2181',
+                                                                                        
'hfile.block.cache.size': '0.3'}},
+                  'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10',
+                                                                             'timeline.metrics.cache.size':
'100',
+                                                                             'timeline.metrics.cluster.aggregate.splitpoints':
'load_one',
+                                                                             'timeline.metrics.host.aggregate.splitpoints':
'load_one',
+                                                                             'timeline.metrics.service.handler.thread.count':
'20',
+                                                                             'timeline.metrics.service.operation.mode':
'distributed',
+                                                                             'timeline.metrics.service.watcher.disabled':
'true',
+                                                                             'timeline.metrics.service.webapp.address':
'0.0.0.0:6188'}},
+                  'hadoop-env': {'properties': {'hdfs_user': 'hdfs',
+                                                                                 'proxyuser_group':
'users'}}}
+
+    self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services1,
hosts)
+    self.assertEquals(configurations, expected)
+
+    services1 = {
+      "services": [
+        {
+          "StackServices": {
+            "service_name": "HDFS"
+          }, "components": [
+          {
+            "StackServiceComponents": {
+              "component_name": "NAMENODE",
+              "hostnames": ["c6401.ambari.apache.org"]
+            }
+          }
+        ]
+        },
+        {
+          "StackServices": {
+            "service_name": "AMBARI_METRICS"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "METRICS_COLLECTOR",
+                "hostnames": ["c6401.ambari.apache.org"]
+              }
+            }, {
+              "StackServiceComponents": {
+                "component_name": "METRICS_MONITOR",
+                "hostnames": ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
+              }
+            }
+          ]
+        }],
+      "configurations": configurations,
+      "ambari-server-properties": {"ambari-server.user":"ambari_user"}
+    }
+    expected = {'ams-env': {'properties': {'metrics_collector_heapsize': '512'}},
+                  'ams-grafana-env': {'properties': {},
+                                                             'property_attributes': {'metrics_grafana_password':
{'visible': 'false'}}},
+                  'ams-hbase-env': {'properties': {'hbase_log_dir': '/var/log/ambari-metrics-collector',
+                                                                                       'hbase_master_heapsize':
'512',
+                                                                                       'hbase_master_xmn_size':
'102',
+                                                                                       'hbase_regionserver_heapsize':
'768',
+                                                                                       'regionserver_xmn_size':
'128'}},
+                  'ams-hbase-site': {'properties': {'hbase.cluster.distributed': 'true',
+                                                                                        
'hbase.hregion.memstore.flush.size': '134217728',
+                                                                                        
'hbase.regionserver.global.memstore.lowerLimit': '0.3',
+                                                                                        
'hbase.regionserver.global.memstore.upperLimit': '0.35',
+                                                                                        
'hbase.rootdir': '/user/ams/hbase',
+                                                                                        
'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase-tmp',
+                                                                                        
'hbase.zookeeper.property.clientPort': '2181',
+                                                                                        
'hfile.block.cache.size': '0.3',
+                                                                                        
'phoenix.coprocessor.maxMetaDataCacheSize': '20480000'}},
+                  'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10',
+                                                                             'timeline.metrics.cache.size':
'100',
+                                                                             'timeline.metrics.cluster.aggregate.splitpoints':
'load_one',
+                                                                             'timeline.metrics.host.aggregate.splitpoints':
'load_one',
+                                                                             'timeline.metrics.service.handler.thread.count':
'20',
+                                                                             'timeline.metrics.service.operation.mode':
'distributed',
+                                                                             'timeline.metrics.service.watcher.disabled':
'true',
+                                                                             'timeline.metrics.service.webapp.address':
'0.0.0.0:6188'}},
+                  'hadoop-env': {'properties': {'hdfs_user': 'hdfs',
+                                                                                 'proxyuser_group':
'users'}}}
+    self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services1,
hosts)
+    self.assertEquals(configurations, expected)
+
   def test_getHostNamesWithComponent(self):
 
     services = {
@@ -1923,6 +2107,168 @@ class TestHDP206StackAdvisor(TestCase):
     validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults,
configurations, services, hosts)
     self.assertEquals(validation_problems, expected)
 
+  def test_validateAmsSiteConfigurations(self):
+    configurations = {
+      "hdfs-site": {
+        "properties": {
+          'dfs.datanode.data.dir': "/hadoop/data"
+        }
+      },
+      "core-site": {
+        "properties": {
+          "fs.defaultFS": "hdfs://c6401.ambari.apache.org:8020"
+        }
+      },
+      "ams-site": {
+        "properties": {
+          "timeline.metrics.service.operation.mode": "embedded"
+        }
+      }
+    }
+    recommendedDefaults = {
+      'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+      'hbase.tmp.dir': '/var/lib/ambari-metrics-collector/hbase',
+      'hbase.cluster.distributed': 'false'
+    }
+    properties = {
+      'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase',
+      'hbase.tmp.dir' : '/var/lib/ambari-metrics-collector/hbase',
+      'hbase.cluster.distributed': 'false',
+      'timeline.metrics.service.operation.mode' : 'embedded'
+    }
+    host1 = {
+      "href" : "/api/v1/hosts/host1",
+      "Hosts" : {
+        "cpu_count" : 1,
+        "host_name" : "host1",
+        "os_arch" : "x86_64",
+        "os_type" : "centos6",
+        "ph_cpu_count" : 1,
+        "public_host_name" : "host1",
+        "rack_info" : "/default-rack",
+        "total_mem" : 2097152,
+        "disk_info": [
+          {
+            "available": str(15<<30), # 15 GB
+            "type": "ext4",
+            "mountpoint": "/"
+          }
+        ]
+      }
+    }
+    host2 = {
+      "href" : "/api/v1/hosts/host2",
+      "Hosts" : {
+        "cpu_count" : 1,
+        "host_name" : "host2",
+        "os_arch" : "x86_64",
+        "os_type" : "centos6",
+        "ph_cpu_count" : 1,
+        "public_host_name" : "host2",
+        "rack_info" : "/default-rack",
+        "total_mem" : 2097152,
+        "disk_info": [
+          {
+            "available": str(15<<30), # 15 GB
+            "type": "ext4",
+            "mountpoint": "/"
+          }
+        ]
+      }
+    }
+
+    hosts = {
+      "items" : [
+        host1, host2
+      ]
+    }
+
+    services = {
+      "services":  [
+        {
+          "StackServices": {
+            "service_name": "AMBARI_METRICS"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "METRICS_COLLECTOR",
+                "hostnames": ["host1", "host2"]
+              }
+            }, {
+              "StackServiceComponents": {
+                "component_name": "METRICS_MONITOR",
+                "hostnames": ["host1", "host2"]
+              }
+            }
+          ]
+        },
+        {
+          "StackServices": {
+            "service_name": "HDFS"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "DATANODE",
+                "hostnames": ["host1"]
+              }
+            }
+          ]
+        }
+      ],
+      "configurations": configurations
+    }
+    # only 1 partition, enough disk space, no warnings
+    res = self.stackAdvisor.validateAmsSiteConfigurations(properties, recommendedDefaults,
configurations, services, hosts)
+    expected = [{'config-name': 'timeline.metrics.service.operation.mode',
+                    'config-type': 'ams-site',
+                    'level': 'ERROR',
+                    'message': "Correct value should be 'distributed' for clusters with more
then 1 Metrics collector",
+                    'type': 'configuration'}]
+    self.assertEquals(res, expected)
+
+
+    services = {
+      "services":  [
+        {
+          "StackServices": {
+            "service_name": "AMBARI_METRICS"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "METRICS_COLLECTOR",
+                "hostnames": ["host1"]
+              }
+            }, {
+              "StackServiceComponents": {
+                "component_name": "METRICS_MONITOR",
+                "hostnames": ["host1"]
+              }
+            }
+          ]
+        },
+        {
+          "StackServices": {
+            "service_name": "HDFS"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "DATANODE",
+                "hostnames": ["host1"]
+              }
+            }
+          ]
+        }
+      ],
+      "configurations": configurations
+    }
+    res = self.stackAdvisor.validateAmsSiteConfigurations(properties, recommendedDefaults,
configurations, services, hosts)
+    expected = []
+    self.assertEquals(res, expected)
+
   def test_validateAmsHbaseSiteConfigurations(self):
     configurations = {
       "hdfs-site": {


Mime
View raw message