ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From avija...@apache.org
Subject ambari git commit: AMBARI-18857 : Add Service wizard: Storm site configs related to Ambari metrics service are not displayed on ui. (avijayan)
Date Mon, 21 Nov 2016 21:00:47 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 41dad5b43 -> ecd57654b


AMBARI-18857 : Add Service wizard: Storm site configs related to Ambari metrics service are
not displayed on ui. (avijayan)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ecd57654
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ecd57654
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ecd57654

Branch: refs/heads/branch-2.5
Commit: ecd57654b23afc812b02c712e30b4d16de47e19a
Parents: 41dad5b
Author: Aravindan Vijayan <avijayan@hortonworks.com>
Authored: Mon Nov 21 13:00:18 2016 -0800
Committer: Aravindan Vijayan <avijayan@hortonworks.com>
Committed: Mon Nov 21 13:00:18 2016 -0800

----------------------------------------------------------------------
 .../0.1.0/configuration/storm-site.xml          | 84 --------------------
 .../AMBARI_METRICS/0.1.0/metainfo.xml           |  4 -
 .../STORM/1.0.1/configuration/storm-site.xml    | 54 +++++++++++++
 .../stacks/HDP/2.5/services/stack_advisor.py    | 42 +++++++++-
 .../stacks/2.5/common/test_stack_advisor.py     | 76 ++++++++++++++++++
 5 files changed, 171 insertions(+), 89 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/ecd57654/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml
deleted file mode 100644
index 7059b90..0000000
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/storm-site.xml
+++ /dev/null
@@ -1,84 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="true">
-  <property>
-    <name>metrics.reporter.register</name>
-    <value>org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter</value>
-    <description>Topology metrics reporter.</description>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
-    <name>storm.cluster.metrics.consumer.register</name>
-    <value>[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"}]</value>
-    <description></description>
-    <value-attributes>
-      <overridable>false</overridable>
-    </value-attributes>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
-    <name>topology.metrics.consumer.register</name>
-    <value>[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink",
"parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency",
"__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count",
"__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$",
"GC/.+\\.count$", "GC/.+\\.timeMs$"]}]</value>
-    <description></description>
-    <value-attributes>
-      <overridable>false</overridable>
-    </value-attributes>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
-    <name>topology.metrics.aggregate.per.worker</name>
-    <value>true</value>
-    <description></description>
-    <value-attributes>
-      <overridable>false</overridable>
-    </value-attributes>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
-    <name>topology.metrics.aggregate.metric.evict.secs</name>
-    <value>5</value>
-    <description></description>
-    <value-attributes>
-      <overridable>false</overridable>
-    </value-attributes>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
-    <name>topology.metrics.expand.map.type</name>
-    <value>true</value>
-    <description></description>
-    <value-attributes>
-      <overridable>false</overridable>
-    </value-attributes>
-    <on-ambari-upgrade add="true"/>
-  </property>
-  <property>
-    <name>topology.metrics.metric.name.separator</name>
-    <value>.</value>
-    <description></description>
-    <value-attributes>
-      <overridable>false</overridable>
-    </value-attributes>
-    <on-ambari-upgrade add="true"/>
-  </property>
-
-</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/ecd57654/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
index d2f5862..43a4320 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
@@ -209,10 +209,6 @@
         <config-type>ams-env</config-type>
       </configuration-dependencies>
 
-      <excluded-config-types>
-        <config-type>storm-site</config-type>
-      </excluded-config-types>
-
       <quickLinksConfigurations>
         <quickLinksConfiguration>
           <fileName>quicklinks.json</fileName>

http://git-wip-us.apache.org/repos/asf/ambari/blob/ecd57654/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-site.xml
b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-site.xml
index 077590a..558beaf 100644
--- a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-site.xml
+++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-site.xml
@@ -117,4 +117,58 @@
     <on-ambari-upgrade add="false"/>
     <deleted>true</deleted>
   </property>
+  <property>
+    <name>storm.cluster.metrics.consumer.register</name>
+    <value>[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"}]</value>
+    <description></description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>topology.metrics.consumer.register</name>
+    <value>[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink",
"parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency",
"__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count",
"__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$",
"GC/.+\\.count$", "GC/.+\\.timeMs$"]}]</value>
+    <description></description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>topology.metrics.aggregate.per.worker</name>
+    <value>true</value>
+    <description></description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>topology.metrics.aggregate.metric.evict.secs</name>
+    <value>5</value>
+    <description></description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>topology.metrics.expand.map.type</name>
+    <value>true</value>
+    <description></description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>topology.metrics.metric.name.separator</name>
+    <value>.</value>
+    <description></description>
+    <value-attributes>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/ecd57654/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index f9a3a9a..8046695 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -111,11 +111,36 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
       "YARN": {"yarn-site": self.validateYarnConfigurations},
       "RANGER": {"ranger-tagsync-site": self.validateRangerTagsyncConfigurations},
       "SPARK2": {"spark2-defaults": self.validateSpark2Defaults,
-                 "spark2-thrift-sparkconf": self.validateSpark2ThriftSparkConf}
+                 "spark2-thrift-sparkconf": self.validateSpark2ThriftSparkConf},
+      "STORM": {"storm-site": self.validateStormConfigurations},
     }
     self.mergeValidators(parentValidators, childValidators)
     return parentValidators
 
+  def validateStormConfigurations(self, properties, recommendedDefaults, configurations,
services, hosts):
+    super(HDP25StackAdvisor, self).validateStormConfigurations(properties, recommendedDefaults,
configurations, services, hosts)
+    validationItems = []
+
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    # Storm AMS integration
+    if 'AMBARI_METRICS' in servicesList:
+      if "storm.cluster.metrics.consumer.register" in properties and \
+          'null' in properties.get("storm.cluster.metrics.consumer.register"):
+
+        validationItems.append({"config-name": 'storm.cluster.metrics.consumer.register',
+                              "item": self.getWarnItem(
+                                "Should be set to recommended value to report metrics to
Ambari Metrics service.")})
+
+      if "topology.metrics.consumer.register" in properties and \
+          'null' in properties.get("topology.metrics.consumer.register"):
+
+        validationItems.append({"config-name": 'topology.metrics.consumer.register',
+                                "item": self.getWarnItem(
+                                  "Should be set to recommended value to report metrics to
Ambari Metrics service.")})
+
+
+    return self.toConfigurationValidationProblems(validationItems, "storm-site")
+
   def validateAtlasConfigurations(self, properties, recommendedDefaults, configurations,
services, hosts):
     application_properties = getSiteProperties(configurations, "application-properties")
     validationItems = []
@@ -449,6 +474,21 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     else:
       putStormSiteAttributes('nimbus.authorizer', 'delete', 'true')
 
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    # Storm AMS integration
+    if 'AMBARI_METRICS' in servicesList:
+      putStormSiteProperty('storm.cluster.metrics.consumer.register', '[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"}]')
+      putStormSiteProperty('topology.metrics.consumer.register',
+                           '[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink",
'
+                           '"parallelism.hint": 1, '
+                           '"whitelist": ["kafkaOffset\\\..+/", "__complete-latency", "__process-latency",
'
+                           '"__receive\\\.population$", "__sendqueue\\\.population$", "__execute-count",
"__emit-count", '
+                           '"__ack-count", "__fail-count", "memory/heap\\\.usedBytes$", "memory/nonHeap\\\.usedBytes$",
'
+                           '"GC/.+\\\.count$", "GC/.+\\\.timeMs$"]}]')
+    else:
+      putStormSiteProperty('storm.cluster.metrics.consumer.register', 'null')
+      putStormSiteProperty('topology.metrics.consumer.register', 'null')
+
   def constructAtlasRestAddress(self, services, hosts):
     """
     :param services: Collection of services in the cluster with configs

http://git-wip-us.apache.org/repos/asf/ambari/blob/ecd57654/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 7124307..7759c09 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -7657,6 +7657,75 @@ class TestHDP25StackAdvisor(TestCase):
     # Test nimbus.authorizer with Ranger Storm plugin disabled in non-kerberos environment
     self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services,
None)
     self.assertEquals(configurations['storm-site']['property_attributes']['nimbus.authorizer'],
{'delete': 'true'}, "Test nimbus.authorizer with Ranger Storm plugin disabled in non-kerberos
environment")
+    self.assertEquals(configurations['storm-site']['properties']['storm.cluster.metrics.consumer.register'],
'null')
+    self.assertEquals(configurations['storm-site']['properties']['topology.metrics.consumer.register'],
'null')
+
+    services = {
+      "services":
+        [
+          {
+            "StackServices": {
+              "service_name" : "STORM",
+              "service_version" : "1.0.1.0.0"
+            }
+          },
+          {
+            "StackServices": {
+              "service_name": "RANGER",
+              "service_version": "0.6.0"
+
+            },
+            "components": [
+              {
+                "StackServiceComponents": {
+                  "component_name": "RANGER_ADMIN",
+                  "hostnames": ["host1"]
+                }
+              }
+            ]
+          },
+          {
+            "StackServices": {
+              "service_name": "AMBARI_METRICS"
+            },
+            "components": [{
+              "StackServiceComponents": {
+                "component_name": "METRICS_COLLECTOR",
+                "hostnames": ["host1"]
+              }
+
+            }, {
+              "StackServiceComponents": {
+                "component_name": "METRICS_MONITOR",
+                "hostnames": ["host1"]
+              }
+
+            }]
+          }
+        ],
+      "Versions": {
+        "stack_version": "2.5"
+      },
+      "configurations": {
+        "storm-site": {
+          "properties": {
+            "nimbus.authorizer" : "org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer",
+            "nimbus.impersonation.acl" :"{{{storm_bare_jaas_principal}} : {hosts: ['*'],
groups: ['*']}}"
+          },
+          "property_attributes": {}
+        },
+        "storm-env": {
+          "properties":{
+            "storm_principal_name": "storm_user@ECAMPLE.COM"
+          },
+        },
+        "ranger-storm-plugin-properties": {
+          "properties": {
+            "ranger-storm-plugin-enabled": "No"
+          }
+        }
+      }
+    }
 
     # Test nimbus.authorizer with Ranger Storm plugin enabled in non-kerberos environment
     configurations['storm-site']['properties'] = {}
@@ -7664,6 +7733,13 @@ class TestHDP25StackAdvisor(TestCase):
     services['configurations']['ranger-storm-plugin-properties']['properties']['ranger-storm-plugin-enabled']
= 'Yes'
     self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services,
None)
     self.assertEquals(configurations['storm-site']['property_attributes']['nimbus.authorizer'],
{'delete': 'true'}, "Test nimbus.authorizer with Ranger Storm plugin enabled in non-kerberos
environment")
+    self.assertEquals(configurations['storm-site']['properties']['storm.cluster.metrics.consumer.register'],
'[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"}]')
+    self.assertEquals(configurations['storm-site']['properties']['topology.metrics.consumer.register'],
'[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink", '
+                                                                                        
             '"parallelism.hint": 1, '
+                                                                                        
             '"whitelist": ["kafkaOffset\\\..+/", "__complete-latency", "__process-latency",
'
+                                                                                        
             '"__receive\\\.population$", "__sendqueue\\\.population$", "__execute-count",
"__emit-count", '
+                                                                                        
             '"__ack-count", "__fail-count", "memory/heap\\\.usedBytes$", "memory/nonHeap\\\.usedBytes$",
'
+                                                                                        
             '"GC/.+\\\.count$", "GC/.+\\\.timeMs$"]}]')
 
     # Test nimbus.authorizer with Ranger Storm plugin being enabled in kerberos environment
     configurations['storm-site']['properties'] = {}


Mime
View raw message