ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-16210. Update Storm Ambari stack definition to work with 1.0 release (Sriharsha Chintalapani via smohanty)
Date Wed, 04 May 2016 22:23:33 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 3f59d882f -> 264111afc


AMBARI-16210. Update Storm Ambari stack definition to work with 1.0 release (Sriharsha Chintalapani
via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/264111af
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/264111af
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/264111af

Branch: refs/heads/trunk
Commit: 264111afc44b311a37b6da64f6cbca42c8622109
Parents: 3f59d88
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Wed May 4 15:23:22 2016 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Wed May 4 15:23:22 2016 -0700

----------------------------------------------------------------------
 .../0.9.1.2.1/configuration/storm-site.xml      |   3 -
 .../0.9.1.2.1/package/scripts/pacemaker.py      | 138 +++++++++++++++++++
 .../STORM/0.9.1.2.1/package/scripts/service.py  |   2 +-
 .../services/STORM/configuration/storm-site.xml |  67 +++++++++
 .../stacks/HDP/2.5/services/STORM/kerberos.json | 120 ++++++++++++++++
 .../stacks/HDP/2.5/services/STORM/metainfo.xml  |   2 +-
 6 files changed, 327 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/264111af/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
index 5a3901a..01b3f4e 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
@@ -737,7 +737,4 @@
       </property>
     </depends-on>
   </property>
-  <property>
-    <name>storm.topology.submission.notifier.plugin.class</name>
-  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/264111af/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/pacemaker.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/pacemaker.py
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/pacemaker.py
new file mode 100644
index 0000000..2ea02e8
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/pacemaker.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management.libraries.functions import check_process_status
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import format
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
+from storm import storm
+from service import service
+from service_check import ServiceCheck
+from resource_management.libraries.functions.security_commons import build_expectations,
\
+    cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties,
\
+    FILE_TYPE_JAAS_CONF
+
+class PaceMaker(Script):
+
+  def get_component_name(self):
+      return "storm-client"
+
+  def install(self, env):
+      self.install_packages(env)
+      self.configure(env)
+
+  def configure(self, env):
+      import params
+      env.set_params(params)
+      storm()
+
+  def pre_upgrade_restart(self, env, upgrade_type=None):
+      import params
+      env.set_params(params)
+
+      if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
+        conf_select.select(params.stack_name, "storm", params.version)
+        stack_select.select("storm-client", params.version)
+
+  def start(self, env, upgrade_type=None):
+      import params
+      env.set_params(params)
+      self.configure(env)
+
+      service("pacemaker", action="start")
+
+  def stop(self, env, upgrade_type=None):
+      import params
+      env.set_params(params)
+
+      service("pacemaker", action="stop")
+
+  def status(self, env):
+      import status_params
+      env.set_params(status_params)
+      check_process_status(status_params.pid_pacemaker)
+
+  def security_status(self, env):
+      import status_params
+
+      env.set_params(status_params)
+
+      if status_params.security_enabled:
+          # Expect the following files to be available in status_params.config_dir:
+          #   storm_jaas.conf
+
+          try:
+              props_value_check = None
+              props_empty_check = ['StormServer/keyTab', 'StormServer/principal']
+              props_read_check = ['StormServer/keyTab']
+              storm_env_expectations = build_expectations('storm_jaas', props_value_check,
props_empty_check,
+                                                          props_read_check)
+
+              storm_expectations = {}
+              storm_expectations.update(storm_env_expectations)
+
+              security_params = get_params_from_filesystem(status_params.conf_dir,
+                                                     {'storm_jaas.conf': FILE_TYPE_JAAS_CONF})
+
+              result_issues = validate_security_config_properties(security_params, storm_expectations)
+              if not result_issues:  # If all validations passed successfully
+                  # Double check the dict before calling execute
+                  if ( 'storm_jaas' not in security_params
+                       or 'StormServer' not in security_params['storm_jaas']
+                       or 'keyTab' not in security_params['storm_jaas']['StormServer']
+                       or 'principal' not in security_params['storm_jaas']['StormServer']):
+                      self.put_structured_out({"securityState": "ERROR"})
+                      self.put_structured_out({"securityIssuesFound": "Keytab file or principal
are not set property."})
+                      return
+
+                  cached_kinit_executor(status_params.kinit_path_local,
+                                        status_params.storm_user,
+                                        security_params['storm_jaas']['StormServer']['keyTab'],
+                                        security_params['storm_jaas']['StormServer']['principal'],
+                                        status_params.hostname,
+                                        status_params.tmp_dir)
+                  self.put_structured_out({"securityState": "SECURED_KERBEROS"})
+              else:
+                  issues = []
+                  for cf in result_issues:
+                      issues.append("Configuration file %s did not pass the validation. Reason:
%s" % (cf, result_issues[cf]))
+                      self.put_structured_out({"securityIssuesFound": ". ".join(issues)})
+                      self.put_structured_out({"securityState": "UNSECURED"})
+          except Exception as e:
+              self.put_structured_out({"securityState": "ERROR"})
+              self.put_structured_out({"securityStateErrorInfo": str(e)})
+      else:
+        self.put_structured_out({"securityState": "UNSECURED"})
+
+  def get_log_folder(self):
+      import params
+      return params.log_dir
+
+  def get_user(self):
+      import params
+      return params.storm_user
+
+if __name__ == "__main__":
+    PaceMaker().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/264111af/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/service.py
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/service.py
index d0b717f..6da159a 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/service.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/service.py
@@ -39,7 +39,7 @@ def service(name, action = 'start'):
   tries_count = 12
 
   if name == 'ui':
-    process_grep = "backtype.storm.ui.core$"
+    process_grep = "storm.ui.core$"
   elif name == "rest_api":
     process_grep = format("{rest_lib_dir}/storm-rest-.*\.jar$")
   else:

http://git-wip-us.apache.org/repos/asf/ambari/blob/264111af/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/configuration/storm-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/configuration/storm-site.xml
new file mode 100644
index 0000000..19d4961
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/configuration/storm-site.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+  <property>
+    <name>storm.thrift.transport</name>
+    <value>org.apache.storm.security.auth.SimpleTransportPlugin</value>
+    <description>The transport plug-in for Thrift client/server communication.</description>
+  </property>
+  <property>
+    <name>storm.messaging.transport</name>
+    <value>org.apache.storm.messaging.netty.Context</value>
+    <description>The transporter for communication among Storm tasks.</description>
+  </property>
+  <property>
+    <name>nimbus.topology.validator</name>
+    <value>org.apache.storm.nimbus.DefaultTopologyValidator</value>
+    <description>A custom class that implements ITopologyValidator that is run whenever
a
+       topology is submitted. Can be used to provide business-specific logic for
+       whether topologies are allowed to run or not.</description>
+  </property>
+  <property>
+    <name>topology.spout.wait.strategy</name>
+    <value>org.apache.storm.spout.SleepSpoutWaitStrategy</value>
+    <description>A class that implements a strategy for what to do when a spout needs
to wait. Waiting is
+       triggered in one of two conditions:
+
+       1. nextTuple emits no tuples
+       2. The spout has hit maxSpoutPending and can't emit any more tuples</description>
+  </property>
+  <property>
+    <name>topology.kryo.factory</name>
+    <value>org.apache.storm.serialization.DefaultKryoFactory</value>
+    <description>Class that specifies how to create a Kryo instance for serialization.
Storm will then apply
+       topology.kryo.register and topology.kryo.decorators on top of this. The default implementation
+       implements topology.fall.back.on.java.serialization and turns references off.</description>
+  </property>
+  <property>
+    <name>topology.tuple.serializer</name>
+    <value>org.apache.storm.serialization.types.ListDelegateSerializer</value>
+    <description>The serializer class for ListDelegate (tuple payload).
+       The default serializer will be ListDelegateSerializer</description>
+  </property>
+  <property>
+    <name>client.jartransformer.class</name>
+    <description>Storm Topology backward comptability transformer</description>
+    <value>org.apache.storm.hack.StormShadeTransformer</value>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/264111af/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/kerberos.json
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/kerberos.json
new file mode 100644
index 0000000..b88dd44
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/kerberos.json
@@ -0,0 +1,120 @@
+{
+  "services": [
+    {
+      "name": "STORM",
+      "identities": [
+        {
+          "name": "/spnego"
+        },
+        {
+          "name": "/smokeuser"
+        },
+        {
+          "name": "storm_components",
+          "principal": {
+            "value": "${storm-env/storm_user}-${cluster_name|toLower()}@${realm}",
+            "type": "user",
+            "configuration": "storm-env/storm_principal_name"
+          },
+          "keytab": {
+            "file": "${keytab_dir}/storm.headless.keytab",
+            "owner": {
+              "name": "${storm-env/storm_user}",
+              "access": "r"
+            },
+            "group": {
+              "name": "${cluster-env/user_group}",
+              "access": ""
+            },
+            "configuration": "storm-env/storm_keytab"
+          }
+        }
+      ],
+      "configurations": [
+        {
+          "storm-site": {
+            "nimbus.authorizer": "org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer",
+            "drpc.authorizer": "org.apache.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer",
+            "ui.filter": "org.apache.hadoop.security.authentication.server.AuthenticationFilter",
+            "storm.principal.tolocal": "org.apache.storm.security.auth.KerberosPrincipalToLocal",
+            "supervisor.enable": "true",
+            "storm.zookeeper.superACL": "sasl:{{storm_bare_jaas_principal}}",
+            "java.security.auth.login.config": "{{conf_dir}}/storm_jaas.conf",
+            "nimbus.admins": "['{{storm_bare_jaas_principal}}']",
+            "nimbus.supervisor.users": "['{{storm_bare_jaas_principal}}']",
+            "ui.filter.params": "{'type': 'kerberos', 'kerberos.principal': '{{storm_ui_jaas_principal}}',
'kerberos.keytab': '{{storm_ui_keytab_path}}', 'kerberos.name.rules': 'DEFAULT'}"
+          }
+        }
+      ],
+      "components": [
+        {
+          "name": "STORM_UI_SERVER",
+          "identities": [
+            {
+              "name": "/spnego",
+              "principal": {
+                "configuration": "storm-env/storm_ui_principal_name"
+              },
+              "keytab": {
+                "configuration": "storm-env/storm_ui_keytab"
+              }
+            }
+          ]
+        },
+        {
+          "name": "NIMBUS",
+          "identities": [
+            {
+              "name": "nimbus_server",
+              "principal": {
+                "value": "nimbus/_HOST@${realm}",
+                "type": "service",
+                "configuration": "storm-env/nimbus_principal_name"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/nimbus.service.keytab",
+                "owner": {
+                  "name": "${storm-env/storm_user}",
+                  "access": "r"
+                },
+                "group": {
+                  "name": "${cluster-env/user_group}",
+                  "access": ""
+                },
+                "configuration": "storm-env/nimbus_keytab"
+              }
+            }
+          ]
+        },
+        {
+          "name": "DRPC_SERVER",
+          "identities": [
+            {
+              "name": "nimbus_server",
+              "principal": {
+                "value": "nimbus/_HOST@${realm}",
+                "type": "service",
+                "configuration": "storm-env/nimbus_principal_name"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/nimbus.service.keytab",
+                "owner": {
+                  "name": "${storm-env/storm_user}",
+                  "access": "r"
+                },
+                "group": {
+                  "name": "${cluster-env/user_group}",
+                  "access": ""
+                },
+                "configuration": "storm-env/nimbus_keytab"
+              }
+            }
+          ]
+        },
+        {
+          "name" : "SUPERVISOR"
+        }
+      ]
+    }
+  ]
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/264111af/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/metainfo.xml
index a296bfa..c1e9490 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/STORM/metainfo.xml
@@ -21,7 +21,7 @@
   <services>
     <service>
       <name>STORM</name>
-      <version>0.10.0.2.5</version>
+      <version>1.0.1.2.5</version>
     </service>
   </services>
 </metainfo>


Mime
View raw message