ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dmitriu...@apache.org
Subject [2/2] ambari git commit: AMBARI-18321. atlas hook for hive and storm fail to push metadeta (dgrinenko via dlysnichenko)
Date Wed, 07 Sep 2016 16:53:07 GMT
AMBARI-18321. atlas hook for hive and storm fail to push metadeta (dgrinenko via dlysnichenko)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/53503942
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/53503942
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/53503942

Branch: refs/heads/branch-2.4
Commit: 5350394220ce6a84c351f0afa633d3a0cf7e179e
Parents: 05d56de
Author: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Authored: Wed Sep 7 19:52:33 2016 +0300
Committer: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Committed: Wed Sep 7 19:52:33 2016 +0300

----------------------------------------------------------------------
 .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 24 +++++++++++-
 .../package/scripts/metadata_server.py          | 18 ++++++++-
 .../ATLAS/0.1.0.2.3/package/scripts/params.py   | 38 ++++++++++++++++++
 .../0.1.0.2.3/package/scripts/status_params.py  |  1 +
 .../package/templates/atlas_kafka_acl.sh.j2     |  9 +++++
 .../package/templates/kafka_jaas.conf.j2        | 41 ++++++++++++++++++++
 .../ATLAS/0.7.0.2.5/kerberos.json               |  3 ++
 .../package/scripts/hbase_regionserver.py       |  6 ---
 .../stacks/HDP/2.5/services/ATLAS/metainfo.xml  |  6 +++
 .../src/test/python/TestAmbariServer.py         | 15 +++++--
 10 files changed, 149 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
index 6ec636c..b16f597 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
@@ -17,13 +17,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
-
+from resource_management import Package
+from resource_management import StackFeature
 from resource_management.core.resources.system import Directory, File
 from resource_management.core.source import StaticFile, InlineTemplate, Template
 from resource_management.core.exceptions import Fail
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.decorator import retry
 from resource_management.libraries.functions import solr_cloud_util
+from resource_management.libraries.functions.stack_features import check_stack_feature, get_stack_feature_version
 from resource_management.libraries.resources.properties_file import PropertiesFile
 from resource_management.libraries.resources.template_config import TemplateConfig
 
@@ -128,6 +130,26 @@ def metadata(type='server'):
          content=Template("atlas_hbase_setup.rb.j2")
     )
 
+    if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, get_stack_feature_version(params.config))
and\
+      params.security_enabled and not params.host_with_kafka:
+
+      File(params.atlas_kafka_setup,
+           group=params.user_group,
+           owner=params.kafka_user,
+           content=Template("atlas_kafka_acl.sh.j2")
+      )
+
+      File(format("{kafka_conf_dir}/kafka-env.sh"),
+           owner=params.kafka_user,
+           content=InlineTemplate(params.kafka_env_sh_template)
+           )
+
+      File(format("{kafka_conf_dir}/kafka_jaas.conf"),
+           group=params.user_group,
+           owner=params.kafka_user,
+           content=Template("kafka_jaas.conf.j2")
+           )
+
 def upload_conf_set(config_set, jaasFile):
   import params
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
index ed0314b..712c15c 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
@@ -17,6 +17,7 @@ limitations under the License.
 
 """
 from metadata import metadata
+from resource_management import Fail
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
 from resource_management import Execute, File, check_process_status, Script, format_stack_version
@@ -25,7 +26,7 @@ from resource_management.libraries.functions.security_commons import build_expec
   get_params_from_filesystem, validate_security_config_properties, \
   FILE_TYPE_PROPERTIES
 from resource_management.libraries.functions.show_logs import show_logs
-from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.stack_features import check_stack_feature, get_stack_feature_version
 from resource_management.libraries.functions import StackFeature
 from resource_management.core.resources.system import Directory
 from resource_management.core.logger import Logger
@@ -69,7 +70,10 @@ class MetadataServer(Script):
     daemon_cmd = format('source {params.conf_dir}/atlas-env.sh ; {params.metadata_start_script}')
     no_op_test = format('ls {params.pid_file} >/dev/null 2>&1 && ps -p
`cat {params.pid_file}` >/dev/null 2>&1')
     atlas_hbase_setup_command = format("cat {atlas_hbase_setup} | hbase shell -n")
+    atlas_kafka_setup_command = format("bash {atlas_kafka_setup}")
     secure_atlas_hbase_setup_command = format("kinit -kt {hbase_user_keytab} {hbase_principal_name};
") + atlas_hbase_setup_command
+    # in case if principal was distributed across several hosts, pattern need to be replaced
to right one
+    secure_atlas_kafka_setup_command = format("kinit -kt {kafka_keytab} {kafka_principal_name};
").replace("_HOST", params.hostname) + atlas_kafka_setup_command
 
     if params.stack_supports_atlas_ranger_plugin:
       Logger.info('Atlas plugin is enabled, configuring Atlas plugin.')
@@ -78,7 +82,7 @@ class MetadataServer(Script):
       Logger.info('Atlas plugin is not supported or enabled.')
 
     try:
-      effective_version = format_stack_version(params.version) if upgrade_type is not None
else params.stack_version_formatted
+      effective_version = get_stack_feature_version(params.config)
 
       if check_stack_feature(StackFeature.ATLAS_HBASE_SETUP, effective_version):
         if params.security_enabled and params.has_hbase_master:
@@ -94,6 +98,16 @@ class MetadataServer(Script):
                   user=params.hbase_user
           )
 
+      if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, effective_version) and params.security_enabled:
+        try:
+          Execute(secure_atlas_kafka_setup_command,
+                  user=params.kafka_user,
+                  tries=5,
+                  try_sleep=10
+          )
+        except Fail:
+          pass  # do nothing and do not block Atlas start, fail logs would be available via
Execute internals
+
       Execute(daemon_cmd,
               user=params.metadata_user,
               not_if=no_op_test

http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 028a828..45e323c 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -22,6 +22,7 @@ import os
 import sys
 
 # Local Imports
+from resource_management import get_bare_principal
 from status_params import *
 from resource_management import format_stack_version, Script
 from resource_management.libraries.functions import format
@@ -233,6 +234,7 @@ ranger_admin_hosts = default('/clusterHostInfo/ranger_admin_hosts', [])
 has_ranger_admin = not len(ranger_admin_hosts) == 0
 
 atlas_hbase_setup = format("{exec_tmp_dir}/atlas_hbase_setup.rb")
+atlas_kafka_setup = format("{exec_tmp_dir}/atlas_kafka_acl.sh")
 atlas_graph_storage_hbase_table = default('/configurations/application-properties/atlas.graph.storage.hbase.table',
None)
 atlas_audit_hbase_tablename = default('/configurations/application-properties/atlas.audit.hbase.tablename',
None)
 
@@ -240,6 +242,42 @@ hbase_user_keytab = default('/configurations/hbase-env/hbase_user_keytab',
None)
 hbase_principal_name = default('/configurations/hbase-env/hbase_principal_name', None)
 enable_ranger_hbase = False
 
+# ToDo: Kafka port to Atlas
+# Used while upgrading the stack in a kerberized cluster and running kafka-acls.sh
+hosts_with_kafka = default('/clusterHostInfo/kafka_broker_hosts', [])
+host_with_kafka = hostname in hosts_with_kafka
+
+ranger_tagsync_hosts = default("/clusterHostInfo/ranger_tagsync_hosts", [])
+has_ranger_tagsync = len(ranger_tagsync_hosts) > 0
+ranger_user = default('/configurations/ranger-env/ranger_user', None)
+
+kafka_keytab = default('/configurations/kafka-env/kafka_keytab', None)
+kafka_principal_name = default('/configurations/kafka-env/kafka_principal_name', None)
+
+if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, version_for_stack_feature_checks):
+  default_replication_factor = default('/configurations/application-properties/atlas.notification.replicas',
None)
+
+  kafka_env_sh_template = config['configurations']['kafka-env']['content']
+  kafka_home = os.path.join(stack_root,  "current", "kafka-broker")
+  kafka_conf_dir = os.path.join(kafka_home, "config")
+
+  kafka_zk_endpoint = default("/configurations/kafka-broker/zookeeper.connect", None)
+  kafka_kerberos_enabled = (('security.inter.broker.protocol' in config['configurations']['kafka-broker'])
and
+                            ((config['configurations']['kafka-broker']['security.inter.broker.protocol']
== "PLAINTEXTSASL") or
+                             (config['configurations']['kafka-broker']['security.inter.broker.protocol']
== "SASL_PLAINTEXT")))
+  if security_enabled and stack_version_formatted != "" and 'kafka_principal_name' in config['configurations']['kafka-env']
\
+    and check_stack_feature(StackFeature.KAFKA_KERBEROS, stack_version_formatted):
+    _hostname_lowercase = config['hostname'].lower()
+    _kafka_principal_name = config['configurations']['kafka-env']['kafka_principal_name']
+    kafka_jaas_principal = _kafka_principal_name.replace('_HOST', _hostname_lowercase)
+    kafka_keytab_path = config['configurations']['kafka-env']['kafka_keytab']
+    kafka_bare_jaas_principal = get_bare_principal(_kafka_principal_name)
+    kafka_kerberos_params = "-Djava.security.auth.login.config={0}/kafka_jaas.conf".format(kafka_conf_dir)
+  else:
+    kafka_kerberos_params = ''
+    kafka_jaas_principal = None
+    kafka_keytab_path = None
+
 if has_ranger_admin and stack_supports_atlas_ranger_plugin:
   # for create_hdfs_directory
   namenode_host = set(default("/clusterHostInfo/namenode_host", []))

http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
index 0f0d89c..3cf81ca 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
@@ -46,6 +46,7 @@ pid_file = format("{pid_dir}/atlas.pid")
 
 metadata_user = default("/configurations/atlas-env/metadata_user", None)
 hbase_user = default("/configurations/hbase-env/hbase_user", None)
+kafka_user = default("/configurations/kafka-env/kafka_user", None)
 
 # Security related/required params
 hostname = config['hostname']

http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2
new file mode 100644
index 0000000..9e77ac4
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2
@@ -0,0 +1,9 @@
+#!/usr/bin/bash
+
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}}
--add  --topic ATLAS_HOOK --allow-principal User:* --producer
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}}
--add  --topic ATLAS_HOOK --allow-principal User:{{metadata_user}} --consumer --group atlas
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}}
--add  --topic ATLAS_ENTITIES --allow-principal User:{{metadata_user}} --producer
+
+{% if has_ranger_tagsync %}
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}}
--add  --topic ATLAS_ENTITIES --allow-principal User:{{ranger_user}} --consumer --group ranger_entities_consumer
+{% endif %}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2
b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2
new file mode 100644
index 0000000..56c558d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2
@@ -0,0 +1,41 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+KafkaServer {
+   com.sun.security.auth.module.Krb5LoginModule required
+   useKeyTab=true
+   keyTab="{{kafka_keytab_path}}"
+   storeKey=true
+   useTicketCache=false
+   serviceName="{{kafka_bare_jaas_principal}}"
+   principal="{{kafka_jaas_principal}}";
+};
+KafkaClient {
+   com.sun.security.auth.module.Krb5LoginModule required
+   useTicketCache=true
+   renewTicket=true
+   serviceName="{{kafka_bare_jaas_principal}}";
+};
+Client {
+   com.sun.security.auth.module.Krb5LoginModule required
+   useKeyTab=true
+   keyTab="{{kafka_keytab_path}}"
+   storeKey=true
+   useTicketCache=false
+   serviceName="zookeeper"
+   principal="{{kafka_jaas_principal}}";
+};

http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
index a9da650..bc8e351 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
@@ -84,6 +84,9 @@
               "keytab": {
                 "configuration": "ranger-atlas-audit/xasecure.audit.jaas.Client.option.keyTab"
               }
+            },
+            {
+              "name": "/KAFKA/KAFKA_BROKER/kafka_broker"
             }
           ]
         }

http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
index cf65718..370167b 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
@@ -83,10 +83,6 @@ class HbaseRegionServerDefault(HbaseRegionServer):
     env.set_params(params)
     upgrade.post_regionserver(env)
 
-  def post_start(self, env, upgrade_type=None):
-    import params
-
-
   def start(self, env, upgrade_type=None):
     import params
     env.set_params(params)
@@ -95,8 +91,6 @@ class HbaseRegionServerDefault(HbaseRegionServer):
 
     hbase_service('regionserver', action='start')
 
-    self.post_start(env, upgrade_type=upgrade_type)
-
   def stop(self, env, upgrade_type=None):
     import params
     env.set_params(params)

http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
index 4f92cf9..72d612a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
@@ -69,6 +69,9 @@
               <name>ambari-infra-solr-client</name>
               <condition>should_install_infra_solr_client</condition>
             </package>
+            <package>
+              <name>kafka_${stack_version}</name>
+            </package>
           </packages>
         </osSpecific>
         <osSpecific>
@@ -81,6 +84,9 @@
               <name>ambari-infra-solr-client</name>
               <condition>should_install_infra_solr_client</condition>
             </package>
+            <package>
+              <name>kafka-${stack_version}</name>
+            </package>
           </packages>
         </osSpecific>
       </osSpecifics>

http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index a45a4bd..66b85ea 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -53,7 +53,18 @@ shutil.copyfile(project_dir+"/ambari-server/conf/unix/ambari.properties",
"/tmp/
 
 # We have to use this import HACK because the filename contains a dash
 _search_file = os_utils.search_file
-os_utils.search_file = MagicMock(return_value="/tmp/ambari.properties")
+
+
+def search_file_proxy(filename, searchpatch, pathsep=os.pathsep):
+  global _search_file
+
+  if "ambari.properties" in filename:
+    return "/tmp/ambari.properties"
+
+  return _search_file(filename, searchpatch, pathsep)
+
+
+os_utils.search_file = search_file_proxy
 with patch.object(platform, "linux_distribution", return_value = MagicMock(return_value=('Redhat',
'6.4', 'Final'))):
   with patch("os.path.isdir", return_value = MagicMock(return_value=True)):
     with patch("os.access", return_value = MagicMock(return_value=True)):
@@ -62,7 +73,6 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur
           with patch("os.symlink"):
             with patch("glob.glob", return_value = ['/etc/init.d/postgresql-9.3']):
               _ambari_server_ = __import__('ambari-server')
-              os_utils.search_file = _search_file
               with patch("__builtin__.open"):
                 from ambari_commons.firewall import Firewall
                 from ambari_commons.os_check import OSCheck, OSConst
@@ -116,7 +126,6 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur
                 from ambari_server.hostUpdate import update_host_names
                 from ambari_server.checkDatabase import check_database
                 from ambari_server import serverConfiguration
-                serverConfiguration.search_file = _search_file
 
 CURR_AMBARI_VERSION = "2.0.0"
 


Mime
View raw message