ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tbeerbo...@apache.org
Subject ambari git commit: AMBARI-15939 - Atlas Integration :Use stack advisor to suggest correct integration related configs
Date Tue, 19 Apr 2016 13:21:53 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 381ace20d -> 89fd30b40


AMBARI-15939 - Atlas Integration :Use stack advisor to suggest correct integration related
configs


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/89fd30b4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/89fd30b4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/89fd30b4

Branch: refs/heads/trunk
Commit: 89fd30b40f3108bfcbcc73eb2d74c94a2ba14a7a
Parents: 381ace2
Author: tbeerbower <tbeerbower@hortonworks.com>
Authored: Tue Apr 19 09:21:40 2016 -0400
Committer: tbeerbower <tbeerbower@hortonworks.com>
Committed: Tue Apr 19 09:21:40 2016 -0400

----------------------------------------------------------------------
 .../FALCON/0.5.0.2.1/package/scripts/falcon.py  |  19 +-
 .../0.5.0.2.1/package/scripts/params_linux.py   |  25 +-
 .../package/scripts/setup_atlas_falcon.py       |  52 ++++
 .../1.4.4.2.0/configuration/sqoop-site.xml      |   7 -
 .../1.4.4.2.0/package/scripts/params_linux.py   |   6 +
 .../package/scripts/setup_atlas_sqoop.py        |  47 ++++
 .../SQOOP/1.4.4.2.0/package/scripts/sqoop.py    |  16 +-
 .../0.9.1.2.1/configuration/storm-site.xml      |   7 -
 .../0.9.1.2.1/package/scripts/params_linux.py   |   6 +
 .../package/scripts/setup_atlas_storm.py        |  45 ++++
 .../STORM/0.9.1.2.1/package/scripts/storm.py    |  13 +-
 .../configuration/falcon-startup.properties.xml |  10 +-
 .../stacks/HDP/2.3/services/stack_advisor.py    |  70 ++++-
 .../stacks/2.3/common/test_stack_advisor.py     | 258 +++++++++++++++++++
 14 files changed, 505 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
index 46457ad..6e2e678 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
@@ -27,7 +27,6 @@ from resource_management.core.resources.service import Service
 from resource_management.core.resources.service import ServiceConfig
 from resource_management.core.resources.system import Directory
 from resource_management.core.resources.system import File
-from resource_management.core.resources.system import Link
 from resource_management.libraries.script import Script
 from resource_management.libraries.resources import PropertiesFile
 from resource_management.libraries.functions import format
@@ -35,6 +34,7 @@ from resource_management.libraries.functions.show_logs import show_logs
 
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from setup_atlas_falcon import setup_atlas_falcon
 
 @OsFamilyFuncImpl(os_family = OsFamilyImpl.DEFAULT)
 def falcon(type, action = None, upgrade_type=None):
@@ -108,12 +108,7 @@ def falcon(type, action = None, upgrade_type=None):
         create_parents = True,
         cd_access = "a")
 
-    if params.has_atlas:
-      atlas_falcon_hook_dir = os.path.join(params.atlas_home_dir, "hook", "falcon")
-      if os.path.exists(atlas_falcon_hook_dir):
-        Link(os.path.join(params.falcon_conf_dir, params.atlas_conf_file),
-          to = os.path.join(params.atlas_conf_dir, params.atlas_conf_file)
-          )
+    setup_atlas_falcon()
 
   if type == 'server':
     if action == 'config':
@@ -189,16 +184,6 @@ def falcon(type, action = None, upgrade_type=None):
         show_logs(params.falcon_log_dir, params.falcon_user)
         raise
 
-      if params.has_atlas:
-        atlas_falcon_hook_dir = os.path.join(params.atlas_home_dir, "hook", "falcon")
-        if os.path.exists(atlas_falcon_hook_dir):
-          src_files = os.listdir(atlas_falcon_hook_dir)
-          for file_name in src_files:
-            atlas_falcon_hook_file_name = os.path.join(atlas_falcon_hook_dir, file_name)
-            falcon_lib_file_name = os.path.join(params.falcon_webinf_lib, file_name)
-            if (os.path.isfile(atlas_falcon_hook_file_name)):
-              Link(falcon_lib_file_name, to = atlas_falcon_hook_file_name)
-
     if action == 'stop':
       try:
         Execute(format('{falcon_home}/bin/falcon-stop'),

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
index 84d8b57..77e18ff 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
@@ -26,6 +26,7 @@ from resource_management.libraries.functions.get_not_managed_resources import
ge
 from resource_management.libraries.functions import get_kinit_path
 from resource_management.libraries.script.script import Script
 import os
+from resource_management.libraries.functions.expect import expect
 from resource_management.libraries.functions.stack_features import check_stack_feature
 from resource_management.libraries.functions import StackFeature
 
@@ -33,6 +34,9 @@ config = Script.get_config()
 stack_root = status_params.stack_root
 stack_name = default("/hostLevelParams/stack_name", None)
 
+agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
+agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
+
 # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
 version = default("/commandParams/version", None)
 
@@ -112,31 +116,14 @@ dfs_data_mirroring_dir = "/apps/data-mirroring"
 
 atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
 has_atlas = len(atlas_hosts) > 0
+atlas_plugin_package = "atlas-metadata*-hive-plugin"
+atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin"
 
 if has_atlas:
   atlas_conf_file = config['configurations']['atlas-env']['metadata_conf_file']
   atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
   atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else
format('{stack_root}/current/atlas-server')
 
-  application_services = "org.apache.falcon.security.AuthenticationInitializationService,\
-      org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
-      org.apache.falcon.service.ProcessSubscriberService,\
-      org.apache.falcon.entity.store.ConfigurationStore,\
-      org.apache.falcon.rerun.service.RetryService,\
-      org.apache.falcon.rerun.service.LateRunService,\
-      org.apache.falcon.service.LogCleanupService,\
-      org.apache.falcon.metadata.MetadataMappingService,\
-      org.apache.falcon.atlas.service.AtlasService"
-else:
-  application_services = "org.apache.falcon.security.AuthenticationInitializationService,\
-      org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
-      org.apache.falcon.service.ProcessSubscriberService,\
-      org.apache.falcon.entity.store.ConfigurationStore,\
-      org.apache.falcon.rerun.service.RetryService,\
-      org.apache.falcon.rerun.service.LateRunService,\
-      org.apache.falcon.service.LogCleanupService,\
-      org.apache.falcon.metadata.MetadataMappingService"
-
 hdfs_site = config['configurations']['hdfs-site']
 default_fs = config['configurations']['core-site']['fs.defaultFS']
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py
new file mode 100644
index 0000000..4b7c0c3
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/setup_atlas_falcon.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.packaging import Package
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import Link
+from ambari_commons import OSCheck
+
+import os
+
+def setup_atlas_falcon():
+  import params
+
+  if params.has_atlas:
+
+    if not params.host_sys_prepped:
+      Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
+              retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count)
+
+    atlas_falcon_hook_dir = os.path.join(params.atlas_home_dir, "hook", "falcon")
+    if os.path.exists(atlas_falcon_hook_dir):
+      Link(os.path.join(params.falcon_conf_dir, params.atlas_conf_file),
+           to = os.path.join(params.atlas_conf_dir, params.atlas_conf_file)
+           )
+
+      Directory(params.falcon_webinf_lib,
+                owner = params.falcon_user,
+                create_parents = True)
+
+      src_files = os.listdir(atlas_falcon_hook_dir)
+      for file_name in src_files:
+        atlas_falcon_hook_file_name = os.path.join(atlas_falcon_hook_dir, file_name)
+        falcon_lib_file_name = os.path.join(params.falcon_webinf_lib, file_name)
+        if (os.path.isfile(atlas_falcon_hook_file_name)):
+          Link(falcon_lib_file_name, to = atlas_falcon_hook_file_name)

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
index dd299fe..1bc4c43 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
@@ -32,12 +32,5 @@
   </property>
   <property>
     <name>sqoop.job.data.publish.class</name>
-    <value>{{job_data_publish_class}}</value>
-    <depends-on>
-      <property>
-        <type>application-properties</type>
-        <name>atlas.cluster.name</name>
-      </property>
-    </depends-on>
   </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
index 7db974f..658e69f 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
@@ -24,6 +24,7 @@ from resource_management.libraries.script import Script
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.expect import expect
 import os
 
 # a map of the Ambari role to the component name
@@ -46,6 +47,9 @@ stack_name = default("/hostLevelParams/stack_name", None)
 stack_version_unformatted = config['hostLevelParams']['stack_version']
 stack_version_formatted = format_stack_version(stack_version_unformatted)
 
+agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
+agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
+
 # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
 version = default("/commandParams/version", None)
 
@@ -116,6 +120,8 @@ job_data_publish_class = ''
 
 atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
 has_atlas = len(atlas_hosts) > 0
+atlas_plugin_package = "atlas-metadata*-hive-plugin"
+atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin"
 
 if has_atlas:
   atlas_conf_file = config['configurations']['atlas-env']['metadata_conf_file']

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py
new file mode 100644
index 0000000..d18d820
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/setup_atlas_sqoop.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.packaging import Package
+from resource_management.core.resources.system import Link
+from ambari_commons import OSCheck
+
+import os
+
+def setup_atlas_sqoop():
+  import params
+
+  if params.has_atlas:
+
+    if not params.host_sys_prepped:
+      Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
+              retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count)
+
+    atlas_sqoop_hook_dir = os.path.join(params.atlas_home_dir, "hook", "sqoop")
+    if os.path.exists(atlas_sqoop_hook_dir):
+      Link(os.path.join(params.sqoop_conf_dir, params.atlas_conf_file),
+           to = os.path.join(params.atlas_conf_dir, params.atlas_conf_file)
+           )
+
+      src_files = os.listdir(atlas_sqoop_hook_dir)
+      for file_name in src_files:
+        atlas_sqoop_hook_file_name = os.path.join(atlas_sqoop_hook_dir, file_name)
+        sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name)
+        if (os.path.isfile(atlas_sqoop_hook_file_name)):
+          Link(sqoop_lib_file_name, to = atlas_sqoop_hook_file_name)

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
index 1276a54..799447b 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
@@ -23,6 +23,8 @@ from resource_management.libraries.resources.xml_config import XmlConfig
 from resource_management.core.resources.system import File, Link, Directory
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons import OSConst
+from setup_atlas_sqoop import setup_atlas_sqoop
+
 import os
 
 
@@ -59,19 +61,7 @@ def sqoop(type=None):
             group = params.user_group
             )
 
-  if params.has_atlas:
-    atlas_sqoop_hook_dir = os.path.join(params.atlas_home_dir, "hook", "sqoop")
-    if os.path.exists(atlas_sqoop_hook_dir):
-      Link(os.path.join(params.sqoop_conf_dir, params.atlas_conf_file),
-           to = os.path.join(params.atlas_conf_dir, params.atlas_conf_file)
-           )
-
-      src_files = os.listdir(atlas_sqoop_hook_dir)
-      for file_name in src_files:
-        atlas_sqoop_hook_file_name = os.path.join(atlas_sqoop_hook_dir, file_name)
-        sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name)
-        if (os.path.isfile(atlas_sqoop_hook_file_name)):
-          Link(sqoop_lib_file_name, to = atlas_sqoop_hook_file_name)
+  setup_atlas_sqoop()
 
   File(format("{sqoop_conf_dir}/sqoop-env.sh"),
     owner=params.sqoop_user,

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
index cf702c0..5a3901a 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
@@ -739,12 +739,5 @@
   </property>
   <property>
     <name>storm.topology.submission.notifier.plugin.class</name>
-    <value>org.apache.atlas.storm.hook.StormAtlasHook</value>
-    <depends-on>
-      <property>
-        <type>application-properties</type>
-        <name>atlas.cluster.name</name>
-      </property>
-    </depends-on>
   </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
index 3b65ea9..0752b30 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
@@ -37,6 +37,7 @@ from resource_management.libraries.functions import get_kinit_path
 from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
 from resource_management.libraries.functions.stack_features import check_stack_feature
 from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.expect import expect
 
 # server configurations
 config = Script.get_config()
@@ -50,6 +51,9 @@ stack_name = default("/hostLevelParams/stack_name", None)
 upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE)
 version = default("/commandParams/version", None)
 
+agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
+agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
+
 storm_component_home_dir = status_params.storm_component_home_dir
 conf_dir = status_params.conf_dir
 
@@ -190,6 +194,8 @@ jar_jvm_opts = ''
 # Atlas related params
 atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
 has_atlas = len(atlas_hosts) > 0
+atlas_plugin_package = "atlas-metadata*-hive-plugin"
+atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin"
 
 if has_atlas:
   atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else
stack_root + '/current/atlas-server'

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_atlas_storm.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_atlas_storm.py
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_atlas_storm.py
new file mode 100644
index 0000000..6c3e91f
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/setup_atlas_storm.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.resources.packaging import Package
+from resource_management.core.resources.system import Link
+from ambari_commons import OSCheck
+
+import os
+
+def setup_atlas_storm():
+  import params
+
+  if params.has_atlas:
+
+    if not params.host_sys_prepped:
+      Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
+              retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count)
+
+    atlas_storm_hook_dir = os.path.join(params.atlas_home_dir, "hook", "storm")
+    if os.path.exists(atlas_storm_hook_dir):
+      storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib")
+      if os.path.exists(storm_extlib_dir):
+        src_files = os.listdir(atlas_storm_hook_dir)
+        for file_name in src_files:
+          atlas_storm_hook_file_name = os.path.join(atlas_storm_hook_dir, file_name)
+          storm_lib_file_name = os.path.join(storm_extlib_dir, file_name)
+          if (os.path.isfile(atlas_storm_hook_file_name)):
+            Link(storm_lib_file_name, to = atlas_storm_hook_file_name)

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
index 74050d8..2d50767 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
@@ -31,6 +31,7 @@ from resource_management.libraries.functions import StackFeature
 from storm_yaml_utils import yaml_config_template, yaml_config
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons import OSConst
+from setup_atlas_storm import setup_atlas_storm
 
 @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
 def storm(name=None):
@@ -94,17 +95,7 @@ def storm(name=None):
        content=InlineTemplate(params.storm_env_sh_template)
   )
 
-  if params.has_atlas:
-    atlas_storm_hook_dir = os.path.join(params.atlas_home_dir, "hook", "storm")
-    if os.path.exists(atlas_storm_hook_dir):
-      storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib")
-      if os.path.exists(storm_extlib_dir):
-        src_files = os.listdir(atlas_storm_hook_dir)
-        for file_name in src_files:
-          atlas_storm_hook_file_name = os.path.join(atlas_storm_hook_dir, file_name)
-          storm_lib_file_name = os.path.join(storm_extlib_dir, file_name)
-          if (os.path.isfile(atlas_storm_hook_file_name)):
-            Link(storm_lib_file_name, to = atlas_storm_hook_file_name)
+  setup_atlas_storm()
 
   if params.has_metric_collector:
     File(format("{conf_dir}/storm-metrics2.properties"),

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
index 5d4fa1e..4e4ec67 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
@@ -27,7 +27,15 @@
   </property>
   <property>
     <name>*.application.services</name>
-    <value>{{application_services}}</value>
+    <value>org.apache.falcon.security.AuthenticationInitializationService,\
+      org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
+      org.apache.falcon.service.ProcessSubscriberService,\
+      org.apache.falcon.entity.store.ConfigurationStore,\
+      org.apache.falcon.rerun.service.RetryService,\
+      org.apache.falcon.rerun.service.LateRunService,\
+      org.apache.falcon.service.LogCleanupService,\
+      org.apache.falcon.metadata.MetadataMappingService
+    </value>
     <description>Falcon Services</description>
   </property>
   <property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 66076f2..e880377 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -161,7 +161,10 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
       "KAFKA": self.recommendKAFKAConfigurations,
       "RANGER": self.recommendRangerConfigurations,
       "RANGER_KMS": self.recommendRangerKMSConfigurations,
-      "HAWQ": self.recommendHAWQConfigurations
+      "HAWQ": self.recommendHAWQConfigurations,
+      "FALCON": self.recommendFalconConfigurations,
+      "STORM": self.recommendStormConfigurations,
+      "SQOOP": self.recommendSqoopConfigurations
     }
     parentRecommendConfDict.update(childRecommendConfDict)
     return parentRecommendConfDict
@@ -730,6 +733,71 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
         putHdfsClientProperty = self.putProperty(configurations, "hdfs-client", services)
         putHdfsClientProperty("output.replace-datanode-on-failure", propertyValue)
 
+  def recommendSqoopConfigurations(self, configurations, clusterData, services, hosts):
+    putSqoopSiteProperty = self.putProperty(configurations, "sqoop-site", services)
+
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    if "ATLAS" in servicesList:
+      putSqoopSiteProperty('sqoop.job.data.publish.class', 'org.apache.atlas.sqoop.hook.SqoopHook')
+
+  def recommendStormConfigurations(self, configurations, clusterData, services, hosts):
+    putStormStartupProperty = self.putProperty(configurations, "storm-site", services)
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+
+    # atlas
+    notifier_plugin_property = "storm.topology.submission.notifier.plugin.class"
+    if notifier_plugin_property in services["configurations"]["storm-site"]["properties"]:
+      notifier_plugin_value = services["configurations"]["storm-site"]["properties"][notifier_plugin_property]
+    else:
+      notifier_plugin_value = " "
+
+    include_atlas = "ATLAS" in servicesList
+    atlas_hook_class = "org.apache.atlas.storm.hook.StormAtlasHook"
+    if include_atlas and atlas_hook_class not in notifier_plugin_value:
+      if notifier_plugin_value == " ":
+        notifier_plugin_value = atlas_hook_class
+      else:
+        notifier_plugin_value = notifier_plugin_value + "," + atlas_hook_class
+    if not include_atlas and atlas_hook_class in notifier_plugin_value:
+      application_classes = []
+      for application_class in notifier_plugin_value.split(","):
+        if application_class != atlas_hook_class and application_class != " ":
+          application_classes.append(application_class)
+      if application_classes:
+        notifier_plugin_value = ",".join(application_classes)
+      else:
+        notifier_plugin_value = " "
+    putStormStartupProperty(notifier_plugin_property, notifier_plugin_value)
+
+  def recommendFalconConfigurations(self, configurations, clusterData, services, hosts):
+
+    putFalconStartupProperty = self.putProperty(configurations, "falcon-startup.properties",
services)
+    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+
+    # atlas
+    application_services_property = "*.application.services"
+    if application_services_property in services["configurations"]["falcon-startup.properties"]["properties"]:
+      application_services_value = services["configurations"]["falcon-startup.properties"]["properties"][application_services_property]
+    else:
+      application_services_value = " "
+
+    include_atlas = "ATLAS" in servicesList
+    atlas_application_class = "org.apache.falcon.atlas.service.AtlasService"
+    if include_atlas and atlas_application_class not in application_services_value:
+      if application_services_value == " ":
+        application_services_value = atlas_application_class
+      else:
+        application_services_value = application_services_value + "," + atlas_application_class
+    if not include_atlas and atlas_application_class in application_services_value:
+      application_classes = []
+      for application_class in application_services_value.split(","):
+        if application_class != atlas_application_class and application_class != " ":
+          application_classes.append(application_class)
+      if application_classes:
+        application_services_value = ",".join(application_classes)
+      else:
+        application_services_value = " "
+    putFalconStartupProperty(application_services_property, application_services_value)
 
   def getServiceConfigurationValidators(self):
     parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()

http://git-wip-us.apache.org/repos/asf/ambari/blob/89fd30b4/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 6c9fd46..617f73d 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -2355,3 +2355,261 @@ class TestHDP23StackAdvisor(TestCase):
     # kms_user was changed, old property should be removed
     self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData,
services, None)
     self.assertEquals(recommendedConfigurations, expected)
+
+  def test_recommendStormConfigurations(self):
+    self.maxDiff = None
+    configurations = {
+      "storm-site": {
+        "properties": {
+          "storm.topology.submission.notifier.plugin.class": "foo"
+        }
+      }
+    }
+    clusterData = {
+      "cpu": 4,
+      "mapMemory": 3000,
+      "amMemory": 2000,
+      "reduceMemory": 2056,
+      "containers": 3,
+      "ramPerContainer": 256
+    }
+    expected = {
+      'storm-site': {
+        'properties': {
+          'storm.topology.submission.notifier.plugin.class': 'foo,org.apache.atlas.storm.hook.StormAtlasHook',
+        }
+      }
+    }
+    services = {
+      "services": [
+        {
+          "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS",
+          "StackServices": {
+            "service_name": "ATLAS",
+            "service_version": "2.6.0.2.2",
+            "stack_name": "HDP",
+            "stack_version": "2.3"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "advertise_version": "false",
+                "cardinality": "1",
+                "component_category": "MASTER",
+                "component_name": "ATLAS_SERVER",
+                "display_name": "Atlas Server",
+                "is_client": "false",
+                "is_master": "true",
+                "hostnames": []
+              },
+              "dependencies": []
+            }
+          ]
+        },
+      ],
+      "configurations": {
+        "storm-site": {
+          "properties": {
+            "storm.topology.submission.notifier.plugin.class": "foo"
+          }
+        }
+      },
+      "changed-configurations": [ ]
+
+    }
+    hosts = {
+      "items" : [
+        {
+          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+          "Hosts" : {
+            "cpu_count" : 1,
+            "host_name" : "c6401.ambari.apache.org",
+            "os_arch" : "x86_64",
+            "os_type" : "centos6",
+            "ph_cpu_count" : 1,
+            "public_host_name" : "c6401.ambari.apache.org",
+            "rack_info" : "/default-rack",
+            "total_mem" : 1922680
+          }
+        }
+      ]
+    }
+
+    self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)
+
+    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
+    self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)
+
+  def test_recommendFalconConfigurations(self):
+    self.maxDiff = None
+    configurations = {
+      "falcon-startup.properties": {
+        "properties": {
+          "*.application.services": "foo"
+        }
+      }
+    }
+    clusterData = {
+      "cpu": 4,
+      "mapMemory": 3000,
+      "amMemory": 2000,
+      "reduceMemory": 2056,
+      "containers": 3,
+      "ramPerContainer": 256
+    }
+    expected = {
+      'falcon-startup.properties': {
+        'properties': {
+          '*.application.services': 'foo,org.apache.falcon.atlas.service.AtlasService',
+        }
+      }
+    }
+    services = {
+      "services": [
+        {
+          "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS",
+          "StackServices": {
+            "service_name": "ATLAS",
+            "service_version": "2.6.0.2.2",
+            "stack_name": "HDP",
+            "stack_version": "2.3"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "advertise_version": "false",
+                "cardinality": "1",
+                "component_category": "MASTER",
+                "component_name": "ATLAS_SERVER",
+                "display_name": "Atlas Server",
+                "is_client": "false",
+                "is_master": "true",
+                "hostnames": []
+              },
+              "dependencies": []
+            }
+          ]
+        },
+      ],
+      "configurations": {
+        "falcon-startup.properties": {
+          "properties": {
+            "*.application.services": "foo"
+          }
+        }
+      },
+      "changed-configurations": [ ]
+
+    }
+    hosts = {
+      "items" : [
+        {
+          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+          "Hosts" : {
+            "cpu_count" : 1,
+            "host_name" : "c6401.ambari.apache.org",
+            "os_arch" : "x86_64",
+            "os_type" : "centos6",
+            "ph_cpu_count" : 1,
+            "public_host_name" : "c6401.ambari.apache.org",
+            "rack_info" : "/default-rack",
+            "total_mem" : 1922680
+          }
+        }
+      ]
+    }
+
+    self.stackAdvisor.recommendFalconConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)
+
+    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
+    self.stackAdvisor.recommendFalconConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)
+
+  def test_recommendSqoopConfigurations(self):
+    self.maxDiff = None
+    configurations = {
+      "sqoop-site": {
+        "properties": {
+          "sqoop.job.data.publish.class": "foo"
+        }
+      }
+    }
+    clusterData = {
+      "cpu": 4,
+      "mapMemory": 3000,
+      "amMemory": 2000,
+      "reduceMemory": 2056,
+      "containers": 3,
+      "ramPerContainer": 256
+    }
+    expected = {
+      'sqoop-site': {
+        'properties': {
+          'sqoop.job.data.publish.class': 'org.apache.atlas.sqoop.hook.SqoopHook',
+        }
+      }
+    }
+    services = {
+      "services": [
+        {
+          "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS",
+          "StackServices": {
+            "service_name": "ATLAS",
+            "service_version": "2.6.0.2.2",
+            "stack_name": "HDP",
+            "stack_version": "2.3"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "advertise_version": "false",
+                "cardinality": "1",
+                "component_category": "MASTER",
+                "component_name": "ATLAS_SERVER",
+                "display_name": "Atlas Server",
+                "is_client": "false",
+                "is_master": "true",
+                "hostnames": []
+              },
+              "dependencies": []
+            }
+          ]
+        },
+      ],
+      "configurations": {
+        "sqoop-site": {
+          "properties": {
+            "sqoop.job.data.publish.class": "foo"
+          }
+        }
+      },
+      "changed-configurations": [ ]
+
+    }
+    hosts = {
+      "items" : [
+        {
+          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+          "Hosts" : {
+            "cpu_count" : 1,
+            "host_name" : "c6401.ambari.apache.org",
+            "os_arch" : "x86_64",
+            "os_type" : "centos6",
+            "ph_cpu_count" : 1,
+            "public_host_name" : "c6401.ambari.apache.org",
+            "rack_info" : "/default-rack",
+            "total_mem" : 1922680
+          }
+        }
+      ]
+    }
+
+    self.stackAdvisor.recommendSqoopConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)
+
+    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
+    self.stackAdvisor.recommendSqoopConfigurations(configurations, clusterData, services,
hosts)
+    self.assertEquals(configurations, expected)


Mime
View raw message