ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tbeerbo...@apache.org
Subject ambari git commit: AMBARI-14921 - Atlas Integration : Support Atlas / Sqoop integration
Date Thu, 18 Feb 2016 17:16:06 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 0a9101e97 -> 260419bb8


AMBARI-14921 - Atlas Integration : Support Atlas / Sqoop integration


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/260419bb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/260419bb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/260419bb

Branch: refs/heads/trunk
Commit: 260419bb848ce962bbc238379221096eb2a31a58
Parents: 0a9101e
Author: tbeerbower <tbeerbower@hortonworks.com>
Authored: Thu Feb 18 11:22:03 2016 -0500
Committer: tbeerbower <tbeerbower@hortonworks.com>
Committed: Thu Feb 18 12:15:55 2016 -0500

----------------------------------------------------------------------
 .../1.4.4.2.0/configuration/sqoop-site.xml      | 43 ++++++++++++++++++++
 .../SQOOP/1.4.4.2.0/metainfo.xml                |  6 +++
 .../1.4.4.2.0/package/scripts/params_linux.py   | 20 ++++++++-
 .../SQOOP/1.4.4.2.0/package/scripts/sqoop.py    | 26 ++++++++++++
 .../python/stacks/2.0.6/SQOOP/test_sqoop.py     | 12 ++++++
 .../python/stacks/2.0.6/configs/default.json    |  5 +++
 .../python/stacks/2.0.6/configs/secured.json    |  5 +++
 .../test/python/stacks/2.3/configs/default.json |  5 +++
 8 files changed, 121 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
new file mode 100644
index 0000000..dd299fe
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>atlas.cluster.name</name>
+    <value>{{cluster_name}}</value>
+    <depends-on>
+      <property>
+        <type>application-properties</type>
+        <name>atlas.cluster.name</name>
+      </property>
+    </depends-on>
+  </property>
+  <property>
+    <name>sqoop.job.data.publish.class</name>
+    <value>{{job_data_publish_class}}</value>
+    <depends-on>
+      <property>
+        <type>application-properties</type>
+        <name>atlas.cluster.name</name>
+      </property>
+    </depends-on>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
index b5db91b..2c9d881 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
@@ -55,6 +55,11 @@
           </commandScript>
           <configFiles>
             <configFile>
+              <type>xml</type>
+              <fileName>squoop-site.xml</fileName>
+              <dictionaryName>squoop-site</dictionaryName>
+            </configFile>
+            <configFile>
               <type>env</type>
               <fileName>sqoop-env.sh</fileName>
               <dictionaryName>sqoop-env</dictionaryName>
@@ -89,6 +94,7 @@
       
       <configuration-dependencies>
         <config-type>sqoop-env</config-type>
+        <config-type>sqoop-site</config-type>
       </configuration-dependencies>
     </service>
   </services>

http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
index 3a3f93e..0006078 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
@@ -21,6 +21,7 @@ from resource_management.libraries.functions.version import format_hdp_stack_ver
 from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions.get_kinit_path import get_kinit_path
 from resource_management.libraries.script import Script
+import os
 
 # a map of the Ambari role to the component name
 # for use with /usr/hdp/current/<component>
@@ -31,6 +32,9 @@ SERVER_ROLE_DIRECTORY_MAP = {
 component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "SQOOP")
 
 config = Script.get_config()
+
+cluster_name = config['clusterName']
+
 ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
 
 stack_name = default("/hostLevelParams/stack_name", None)
@@ -103,4 +107,18 @@ if "jdbc_drivers" in config['configurations']['sqoop-env']:
       continue
     sqoop_jdbc_drivers_dict[jdbc_jar_name] = jdbc_symlink_name
     sqoop_jdbc_drivers_name_dict[jdbc_jar_name] = jdbc_driver_name
-jdk_location = config['hostLevelParams']['jdk_location']
\ No newline at end of file
+jdk_location = config['hostLevelParams']['jdk_location']
+
+job_data_publish_class = ''
+
+########################################################
+############# Atlas related params #####################
+########################################################
+
+atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
+has_atlas = len(atlas_hosts) > 0
+
+if has_atlas:
+  atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else
'/usr/hdp/current/atlas-server'
+  atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
+  job_data_publish_class = 'org.apache.atlas.sqoop.hook.SqoopHook'
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
index 66a5ee4..392bff7 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
@@ -19,6 +19,7 @@ limitations under the License.
 
 from resource_management.core.source import InlineTemplate, DownloadSource
 from resource_management.libraries.functions import format
+from resource_management.libraries.resources.xml_config import XmlConfig
 from resource_management.core.resources.system import File, Link, Directory
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons import OSConst
@@ -46,6 +47,31 @@ def sqoop(type=None):
             group = params.user_group,
             create_parents = True
   )
+
+  configs = {}
+  configs.update(params.config['configurations']['sqoop-site'])
+
+  XmlConfig("sqoop-site.xml",
+            conf_dir = params.sqoop_conf_dir,
+            configurations = configs,
+            configuration_attributes=params.config['configuration_attributes']['sqoop-site'],
+            owner = params.sqoop_user,
+            group = params.user_group
+            )
+
+  if params.has_atlas:
+    Link(params.sqoop_conf_dir + "/application.properties",
+         to = params.atlas_conf_dir + "/application.properties"
+         )
+
+    atlas_sqoop_hook_dir = params.atlas_home_dir + "/hook/sqoop"
+    src_files = os.listdir(atlas_sqoop_hook_dir)
+    for file_name in src_files:
+      atlas_sqoop_hook_file_name = os.path.join(atlas_sqoop_hook_dir, file_name)
+      sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name)
+      if (os.path.isfile(atlas_sqoop_hook_file_name)):
+        Link(sqoop_lib_file_name, to = atlas_sqoop_hook_file_name)
+
   File(format("{sqoop_conf_dir}/sqoop-env.sh"),
     owner=params.sqoop_user,
     group = params.user_group,

http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py b/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
index 5a10305..e12db4d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
@@ -39,6 +39,12 @@ class TestSqoop(RMFTestCase):
                               create_parents = True,
                               owner = 'sqoop',
                               group = 'hadoop',)
+    self.assertResourceCalled('XmlConfig', 'sqoop-site.xml',
+                              owner = 'sqoop',
+                              group = 'hadoop',
+                              conf_dir = '/usr/lib/sqoop/conf',
+                              configurations = self.getConfig()['configurations']['sqoop-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['sqoop-site'])
     self.assertResourceCalled('File', '/usr/lib/sqoop/conf/sqoop-env.sh',
                               owner = 'sqoop',
                               group = 'hadoop',
@@ -89,6 +95,12 @@ class TestSqoop(RMFTestCase):
                               group = 'hadoop',
                               create_parents = True,
                               )
+    self.assertResourceCalled('XmlConfig', 'sqoop-site.xml',
+                              owner = 'sqoop',
+                              group = 'hadoop',
+                              conf_dir = '/usr/lib/sqoop/conf',
+                              configurations = self.getConfig()['configurations']['sqoop-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['sqoop-site'])
     self.assertResourceCalled('File', '/usr/lib/sqoop/conf/sqoop-env.sh',
                               content = InlineTemplate(self.getConfig()['configurations']['sqoop-env']['content']),
                               owner = 'sqoop',

http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index 3d3987b..adf12ad 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -40,6 +40,10 @@
     "taskId": 152,
     "public_hostname": "c6401.ambari.apache.org",
     "configurations": {
+        "sqoop-site": {
+            "atlas.cluster.name": "c1",
+            "sqoop.job.data.publish.class": "org.apache.atlas.sqoop.hook.SqoopHook"
+        },
         "mapred-site": {
             "mapreduce.jobhistory.address": "c6402.ambari.apache.org:10020",
             "mapreduce.cluster.administrators": " hadoop",
@@ -877,6 +881,7 @@
         "ams-hbase-log4j": {},
         "ams-site": {},
         "ams-ssl-server": {},
+        "sqoop-site": {},
         "yarn-site": {
         "final": {
           "yarn.nodemanager.disk-health-checker.min-healthy-disks": "true",

http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
index d673973..e17559d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
@@ -37,6 +37,10 @@
     "taskId": 186, 
     "public_hostname": "c6401.ambari.apache.org", 
     "configurations": {
+        "sqoop-site": {
+            "atlas.cluster.name": "c1",
+            "sqoop.job.data.publish.class": "org.apache.atlas.sqoop.hook.SqoopHook"
+        },
         "mapred-site": {
             "mapreduce.jobhistory.address": "c6402.ambari.apache.org:10020", 
             "mapreduce.jobhistory.webapp.spnego-keytab-file": "/etc/security/keytabs/spnego.service.keytab",

@@ -765,6 +769,7 @@
         }
     },
     "configuration_attributes": {
+      "sqoop-site": {},
       "yarn-site": {
         "final": {
           "yarn.nodemanager.disk-health-checker.min-healthy-disks": "true",

http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.3/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json
index a36455c..4a72f08 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json
@@ -33,6 +33,10 @@
         "slider-client": {
             "slider.yarn.queue": "default"
         },
+        "sqoop-site": {
+          "atlas.cluster.name": "c1",
+          "sqoop.job.data.publish.class": "org.apache.atlas.sqoop.hook.SqoopHook"
+        },
         "mahout-env": {
              "mahout_user": "mahout"
         },
@@ -214,6 +218,7 @@
       }
     },
     "configuration_attributes": {
+        "sqoop-site": {},
         "yarn-site": {
             "final": {
                 "yarn.nodemanager.disk-health-checker.min-healthy-disks": "true",


Mime
View raw message