ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rle...@apache.org
Subject [33/50] [abbrv] ambari git commit: AMBARI-21023. HDP 3.0 TP - create service definition for Sqoop with configs, kerberos, widgets, etc.(vbrodetsky)
Date Tue, 30 May 2017 18:29:55 GMT
AMBARI-21023. HDP 3.0 TP - create service definition for Sqoop with configs, kerberos, widgets,
etc.(vbrodetsky)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b3339123
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b3339123
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b3339123

Branch: refs/heads/branch-feature-AMBARI-20859
Commit: b33391236b385e2e4620031b9773261bce8efdf6
Parents: 47b845f
Author: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Authored: Tue May 16 22:16:59 2017 +0300
Committer: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Committed: Tue May 16 22:16:59 2017 +0300

----------------------------------------------------------------------
 .../sqoop-atlas-application.properties.xml      |  47 +++++++
 .../SQOOP/1.4.4.3.0/configuration/sqoop-env.xml |  87 ++++++++++++
 .../1.4.4.3.0/configuration/sqoop-site.xml      |  38 ++++++
 .../SQOOP/1.4.4.3.0/kerberos.json               |  20 +++
 .../SQOOP/1.4.4.3.0/metainfo.xml                | 115 ++++++++++++++++
 .../SQOOP/1.4.4.3.0/package/scripts/__init__.py |  19 +++
 .../SQOOP/1.4.4.3.0/package/scripts/params.py   |  27 ++++
 .../1.4.4.3.0/package/scripts/params_linux.py   | 135 +++++++++++++++++++
 .../1.4.4.3.0/package/scripts/params_windows.py |  30 +++++
 .../1.4.4.3.0/package/scripts/service_check.py  |  62 +++++++++
 .../SQOOP/1.4.4.3.0/package/scripts/sqoop.py    | 124 +++++++++++++++++
 .../1.4.4.3.0/package/scripts/sqoop_client.py   |  66 +++++++++
 .../SQOOP/1.4.4.3.0/role_command_order.json     |   6 +
 .../stacks/HDP/3.0/services/SQOOP/metainfo.xml  |  27 ++++
 14 files changed, 803 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml
new file mode 100644
index 0000000..1364776
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <!--
+  These two security properties will only be read if the cluster is Kerberized, but ok to
add them even without Kerberos.
+  It's important to have at least one property in this config file so it gets added while
merging configs during EU/RU
+  from an earlier stack to HDP 2.5+.
+  Also, it allows a fresh install with Sqoop to expose this config type in the UI.
+  -->
+  <property>
+    <name>atlas.jaas.KafkaClient.option.useTicketCache</name>
+    <value>true</value>
+    <description>
+      Set this to "true" if you want the TGT to be obtained from the ticket cache.
+      Set this option to "false" if you do not want this module to use the ticket cache.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>atlas.jaas.KafkaClient.option.renewTicket</name>
+    <value>true</value>
+    <description>
+      Set this to "true" if you want the TGT to renew the ticket when it expires.
+      Set this option to "false" if you do not want this module to renew tickets.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml
new file mode 100644
index 0000000..6e16ab33
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+  <!-- sqoop-env.sh -->
+  <property>
+    <name>sqoop.atlas.hook</name>
+    <value>false</value>
+    <display-name>Enable Atlas Hook</display-name>
+    <description>Enable Atlas Hook</description>
+    <value-attributes>
+      <type>boolean</type>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+    <depends-on>
+      <property>
+        <type>application-properties</type>
+        <name>atlas.rest.address</name>
+      </property>
+    </depends-on>
+  </property>
+  <property>
+    <name>content</name>
+    <display-name>sqoop-env template</display-name>
+    <description>This is the jinja template for sqoop-env.sh file</description>
+    <value>
+# Set Hadoop-specific environment variables here.
+
+#Set path to where bin/hadoop is available
+#Set path to where bin/hadoop is available
+export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+
+#set the path to where bin/hbase is available
+export HBASE_HOME=${HBASE_HOME:-{{hbase_home}}}
+
+#Set the path to where bin/hive is available
+export HIVE_HOME=${HIVE_HOME:-{{hive_home}}}
+
+#Set the path for where zookeper config dir is
+export ZOOCFGDIR=${ZOOCFGDIR:-/etc/zookeeper/conf}
+
+# add libthrift in hive to sqoop class path first so hive imports work
+export SQOOP_USER_CLASSPATH="`ls ${HIVE_HOME}/lib/libthrift-*.jar 2&gt; /dev/null`:${SQOOP_USER_CLASSPATH}"
+    </value>
+    <value-attributes>
+      <type>content</type>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>sqoop_user</name>
+    <display-name>Sqoop User</display-name>
+    <description>User to run Sqoop as</description>
+    <property-type>USER</property-type>
+    <value>sqoop</value>
+    <value-attributes>
+      <type>user</type>
+      <overridable>false</overridable>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>jdbc_drivers</name>
+    <description>Comma separated list of additional JDBC drivers class names</description>
+    <value> </value>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml
new file mode 100644
index 0000000..389550d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <!-- This property was valid in HDP 2.3 and 2.4, but removed in HDP 2.5 -->
+  <property>
+    <name>sqoop.job.data.publish.class</name>
+    <on-ambari-upgrade add="false"/>
+    <depends-on>
+      <property>
+        <type>sqoop-env</type>
+        <name>sqoop.atlas.hook</name>
+      </property>
+      <property>
+        <type>application-properties</type>
+        <name>atlas.rest.address</name>
+      </property>
+    </depends-on>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json
new file mode 100644
index 0000000..de12e7c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json
@@ -0,0 +1,20 @@
+{
+  "services": [
+    {
+      "name": "SQOOP",
+      "configurations": [
+        {
+          "sqoop-atlas-application.properties": {
+            "atlas.jaas.KafkaClient.option.useTicketCache": "true",
+            "atlas.jaas.KafkaClient.option.renewTicket": "true"
+          }
+        }
+      ],
+      "components": [
+        {
+          "name": "SQOOP"
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml
new file mode 100644
index 0000000..999d93a
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml
@@ -0,0 +1,115 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>SQOOP</name>
+      <displayName>Sqoop</displayName>
+      <comment>Tool for transferring bulk data between Apache Hadoop and
+        structured data stores such as relational databases
+      </comment>
+      <version>1.4.4.3.0</version>
+
+      <components>
+        <component>
+          <name>SQOOP</name>
+          <displayName>Sqoop Client</displayName>
+          <category>CLIENT</category>
+          <cardinality>1+</cardinality>
+          <versionAdvertised>true</versionAdvertised>
+          <dependencies>
+            <dependency>
+              <name>HDFS/HDFS_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+            <dependency>
+              <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+          </dependencies>
+          <commandScript>
+            <script>scripts/sqoop_client.py</script>
+            <scriptType>PYTHON</scriptType>
+          </commandScript>
+          <configFiles>
+            <configFile>
+              <type>xml</type>
+              <fileName>sqoop-site.xml</fileName>
+              <dictionaryName>sqoop-site</dictionaryName>
+            </configFile>
+            <configFile>
+              <type>env</type>
+              <fileName>sqoop-env.sh</fileName>
+              <dictionaryName>sqoop-env</dictionaryName>
+            </configFile>
+          </configFiles>
+        </component>
+      </components>
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>any</osFamily>
+          <packages>
+            <package>
+              <name>mysql-connector-java</name>
+              <skipUpgrade>true</skipUpgrade>
+              <condition>should_install_mysql_connector</condition>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily>
+          <packages>
+            <package>
+              <name>sqoop_${stack_version}</name>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily>
+          <packages>
+            <package>
+              <name>sqoop-${stack_version}</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+      <commandScript>
+        <script>scripts/service_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>300</timeout>
+      </commandScript>
+      
+      <requiredServices>
+        <service>HDFS</service>
+      </requiredServices>
+      
+      <configuration-dependencies>
+        <config-type>sqoop-env</config-type>
+        <config-type>sqoop-site</config-type>
+        <config-type>application.properties</config-type>
+      </configuration-dependencies>
+    </service>
+  </services>
+</metainfo>

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py
new file mode 100644
index 0000000..5561e10
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py
new file mode 100644
index 0000000..61573ee
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py
@@ -0,0 +1,27 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+
+if OSCheck.is_windows_family():
+  from params_windows import *
+else:
+  from params_linux import *
+
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py
new file mode 100644
index 0000000..c1138b3
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py
@@ -0,0 +1,135 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.get_kinit_path import get_kinit_path
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.expect import expect
+from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
+
+
+# a map of the Ambari role to the component name
+# for use with <stack-root>/current/<component>
+SERVER_ROLE_DIRECTORY_MAP = {
+  'SQOOP' : 'sqoop-client'
+}
+
+component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "SQOOP")
+
+config = Script.get_config()
+stack_root = Script.get_stack_root()
+
+# Needed since this is an Atlas Hook service.
+cluster_name = config['clusterName']
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+stack_name = default("/hostLevelParams/stack_name", None)
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
+agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
+
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
+# default hadoop params
+sqoop_conf_dir = "/usr/lib/sqoop/conf"
+sqoop_lib = "/usr/lib/sqoop/lib"
+hadoop_home = '/usr/lib/hadoop'
+hbase_home = "/usr/lib/hbase"
+hive_home = "/usr/lib/hive"
+sqoop_bin_dir = "/usr/bin"
+zoo_conf_dir = "/etc/zookeeper"
+
+# For stack versions supporting rolling upgrade
+if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+  sqoop_conf_dir = format("{stack_root}/current/sqoop-client/conf")
+  sqoop_lib = format("{stack_root}/current/sqoop-client/lib")
+  hadoop_home = format("{stack_root}/current/hadoop-client")
+  hbase_home = format("{stack_root}/current/hbase-client")
+  hive_home = format("{stack_root}/current/hive-client")
+  sqoop_bin_dir = format("{stack_root}/current/sqoop-client/bin/")
+  zoo_conf_dir = format("{stack_root}/current/zookeeper-client/conf")
+
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+user_group = config['configurations']['cluster-env']['user_group']
+sqoop_env_sh_template = config['configurations']['sqoop-env']['content']
+
+sqoop_user = config['configurations']['sqoop-env']['sqoop_user']
+
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
+#JDBC driver jar name
+sqoop_jdbc_drivers_dict = []
+sqoop_jdbc_drivers_name_dict = {}
+sqoop_jdbc_drivers_to_remove = {}
+if "jdbc_drivers" in config['configurations']['sqoop-env']:
+  sqoop_jdbc_drivers = config['configurations']['sqoop-env']['jdbc_drivers'].split(',')
+
+  for driver_name in sqoop_jdbc_drivers:
+    previous_jdbc_jar_name = None
+    driver_name = driver_name.strip()
+    if driver_name and not driver_name == '':
+      if driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver":
+        jdbc_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
+        previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name",
None)
+        jdbc_driver_name = "mssql"
+      elif driver_name == "com.mysql.jdbc.Driver":
+        jdbc_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
+        previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name",
None)
+        jdbc_driver_name = "mysql"
+      elif driver_name == "org.postgresql.Driver":
+        jdbc_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
+        previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name",
None)
+        jdbc_driver_name = "postgres"
+      elif driver_name == "oracle.jdbc.driver.OracleDriver":
+        jdbc_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
+        previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name",
None)
+        jdbc_driver_name = "oracle"
+      elif driver_name == "org.hsqldb.jdbc.JDBCDriver":
+        jdbc_name = default("/hostLevelParams/custom_hsqldb_jdbc_name", None)
+        previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_hsqldb_jdbc_name",
None)
+        jdbc_driver_name = "hsqldb"
+    else:
+      continue
+    sqoop_jdbc_drivers_dict.append(jdbc_name)
+    sqoop_jdbc_drivers_to_remove[jdbc_name] = previous_jdbc_jar_name
+    sqoop_jdbc_drivers_name_dict[jdbc_name] = jdbc_driver_name
+jdk_location = config['hostLevelParams']['jdk_location']
+
+
+########################################################
+############# Atlas related params #####################
+########################################################
+#region Atlas Hooks
+sqoop_atlas_application_properties = default('/configurations/sqoop-atlas-application.properties',
{})
+enable_atlas_hook = default('/configurations/sqoop-env/sqoop.atlas.hook', False)
+atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
+#endregion

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py
new file mode 100644
index 0000000..f930765
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py
@@ -0,0 +1,30 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+import os
+
+config = Script.get_config()
+
+sqoop_user = "sqoop"
+
+stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], ".."))
+sqoop_env_cmd_template = config['configurations']['sqoop-env']['content']
+sqoop_home_dir = os.environ["SQOOP_HOME"]
+sqoop_conf_dir = os.path.join(sqoop_home_dir, "conf")
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py
new file mode 100644
index 0000000..bb503f5
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+
+from resource_management.libraries.script.script import Script
+from resource_management.core.resources import Execute
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import format
+from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_commons import OSConst
+import os
+
+class SqoopServiceCheck(Script):
+  pass
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class SqoopServiceCheckDefault(SqoopServiceCheck):
+
+  def get_component_name(self):
+    return "sqoop-server"
+
+  def service_check(self, env):
+    import params
+    env.set_params(params)
+    if params.security_enabled:
+      Execute(format("{kinit_path_local}  -kt {smoke_user_keytab} {smokeuser_principal}"),
+              user = params.smokeuser,
+      )
+    Execute("sqoop version",
+            user = params.smokeuser,
+            path = params.sqoop_bin_dir,
+            logoutput = True
+    )
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class SqoopServiceCheckWindows(SqoopServiceCheck):
+  def service_check(self, env):
+    import params
+    env.set_params(params)
+    smoke_cmd = os.path.join(params.stack_root,"Run-SmokeTests.cmd")
+    service = "SQOOP"
+    Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True)
+
+if __name__ == "__main__":
+  SqoopServiceCheck().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py
new file mode 100644
index 0000000..436402c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py
@@ -0,0 +1,124 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+# Python Imports
+import os
+
+# Local Imports
+from resource_management.core.source import InlineTemplate, DownloadSource
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.get_config import get_config
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.core.resources.system import File, Link, Directory
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from ambari_commons import OSConst
+from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster,
setup_atlas_hook, setup_atlas_jar_symlinks
+from ambari_commons.constants import SERVICE
+
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def sqoop(type=None):
+  import params
+  File(os.path.join(params.sqoop_conf_dir, "sqoop-env.cmd"),
+       content=InlineTemplate(params.sqoop_env_cmd_template)
+  )
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def sqoop(type=None):
+  import params
+  Link(params.sqoop_lib + "/mysql-connector-java.jar",
+       to = '/usr/share/java/mysql-connector-java.jar'
+  )
+
+  jdbc_connector()
+  
+  Directory(params.sqoop_conf_dir,
+            owner = params.sqoop_user,
+            group = params.user_group,
+            create_parents = True
+  )
+
+  configs = {}
+  sqoop_site_config = get_config('sqoop-site')
+  if sqoop_site_config:
+    configs.update(sqoop_site_config)
+
+    XmlConfig("sqoop-site.xml",
+              conf_dir = params.sqoop_conf_dir,
+              configurations = configs,
+              configuration_attributes=params.config['configuration_attributes']['sqoop-site'],
+              owner = params.sqoop_user,
+              group = params.user_group
+              )
+
+  # Generate atlas-application.properties.xml file and symlink the hook jars
+  if params.enable_atlas_hook:
+    atlas_hook_filepath = os.path.join(params.sqoop_conf_dir, params.atlas_hook_filename)
+    setup_atlas_hook(SERVICE.SQOOP, params.sqoop_atlas_application_properties, atlas_hook_filepath,
params.sqoop_user, params.user_group)
+    setup_atlas_jar_symlinks("sqoop", params.sqoop_lib)
+
+
+  File(format("{sqoop_conf_dir}/sqoop-env.sh"),
+    owner=params.sqoop_user,
+    group = params.user_group,
+    content=InlineTemplate(params.sqoop_env_sh_template)
+  )
+  update_config_permissions(["sqoop-env-template.sh",
+                             "sqoop-site-template.xml",
+                             "sqoop-site.xml"])
+  pass
+
+def update_config_permissions(names):
+  import params
+  for filename in names:
+    full_filename = os.path.join(params.sqoop_conf_dir, filename)
+    File(full_filename,
+          owner = params.sqoop_user,
+          group = params.user_group,
+          only_if = format("test -e {full_filename}")
+    )
+
+def jdbc_connector():
+  import params
+  from urllib2 import HTTPError
+  from resource_management import Fail
+  for jar_name in params.sqoop_jdbc_drivers_dict:
+    if not jar_name or 'mysql' in jar_name:
+      continue
+    downloaded_custom_connector = format("{sqoop_lib}/{jar_name}")
+    custom_connector_to_remove = format("{sqoop_lib}/" + str(params.sqoop_jdbc_drivers_to_remove[jar_name]))
+    jdbc_driver_label = params.sqoop_jdbc_drivers_name_dict[jar_name]
+    driver_curl_source = format("{jdk_location}/{jar_name}")
+    environment = {
+      "no_proxy": format("{ambari_server_hostname}")
+    }
+    try:
+      if custom_connector_to_remove and os.path.isfile(custom_connector_to_remove):
+        File(custom_connector_to_remove, action='delete')
+
+      File(downloaded_custom_connector,
+           content = DownloadSource(driver_curl_source),
+           mode = 0644,
+      )
+    except HTTPError:
+      error_string = format("Could not download {driver_curl_source}\n\
+                 Please upload jdbc driver to server by run command:\n\
+                 ambari-server setup --jdbc-db={jdbc_driver_label} --jdbc-driver=<PATH
TO DRIVER>\n\
+                 at {ambari_server_hostname}") 
+      raise Fail(error_string)
+                 

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py
new file mode 100644
index 0000000..d420fab
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ClientComponentHasNoStatus
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from sqoop import sqoop
+from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_commons import OSConst
+
+class SqoopClient(Script):
+  def install(self, env):
+    self.install_packages(env)
+    self.configure(env)
+
+  def configure(self, env):
+    import params
+    env.set_params(params)
+    sqoop(type='client')
+
+  def status(self, env):
+    raise ClientComponentHasNoStatus()
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class SqoopClientDefault(SqoopClient):
+  def get_component_name(self):
+    return "sqoop-client"
+
+  def pre_upgrade_restart(self, env, upgrade_type=None):
+    import params
+    env.set_params(params)
+
+    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):

+      conf_select.select(params.stack_name, "sqoop", params.version)
+      stack_select.select("sqoop-client", params.version)
+
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class SqoopClientWindows(SqoopClient):
+  pass
+
+if __name__ == "__main__":
+  SqoopClient().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json
b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json
new file mode 100644
index 0000000..f9a6cb4
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json
@@ -0,0 +1,6 @@
+{
+  "general_deps" : {
+    "_comment" : "dependencies for SQOOP",
+    "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"]
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml
new file mode 100644
index 0000000..757bce5
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+    <schemaVersion>2.0</schemaVersion>
+    <services>
+        <service>
+            <name>SQOOP</name>
+            <version>1.4.4.3.0</version>
+            <extends>common-services/SQOOP/1.4.4.3.0</extends>
+        </service>
+    </services>
+</metainfo>


Mime
View raw message