ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmar...@apache.org
Subject [04/58] [partial] ambari git commit: [RTC 136620]: Introduce BigInsights stacks on Ambari 2.4 branch
Date Wed, 17 Aug 2016 00:32:58 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SQOOP/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SQOOP/package/scripts/sqoop.py
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SQOOP/package/scripts/sqoop.py
new file mode 100644
index 0000000..587c414
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SQOOP/package/scripts/sqoop.py
@@ -0,0 +1,85 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.source import InlineTemplate, DownloadSource
+from resource_management.libraries.functions import format
+from resource_management.core.resources.system import File, Link, Directory
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from ambari_commons import OSConst
+import os
+
+def sqoop(type=None):
+  import params
+  Link(params.sqoop_lib + "/mysql-connector-java.jar",
+       to = '/usr/share/java/mysql-connector-java.jar'
+  ) 
+
+  jdbc_connector()
+
+  Directory(params.sqoop_conf_dir,
+            owner = params.sqoop_user,
+            group = params.user_group,
+            recursive = True
+  )
+  File(format("{sqoop_conf_dir}/sqoop-env.sh"),
+    owner=params.sqoop_user,
+    group = params.user_group,
+    content=InlineTemplate(params.sqoop_env_sh_template)
+  )
+  update_config_permissions(["sqoop-env-template.sh",
+                             "sqoop-site-template.xml",
+                             "sqoop-site.xml"])
+  pass
+
+def update_config_permissions(names):
+  import params
+  for filename in names:
+    full_filename = os.path.join(params.sqoop_conf_dir, filename)
+    File(full_filename,
+         owner = params.sqoop_user,
+         group = params.user_group,
+         only_if = format("test -e {full_filename}")
+    )
+
+def jdbc_connector():
+  import params
+  from urllib2 import HTTPError
+  from resource_management import Fail
+  for jar_name in params.sqoop_jdbc_drivers_dict:
+    if 'mysql-connector-java.jar' in jar_name:
+      continue
+    downloaded_custom_connector = format("{sqoop_lib}/{jar_name}")
+    jdbc_symlink_remote = params.sqoop_jdbc_drivers_dict[jar_name]
+    jdbc_driver_label = params.sqoop_jdbc_drivers_name_dict[jar_name]
+    driver_curl_source = format("{jdk_location}/{jdbc_symlink_remote}")
+    environment = {
+      "no_proxy": format("{ambari_server_hostname}")
+    }
+    try:
+      File(downloaded_custom_connector,
+           content = DownloadSource(driver_curl_source),
+           mode = 0644,
+      )
+    except HTTPError:
+      error_string = format("Could not download {driver_curl_source}\n\
+                 Please upload jdbc driver to server by run command:\n\
+                 ambari-server setup --jdbc-db={jdbc_driver_label} --jdbc-driver=<PATH
TO DRIVER>\n\
+                 at {ambari_server_hostname}") 
+      raise Fail(error_string)
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SQOOP/package/scripts/sqoop_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SQOOP/package/scripts/sqoop_client.py
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SQOOP/package/scripts/sqoop_client.py
new file mode 100644
index 0000000..e0428e3
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/SQOOP/package/scripts/sqoop_client.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import iop_select
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+
+from sqoop import sqoop
+
+
+class SqoopClient(Script):
+
+  def get_stack_to_component(self):
+    return {"BigInsights": "sqoop-client"}
+
+  def pre_rolling_restart(self, env):
+    import params
+    env.set_params(params)
+
+    if params.version and compare_versions(format_hdp_stack_version(params.version), '4.0.0.0')
>= 0:
+      conf_select.select(params.stack_name, "sqoop", params.version)
+      iop_select.select("sqoop-client", params.version)
+      #Execute(format("iop-select set sqoop-client {version}"))
+
+  def install(self, env):
+    self.install_packages(env)
+    self.configure(env)
+
+  def configure(self, env):
+    import params
+    env.set_params(params)
+    sqoop(type='client')
+
+  def status(self, env):
+    raise ClientComponentHasNoStatus()
+
+if __name__ == "__main__":
+  SqoopClient().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-env.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-env.xml
new file mode 100644
index 0000000..88bcc48
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-env.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="false">
+
+  <property>
+    <name>titan_user</name>
+    <description>User to run Titan as</description>
+    <property-type>USER</property-type>
+    <value>titan</value>
+  </property>
+
+  <property>
+    <name>content</name>
+    <description>This is the template for titan-env.sh file</description>
+    <value>
+# Set JAVA HOME
+export JAVA_HOME={{java64_home}}
+
+# Add hadoop and hbase configuration directories into classpath
+export HADOOP_CONF_DIR={{hadoop_config_dir}}
+export HBASE_CONF_DIR={{hbase_config_dir}}
+CLASSPATH=$HADOOP_CONF_DIR:$HBASE_CONF_DIR:$CLASSPATH
+    </value>
+  </property>
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-hbase-solr.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-hbase-solr.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-hbase-solr.xml
new file mode 100644
index 0000000..e167835
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-hbase-solr.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="false" supports_adding_forbidden="true">
+
+  <property>
+    <name>content</name>
+    <description>Describe the configurations for Solr</description>
+    <value># Titan configuration sample: HBase and Solr
+# ATTENTION: If you would like to use this property, do manually execute titan-solr-connection.sh
before build index.
+
+# This file connects to HBase using a Zookeeper quorum
+# (storage.hostname) consisting solely of localhost. It also
+# connects to Solr running on localhost using Solr's HTTP API.
+# Zookeeper, the HBase services, and Solr must already be
+# running and available before starting Titan with this file.
+storage.backend=hbase
+storage.hostname={{storage_host}}
+storage.hbase.table=titan_solr
+storage.hbase.ext.zookeeper.znode.parent={{hbase_zookeeper_parent}}
+
+cache.db-cache = true
+cache.db-cache-clean-wait = 20
+cache.db-cache-time = 180000
+cache.db-cache-size = 0.5
+
+# The indexing backend used to extend and optimize Titan's query
+# functionality. This setting is optional. Titan can use multiple
+# heterogeneous index backends. Hence, this option can appear more than
+# once, so long as the user-defined name between "index" and "backend" is
+# unique among appearances.Similar to the storage backend, this should be
+# set to one of Titan's built-in shorthand names for its standard index
+# backends (shorthands: lucene, elasticsearch, es, solr) or to the full
+# package and classname of a custom/third-party IndexProvider
+# implementation.
+
+index.search.backend=solr
+index.search.solr.mode=cloud
+index.search.solr.zookeeper-url={{solr_server_host}}/solr
+index.search.solr.configset=titan
+    </value>
+    <value-attributes>
+      <show-property-name>false</show-property-name>
+    </value-attributes>
+  </property>
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-log4j.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-log4j.xml
new file mode 100644
index 0000000..c168699
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/configuration/titan-log4j.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="false">
+
+  <property>
+    <name>content</name>
+    <description>Custom log4j-console.properties</description>
+    <value>
+      # Used by gremlin.sh
+
+      log4j.appender.A2=org.apache.log4j.ConsoleAppender
+      log4j.appender.A2.Threshold=TRACE
+      log4j.appender.A2.layout=org.apache.log4j.PatternLayout
+      log4j.appender.A2.layout.ConversionPattern=%d{HH:mm:ss} %-5p %c %x - %m%n
+
+      log4j.rootLogger=${gremlin.log4j.level}, A2
+
+      #log4j.logger.com.thinkaurelius.titan.graphdb.database.idassigner.placement=DEBUG
+      #log4j.logger.com.thinkaurelius.titan.diskstorage.hbase.HBaseStoreManager=DEBUG
+
+      # Disable spurious Hadoop config deprecation warnings under 2.2.0.
+      #
+      # See https://issues.apache.org/jira/browse/HADOOP-10178
+      #
+      # This can and should be deleted when we upgrade our Hadoop 2.2.0
+      # dependency to 2.3.0 or 3.0.0.
+      log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=OFF
+
+      # Configure MR at its own loglevel. We usually want MR at INFO,
+      # even if the rest of the loggers are at WARN or ERROR or FATAL,
+      # because job progress information is at INFO.
+      log4j.logger.org.apache.hadoop.mapred=${gremlin.mr.log4j.level}
+      log4j.logger.org.apache.hadoop.mapreduce=${gremlin.mr.log4j.level}
+
+      # This generates 3 INFO lines per jar on the classpath -- usually more
+      # noise than desirable in the REPL. Switching it to the default
+      # log4j level means it will be at WARN by default, which is ideal.
+      log4j.logger.org.apache.hadoop.mapred.LocalDistributedCacheManager=${gremlin.log4j.level}
+    </value>
+    <value-attributes>
+      <show-property-name>false</show-property-name>
+    </value-attributes>
+  </property>
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/kerberos.json
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/kerberos.json
new file mode 100644
index 0000000..ccabbf0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/kerberos.json
@@ -0,0 +1,17 @@
+{
+  "services": [
+    {
+      "name": "TITAN",
+      "components": [
+        {
+          "name": "TITAN",
+          "identities": [
+            {
+              "name": "/HDFS/NAMENODE/hdfs"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/metainfo.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/metainfo.xml
new file mode 100644
index 0000000..73f4635
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/metainfo.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>TITAN</name>
+      <displayName>Titan</displayName>
+      <comment>Titan is a scalable graph database optimized for storing and querying
graphs containing hundreds of
+        billions of vertices and edges distributed across a multi-machine cluster.</comment>
+      <version>1.0.0</version>
+      <components>
+        <component>
+          <name>TITAN</name>
+          <displayName>Titan</displayName>
+          <category>CLIENT</category>
+          <cardinality>0+</cardinality>
+          <versionAdvertised>true</versionAdvertised>
+          <commandScript>
+            <script>scripts/titan_client.py</script>
+            <scriptType>PYTHON</scriptType>
+	        <timeout>600</timeout>
+          </commandScript>
+          <configFiles>
+            <configFile>
+               <type>env</type>
+               <fileName>titan-env.sh</fileName>
+               <dictionaryName>titan-env</dictionaryName>
+            </configFile>
+            <configFile>
+                <type>env</type>
+                <fileName>log4j-console.properties</fileName>
+                <dictionaryName>titan-log4j</dictionaryName>
+            </configFile>
+            <configFile>
+                <type>env</type>
+                <fileName>titan-hbase-solr.properties</fileName>
+                <dictionaryName>titan-hbase-solr</dictionaryName>
+            </configFile>
+          </configFiles>
+        </component>
+      </components>
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>any</osFamily>
+          <packages>
+            <package>
+              <name>titan_4_2_*</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+
+      <commandScript>
+        <script>scripts/service_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>300</timeout>
+      </commandScript>
+
+      <requiredServices>		
+        <service>HDFS</service>
+        <service>HBASE</service>
+        <service>SOLR</service>
+      </requiredServices>
+      
+      <configuration-dependencies>
+        <config-type>titan-env</config-type>
+        <config-type>titan-hbase-solr</config-type>
+        <config-type>titan-log4j</config-type>
+      </configuration-dependencies>
+    </service>
+  </services>
+</metainfo>

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/metainfo.xml~
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/metainfo.xml~
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/metainfo.xml~
new file mode 100644
index 0000000..88f7105
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/metainfo.xml~
@@ -0,0 +1,88 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>TITAN</name>
+      <displayName>Titan</displayName>
+      <comment>Titan is a scalable graph database optimized for storing and querying
graphs containing hundreds of
+        billions of vertices and edges distributed across a multi-machine cluster.</comment>
+      <version>1.0.0.4.2</version>
+      <components>
+        <component>
+          <name>TITAN</name>
+          <displayName>Titan</displayName>
+          <category>CLIENT</category>
+          <cardinality>0+</cardinality>
+          <versionAdvertised>true</versionAdvertised>
+          <commandScript>
+            <script>scripts/titan_client.py</script>
+            <scriptType>PYTHON</scriptType>
+	        <timeout>600</timeout>
+          </commandScript>
+          <configFiles>
+            <configFile>
+               <type>env</type>
+               <fileName>titan-env.sh</fileName>
+               <dictionaryName>titan-env</dictionaryName>
+            </configFile>
+            <configFile>
+                <type>env</type>
+                <fileName>log4j-console.properties</fileName>
+                <dictionaryName>titan-log4j</dictionaryName>
+            </configFile>
+            <configFile>
+                <type>env</type>
+                <fileName>titan-hbase-solr.properties</fileName>
+                <dictionaryName>titan-hbase-solr</dictionaryName>
+            </configFile>
+          </configFiles>
+        </component>
+      </components>
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>any</osFamily>
+          <packages>
+            <package>
+              <name>titan_4_2_*</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+
+      <commandScript>
+        <script>scripts/service_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>300</timeout>
+      </commandScript>
+
+      <requiredServices>		
+        <service>HDFS</service>
+        <service>HBASE</service>
+        <service>SOLR</service>
+      </requiredServices>
+      
+      <configuration-dependencies>
+        <config-type>titan-env</config-type>
+        <config-type>titan-hbase-solr</config-type>
+        <config-type>titan-log4j</config-type>
+      </configuration-dependencies>
+    </service>
+  </services>
+</metainfo>

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/files/titanSmoke.groovy
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/files/titanSmoke.groovy
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/files/titanSmoke.groovy
new file mode 100644
index 0000000..79438be
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/files/titanSmoke.groovy
@@ -0,0 +1,20 @@
+/*Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License */
+
+import com.thinkaurelius.titan.core.TitanFactory;
+
+graph = TitanFactory.open("/etc/titan/conf/titan-hbase-solr.properties")
+g = graph.traversal()
+l = g.V().values('name').toList()

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/params.py
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/params.py
new file mode 100644
index 0000000..1c8cf81
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/params.py
@@ -0,0 +1,128 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import iop_select
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.version import format_hdp_stack_version
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import get_kinit_path
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+stack_name = default("/hostLevelParams/stack_name", None)
+
+stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+iop_stack_version = format_hdp_stack_version(stack_version_unformatted)
+
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
+titan_user = config['configurations']['titan-env']['titan_user']
+user_group = config['configurations']['cluster-env']['user_group']
+titan_bin_dir = '/usr/iop/current/titan-client/bin'
+
+
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
+
+# titan configurations
+titan_conf_dir = "/usr/iop/current/titan-client/conf"
+titan_hbase_solr_props = config['configurations']['titan-hbase-solr']['content']
+titan_env_props = config['configurations']['titan-env']['content']
+log4j_console_props = config['configurations']['titan-log4j']['content']
+
+# not supporting 32 bit jdk.
+java64_home = config['hostLevelParams']['java_home']
+hadoop_config_dir = '/etc/hadoop/conf'
+hbase_config_dir = '/etc/hbase/conf'
+
+# Titan required 'storage.hostname' which is hbase cluster in IOP 4.2.
+# The host name should be zooKeeper quorum
+storage_hosts = config['clusterHostInfo']['zookeeper_hosts']
+storage_host_list = []
+for hostname in storage_hosts:
+  storage_host_list.append(hostname)
+storage_host = ",".join(storage_host_list)
+hbase_zookeeper_parent = config['configurations']['hbase-site']['zookeeper.znode.parent']
+
+# Solr cloud host
+solr_hosts = config['clusterHostInfo']['solr_hosts']
+solr_host_list = []
+for hostname in solr_hosts:
+  solr_host_list.append(hostname)
+solr_host = ",".join(solr_host_list)
+solr_server_host = solr_hosts[0]
+
+# Titan client, it does not work right now, there is no 'titan_host' in 'clusterHostInfo'
+# It will return "Configuration parameter 'titan_host' was not found in configurations dictionary!"
+# So here is a known issue as task 118900, will install titan and solr on same node right
now.
+# titan_host = config['clusterHostInfo']['titan_host']
+titan_host = solr_server_host
+
+# Conf directory and jar should be copy to solr site
+titan_dir = format('/usr/iop/current/titan-client')
+titan_ext_dir = format('/usr/iop/current/titan-client/ext')
+titan_solr_conf_dir = format('/usr/iop/current/titan-client/conf/solr')
+titan_solr_jar_file = format('/usr/iop/current/titan-client/lib/jts-1.13.jar')
+
+titan_solr_hdfs_dir = "/apps/titan"
+titan_solr_hdfs_conf_dir = "/apps/titan/conf"
+titan_solr_hdfs_jar = "/apps/titan/jts-1.13.jar"
+titan_tmp_dir = format('{tmp_dir}/titan')
+titan_solr_dir = format('{titan_tmp_dir}/solr_installed')
+configuration_tags = config['configurationTags']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+titan_hdfs_mode = 0775
+
+#for create_hdfs_directory
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+kinit_path_local = get_kinit_path()
+hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hdfs_site = config['configurations']['hdfs-site']
+hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', 'missing_principal').replace("_HOST",
hostname)
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+
+import functools
+#to create hdfs directory we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+  HdfsResource,
+  user = hdfs_user,
+  security_enabled = security_enabled,
+  keytab = hdfs_user_keytab,
+  kinit_path_local = kinit_path_local,
+  hadoop_bin_dir = hadoop_bin_dir,
+  hadoop_conf_dir = hadoop_conf_dir,
+  principal_name = hdfs_principal_name,
+  hdfs_site = hdfs_site,
+  default_fs = default_fs
+)
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/service_check.py
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/service_check.py
new file mode 100644
index 0000000..3c011a1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/service_check.py
@@ -0,0 +1,64 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import os
+from resource_management import *
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import InlineTemplate, StaticFile
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.validate import call_and_match_output
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+class TitanServiceCheck(Script):
+    pass
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class TitanServiceCheckLinux(TitanServiceCheck):
+    def service_check(self, env):
+        import params
+        env.set_params(params)
+
+        File( format("{tmp_dir}/titanSmoke.groovy"),
+              content = StaticFile("titanSmoke.groovy"),
+              mode = 0755
+              )
+
+        if params.iop_stack_version != "" and compare_versions(params.iop_stack_version,
'4.2') >= 0:
+            if params.security_enabled:
+                kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
+                Execute(kinit_cmd,
+                        user=params.smokeuser
+                        )
+
+            Execute(format("gremlin {tmp_dir}/titanSmoke.groovy"),
+                    tries     = 3,
+                    try_sleep = 5,
+                    path      = format('{titan_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
+                    user      = params.smokeuser,
+                    logoutput = True
+                    )
+
+if __name__ == "__main__":
+    # print "Track service check status"
+    TitanServiceCheckLinux().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/titan.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/titan.py
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/titan.py
new file mode 100644
index 0000000..1d641dc
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/titan.py
@@ -0,0 +1,70 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import os
+from resource_management import *
+from resource_management.core.source import InlineTemplate
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def titan():
+    import params
+
+    Directory(params.titan_conf_dir,
+               recursive=True,
+               owner=params.titan_user,
+               group=params.user_group
+               )
+
+    File(format("{params.titan_conf_dir}/titan-env.sh"),
+             mode=0644,
+             group=params.user_group,
+             owner=params.titan_user,
+             content=InlineTemplate(params.titan_env_props)
+             )
+
+    # titan-hbase-solr_properties is always set to a default even if it's not in the payload
+    File(format("{params.titan_conf_dir}/titan-hbase-solr.properties"),
+         mode=0644,
+         group=params.user_group,
+         owner=params.titan_user,
+         content=InlineTemplate(params.titan_hbase_solr_props)
+         )
+
+    if (params.log4j_console_props != None):
+        File(format("{params.titan_conf_dir}/log4j-console.properties"),
+             mode=0644,
+             group=params.user_group,
+             owner=params.titan_user,
+             content=InlineTemplate(params.log4j_console_props)
+             )
+    elif (os.path.exists(format("{params.titan_conf_dir}/log4j-console.properties"))):
+        File(format("{params.titan_conf_dir}/log4j-console.properties"),
+             mode=0644,
+             group=params.user_group,
+             owner=params.titan_user
+             )
+    # Change titan ext directory for multiple user access
+    Directory(params.titan_ext_dir,
+               recursive=True,
+               owner=params.titan_user,
+               group=params.user_group,
+               mode=0775
+               )

http://git-wip-us.apache.org/repos/asf/ambari/blob/66984d9a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/titan_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/titan_client.py
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/titan_client.py
new file mode 100644
index 0000000..5c16fcc
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/TITAN/package/scripts/titan_client.py
@@ -0,0 +1,58 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import sys
+import os
+from resource_management import *
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import iop_select
+from titan import titan
+
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+class TitanClient(Script):
+    def configure(self, env):
+        import params
+        env.set_params(params)
+        titan()
+
+    def status(self, env):
+        raise ClientComponentHasNoStatus()
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class TitanClientLinux(TitanClient):
+    def get_stack_to_component(self):
+        return {"BigInsights": "titan-client"}
+
+    def pre_rolling_restart(self, env):
+        import params
+        env.set_params(params)
+
+        if params.version and compare_versions(format_hdp_stack_version(params.version),
'4.2.0.0') >= 0:
+            conf_select.select(params.stack_name, "titan", params.version)
+            iop_select.select("titan-client", params.version)
+
+    def install(self, env):
+        self.install_packages(env)
+        self.configure(env)
+
+if __name__ == "__main__":
+    TitanClient().execute()


Mime
View raw message