ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject git commit: AMBARI-4017. Pig on HDP2. Using resource management lib (aonishuk)
Date Mon, 09 Dec 2013 16:04:48 GMT
Updated Branches:
  refs/heads/trunk 328c10e05 -> 71fa52975


AMBARI-4017. Pig on HDP2. Using resource management lib (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/71fa5297
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/71fa5297
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/71fa5297

Branch: refs/heads/trunk
Commit: 71fa52975f49262c855a5adcd202a503c0e3bdfa
Parents: 328c10e
Author: Andrew Onischuk <aonishuk@hortonworks.com>
Authored: Mon Dec 9 07:58:53 2013 -0800
Committer: Andrew Onischuk <aonishuk@hortonworks.com>
Committed: Mon Dec 9 08:04:35 2013 -0800

----------------------------------------------------------------------
 .../stacks/HDP/2.0._/services/PIG/metainfo.xml  | 46 +++++++++---
 .../services/PIG/package/files/pigSmoke.sh      | 18 +++++
 .../services/PIG/package/scripts/params.py      | 36 ++++++++++
 .../2.0._/services/PIG/package/scripts/pig.py   | 46 ++++++++++++
 .../services/PIG/package/scripts/pig_client.py  | 49 +++++++++++++
 .../PIG/package/scripts/service_check.py        | 75 ++++++++++++++++++++
 .../PIG/package/templates/log4j.properties.j2   | 30 ++++++++
 .../PIG/package/templates/pig-env.sh.j2         | 17 +++++
 .../PIG/package/templates/pig.properties.j2     | 55 ++++++++++++++
 9 files changed, 364 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/71fa5297/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/metainfo.xml
index 44e9cda..b491900 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/metainfo.xml
@@ -16,15 +16,45 @@
    limitations under the License.
 -->
 <metainfo>
-    <user>root</user>
-    <comment>Scripting platform for analyzing large datasets</comment>
-    <version>0.12.0.2.0.6.0</version>
-
-    <components>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>PIG</name>
+      <comment>Scripting platform for analyzing large datasets</comment>
+      <version>0.12.0.2.0.6.0</version>
+      <components>
         <component>
-            <name>PIG</name>
-            <category>CLIENT</category>
+          <name>PIG</name>
+          <category>CLIENT</category>
+          <commandScript>
+            <script>scripts/pig_client.py</script>
+            <scriptType>PYTHON</scriptType>
+            <timeout>600</timeout>
+          </commandScript>
         </component>
-    </components>
+      </components>
+      <osSpecifics>
+        <osSpecific>
+          <osType>centos6</osType>
+          <packages>
+            <package>
+              <type>rpm</type>
+              <name>pig</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+
+      <commandScript>
+        <script>scripts/service_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>300</timeout>
+      </commandScript>
+
+      <configuration-dependencies>
+        <config-type>global</config-type>
+      </configuration-dependencies>
 
+    </service>
+  </services>
 </metainfo>

http://git-wip-us.apache.org/repos/asf/ambari/blob/71fa5297/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/files/pigSmoke.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/files/pigSmoke.sh
b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/files/pigSmoke.sh
new file mode 100644
index 0000000..a22456e
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/files/pigSmoke.sh
@@ -0,0 +1,18 @@
+/*Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License */
+
+A = load 'passwd' using PigStorage(':');
+B = foreach A generate \$0 as id;
+store B into 'pigsmoke.out';

http://git-wip-us.apache.org/repos/asf/ambari/blob/71fa5297/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/params.py
new file mode 100644
index 0000000..9196d80
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/params.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python2.6
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+
+pig_conf_dir = "/etc/pig/conf"
+hadoop_conf_dir = "/etc/hadoop/conf"
+hdfs_user = config['configurations']['global']['hdfs_user']
+smokeuser = config['configurations']['global']['smokeuser']
+user_group = config['configurations']['global']['user_group']
+
+# not supporting 32 bit jdk.
+java64_home = config['configurations']['global']['java64_home']
+hadoop_home = "/usr"
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/71fa5297/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/pig.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/pig.py
b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/pig.py
new file mode 100644
index 0000000..c2d7b02
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/pig.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python2.6
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+
+def pig():
+  import params
+
+  Directory( params.pig_conf_dir,
+    owner = params.hdfs_user,
+    group = params.user_group
+  )
+
+  pig_TemplateConfig( ['pig-env.sh','pig.properties','log4j.properties'])
+  
+  
+def pig_TemplateConfig(name):
+  import params
+  
+  if not isinstance(name, list):
+    name = [name]
+    
+  for x in name:
+    TemplateConfig( format("{pig_conf_dir}/{x}"),
+        owner = params.hdfs_user
+    )
+  
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/71fa5297/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/pig_client.py
b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/pig_client.py
new file mode 100644
index 0000000..048f755
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/pig_client.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python2.6
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import sys
+from resource_management import *
+from pig import pig
+
+         
+class PigClient(Script):
+  def install(self, env):
+    self.install_packages(env)
+    self.configure(env)
+    
+  def configure(self, env):
+    import params
+    env.set_params(params)
+    pig()
+    
+#for tests
+def main():
+  command_type = 'install'
+  command_data_file = '/root/workspace/Pig/input.json'
+  basedir = '/root/workspace/Pig/main'
+  sys.argv = ["", command_type, command_data_file, basedir]
+  
+  PigClient().execute()
+  
+if __name__ == "__main__":
+  #main()
+  PigClient().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/71fa5297/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/service_check.py
b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/service_check.py
new file mode 100644
index 0000000..3cca087
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/scripts/service_check.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python2.6
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+
+class PigServiceCheck(Script):
+  def service_check(self, env):
+    import params
+    env.set_params(params)
+    
+    input_file = 'passwd'
+    output_file = "pigsmoke.out"
+  
+    cleanup_cmd = format("dfs -rmr {output_file} {input_file}")
+    #cleanup put below to handle retries; if retrying there wil be a stale file that needs
cleanup; exit code is fn of second command
+    create_file_cmd = format("{cleanup_cmd}; hadoop dfs -put /etc/passwd {input_file} ")
#TODO: inconsistent that second command needs hadoop
+    test_cmd = format("fs -test -e {output_file}")
+  
+    ExecuteHadoop( create_file_cmd,
+      tries     = 3,
+      try_sleep = 5,
+      user      = params.smokeuser,
+      conf_dir = params.hadoop_conf_dir
+    )
+  
+    File( '/tmp/pigSmoke.sh',
+      content = StaticFile("pigSmoke.sh"),
+      mode = 0755
+    )
+  
+    Execute( "pig /tmp/pigSmoke.sh",
+      tries     = 3,
+      try_sleep = 5,
+      path      = '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+      user      = params.smokeuser,
+      logoutput = True
+    )
+  
+    ExecuteHadoop( test_cmd,
+      user      = params.smokeuser,
+      conf_dir = params.hadoop_conf_dir
+    )
+    
+def main():
+  import sys
+  command_type = 'service_check'
+  command_data_file = '/root/workspace/Pig/input.json'
+  basedir = '/root/workspace/Pig/main'
+  sys.argv = ["", command_type, command_data_file, basedir]
+  
+  PigServiceCheck().execute()
+  
+if __name__ == "__main__":
+  #main()
+  PigServiceCheck().execute()
+  

http://git-wip-us.apache.org/repos/asf/ambari/blob/71fa5297/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/log4j.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/log4j.properties.j2
b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/log4j.properties.j2
new file mode 100644
index 0000000..9ef6e2c
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/log4j.properties.j2
@@ -0,0 +1,30 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+#
+
+# ***** Set root logger level to DEBUG and its only appender to A.
+log4j.logger.org.apache.pig=info, A
+
+# ***** A is set to be a ConsoleAppender.
+log4j.appender.A=org.apache.log4j.ConsoleAppender
+# ***** A uses PatternLayout.
+log4j.appender.A.layout=org.apache.log4j.PatternLayout
+log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n

http://git-wip-us.apache.org/repos/asf/ambari/blob/71fa5297/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/pig-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/pig-env.sh.j2
b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/pig-env.sh.j2
new file mode 100644
index 0000000..b0e17d4
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/pig-env.sh.j2
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+JAVA_HOME={{java64_home}}
+HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}

http://git-wip-us.apache.org/repos/asf/ambari/blob/71fa5297/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/pig.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/pig.properties.j2
b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/pig.properties.j2
new file mode 100644
index 0000000..6fcb233
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/PIG/package/templates/pig.properties.j2
@@ -0,0 +1,55 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+#
+
+# Pig configuration file. All values can be overwritten by command line arguments.
+
+# log4jconf log4j configuration file
+# log4jconf=./conf/log4j.properties
+
+# a file that contains pig script
+#file=
+
+# load jarfile, colon separated
+#jar=
+
+#verbose print all log messages to screen (default to print only INFO and above to screen)
+#verbose=true
+
+#exectype local|mapreduce, mapreduce is default
+#exectype=local
+
+#pig.logfile=
+
+#Do not spill temp files smaller than this size (bytes)
+#pig.spill.size.threshold=5000000
+#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
+#This should help reduce the number of files being spilled.
+#pig.spill.gc.activation.size=40000000
+
+#the following two parameters are to help estimate the reducer number
+#pig.exec.reducers.bytes.per.reducer=1000000000
+#pig.exec.reducers.max=999
+
+#Use this option only when your Pig job will otherwise die because of
+#using more counter than hadoop configured limit
+#pig.disable.counter=true
+hcat.bin=/usr/bin/hcat


Mime
View raw message