ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [06/50] ambari git commit: AMBARI-20652. HDP 3.0 TP - add Pig client: configs, metainfo, descriptors, service advisor and etc.(vbrodetskyi)
Date Thu, 06 Apr 2017 18:57:55 GMT
AMBARI-20652. HDP 3.0 TP - add Pig client: configs, metainfo, descriptors, service advisor
and etc.(vbrodetskyi)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9fec5ac5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9fec5ac5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9fec5ac5

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 9fec5ac5ae9208f18d9e4180d8482dc14f05c8b3
Parents: 0ffb42b
Author: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Authored: Mon Apr 3 15:37:51 2017 +0300
Committer: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Committed: Mon Apr 3 15:38:44 2017 +0300

----------------------------------------------------------------------
 .../PIG/0.16.1.3.0/configuration/pig-env.xml    |  41 +++++
 .../PIG/0.16.1.3.0/configuration/pig-log4j.xml  |  65 +++++++
 .../0.16.1.3.0/configuration/pig-properties.xml |  93 ++++++++++
 .../PIG/0.16.1.3.0/kerberos.json                |  17 ++
 .../common-services/PIG/0.16.1.3.0/metainfo.xml | 101 +++++++++++
 .../PIG/0.16.1.3.0/package/files/pigSmoke.sh    |  18 ++
 .../PIG/0.16.1.3.0/package/scripts/params.py    |  31 ++++
 .../0.16.1.3.0/package/scripts/params_linux.py  | 104 +++++++++++
 .../package/scripts/params_windows.py           |  45 +++++
 .../PIG/0.16.1.3.0/package/scripts/pig.py       |  80 +++++++++
 .../0.16.1.3.0/package/scripts/pig_client.py    |  71 ++++++++
 .../0.16.1.3.0/package/scripts/service_check.py | 142 +++++++++++++++
 .../PIG/0.16.1.3.0/service_advisor.py           | 171 +++++++++++++++++++
 .../stacks/HDP/3.0/services/PIG/metainfo.xml    |  27 +++
 14 files changed, 1006 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml
new file mode 100644
index 0000000..11afa11
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+  <!-- pig-env.sh -->
+  <property>
+    <name>content</name>
+    <display-name>pig-env template</display-name>
+    <description>This is the jinja template for pig-env.sh file</description>
+    <value>
+JAVA_HOME={{java64_home}}
+HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+
+if [ -d "/usr/lib/tez" ]; then
+  PIG_OPTS="$PIG_OPTS -Dmapreduce.framework.name=yarn"
+fi
+    </value>
+    <value-attributes>
+      <type>content</type>
+    </value-attributes>
+    <on-ambari-upgrade add="fallse"/>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml
new file mode 100644
index 0000000..e5ff9b6
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+  <property>
+    <name>content</name>
+    <display-name>pig-log4j template</display-name>
+    <description>Custom log4j.properties</description>
+    <value>
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+#
+
+# ***** Set root logger level to DEBUG and its only appender to A.
+log4j.logger.org.apache.pig=info, A
+
+# ***** A is set to be a ConsoleAppender.
+log4j.appender.A=org.apache.log4j.ConsoleAppender
+# ***** A uses PatternLayout.
+log4j.appender.A.layout=org.apache.log4j.PatternLayout
+log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+    </value>
+    <value-attributes>
+      <type>content</type>
+      <show-property-name>false</show-property-name>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml
new file mode 100644
index 0000000..fb43f6e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+  <property>
+    <name>content</name>
+    <display-name>pig-properties template</display-name>
+    <description>Describe all the Pig agent configurations</description>
+    <value>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pig default configuration file. All values can be overwritten by pig.properties and command
line arguments.
+# see bin/pig -help
+
+# brief logging (no timestamps)
+brief=false
+
+# debug level, INFO is default
+debug=INFO
+
+# verbose print all log messages to screen (default to print only INFO and above to screen)
+verbose=false
+
+# exectype local|mapreduce|tez, mapreduce is default
+exectype=tez
+
+# Enable insertion of information about script into hadoop job conf
+pig.script.info.enabled=true
+
+# Do not spill temp files smaller than this size (bytes)
+pig.spill.size.threshold=5000000
+
+# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
+# This should help reduce the number of files being spilled.
+pig.spill.gc.activation.size=40000000
+
+# the following two parameters are to help estimate the reducer number
+pig.exec.reducers.bytes.per.reducer=1000000000
+pig.exec.reducers.max=999
+
+# Temporary location to store the intermediate data.
+pig.temp.dir=/tmp/
+
+# Threshold for merging FRJoin fragment files
+pig.files.concatenation.threshold=100
+pig.optimistic.files.concatenation=false;
+
+pig.disable.counter=false
+
+hcat.bin=/usr/bin/hcat
+
+    </value>
+    <value-attributes>
+      <type>content</type>
+      <empty-value-valid>true</empty-value-valid>
+      <show-property-name>false</show-property-name>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json
new file mode 100644
index 0000000..22dd6cb
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json
@@ -0,0 +1,17 @@
+{
+  "services": [
+    {
+      "name": "PIG",
+      "components": [
+        {
+          "name": "PIG",
+          "identities": [
+            {
+              "name": "/HDFS/NAMENODE/hdfs"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml
new file mode 100644
index 0000000..041255a
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml
@@ -0,0 +1,101 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>PIG</name>
+      <displayName>Pig</displayName>
+      <comment>Scripting platform for analyzing large datasets</comment>
+      <version>0.16.1.3.0</version>
+      <components>
+        <component>
+          <name>PIG</name>
+          <displayName>Pig Client</displayName>
+          <category>CLIENT</category>
+          <cardinality>0+</cardinality>
+          <versionAdvertised>true</versionAdvertised>
+          <commandScript>
+            <script>scripts/pig_client.py</script>
+            <scriptType>PYTHON</scriptType>
+            <timeout>1200</timeout>
+          </commandScript>
+          <configFiles>
+            <configFile>
+              <type>env</type>
+              <fileName>pig-env.sh</fileName>
+              <dictionaryName>pig-env</dictionaryName>
+            </configFile>
+            <configFile>
+              <type>env</type>
+              <fileName>log4j.properties</fileName>
+              <dictionaryName>pig-log4j</dictionaryName>
+            </configFile>
+            <configFile>
+              <type>env</type>
+              <fileName>pig.properties</fileName>
+              <dictionaryName>pig-properties</dictionaryName>
+            </configFile>                         
+          </configFiles>          
+        </component>
+      </components>
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily>
+          <packages>
+            <package>
+              <name>pig_${stack_version}</name>
+            </package>
+            <package>
+              <name>datafu_${stack_version}</name>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily>
+          <packages>
+            <package>
+              <name>pig-${stack_version}</name>
+            </package>
+            <package>
+              <name>datafu-${stack_version}</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+
+      <commandScript>
+        <script>scripts/service_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>300</timeout>
+      </commandScript>
+
+      <requiredServices>
+        <service>YARN</service>
+        <service>TEZ</service>
+      </requiredServices>
+
+      <configuration-dependencies>
+        <config-type>pig-env</config-type>
+        <config-type>pig-log4j</config-type>
+        <config-type>pig-properties</config-type>
+      </configuration-dependencies>
+
+    </service>
+  </services>
+</metainfo>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh
new file mode 100644
index 0000000..a22456e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh
@@ -0,0 +1,18 @@
+/*Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License */
+
+A = load 'passwd' using PigStorage(':');
+B = foreach A generate \$0 as id;
+store B into 'pigsmoke.out';

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py
new file mode 100644
index 0000000..3aebda0
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py
@@ -0,0 +1,31 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.copy_tarball import get_sysprep_skip_copy_tarballs_hdfs
+
+if OSCheck.is_windows_family():
+  from params_windows import *
+else:
+  from params_linux import *
+
+sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs()
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py
new file mode 100644
index 0000000..5af1274
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py
@@ -0,0 +1,104 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+stack_name = default("/hostLevelParams/stack_name", None)
+stack_root = Script.get_stack_root()
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
+# hadoop default parameters
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+pig_conf_dir = "/etc/pig/conf"
+hadoop_home = '/usr'
+pig_bin_dir = ""
+
+# hadoop parameters for stack versions supporting rolling_upgrade
+if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+  pig_conf_dir = format("{stack_root}/current/pig-client/conf")
+  hadoop_home = stack_select.get_hadoop_dir("home")
+  pig_bin_dir = format("{stack_root}/current/pig-client/bin")
+
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_hdfs_user_mode = 0770
+user_group = config['configurations']['cluster-env']['user_group']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
+pig_env_sh_template = config['configurations']['pig-env']['content']
+
+# not supporting 32 bit jdk.
+java64_home = config['hostLevelParams']['java_home']
+
+pig_properties = config['configurations']['pig-properties']['content']
+
+log4j_props = config['configurations']['pig-log4j']['content']
+
+
+
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+
+dfs_type = default("/commandParams/dfs_type", "")
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create hdfs directory we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+  HdfsResource,
+  user=hdfs_user,
+  hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+  security_enabled = security_enabled,
+  keytab = hdfs_user_keytab,
+  kinit_path_local = kinit_path_local,
+  hadoop_bin_dir = hadoop_bin_dir,
+  hadoop_conf_dir = hadoop_conf_dir,
+  principal_name = hdfs_principal_name,
+  hdfs_site = hdfs_site,
+  default_fs = default_fs,
+  immutable_paths = get_not_managed_resources(),
+  dfs_type = dfs_type
+ )
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py
new file mode 100644
index 0000000..bd9654f
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py
@@ -0,0 +1,45 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+from resource_management.libraries.script.script import Script
+
+# server configurations
+config = Script.get_config()
+
+stack_root = None
+pig_home = None
+pig_conf_dir = None
+try:
+  stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
+  pig_home = os.environ['PIG_HOME']
+  pig_conf_dir = os.path.join(pig_home,'conf')
+except:
+  pass
+
+pig_properties = config['configurations']['pig-properties']['content']
+
+if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])):
+  log4j_props = config['configurations']['pig-log4j']['content']
+else:
+  log4j_props = None
+
+hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
+pig_user = hadoop_user
+hdfs_user = hadoop_user

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py
new file mode 100644
index 0000000..b6825d5
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py
@@ -0,0 +1,80 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import os
+from resource_management.core.resources.system import Directory, File
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions.format import format
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def pig():
+  import params
+
+  Directory( params.pig_conf_dir,
+    create_parents = True,
+    owner = params.hdfs_user,
+    group = params.user_group
+  )
+
+  File(format("{pig_conf_dir}/pig-env.sh"),
+    owner=params.hdfs_user,
+    mode=0755,
+    content=InlineTemplate(params.pig_env_sh_template)
+  )
+
+  # pig_properties is always set to a default even if it's not in the payload
+  File(format("{params.pig_conf_dir}/pig.properties"),
+              mode=0644,
+              group=params.user_group,
+              owner=params.hdfs_user,
+              content=params.pig_properties
+  )
+
+  if (params.log4j_props != None):
+    File(format("{params.pig_conf_dir}/log4j.properties"),
+      mode=0644,
+      group=params.user_group,
+      owner=params.hdfs_user,
+      content=params.log4j_props
+    )
+  elif (os.path.exists(format("{params.pig_conf_dir}/log4j.properties"))):
+    File(format("{params.pig_conf_dir}/log4j.properties"),
+      mode=0644,
+      group=params.user_group,
+      owner=params.hdfs_user
+    )
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def pig():
+  import params
+  File(os.path.join(params.pig_conf_dir, "pig.properties"),
+       mode="f",
+       owner=params.pig_user,
+       content=params.pig_properties
+  )
+
+  if (params.log4j_props != None):
+    File(os.path.join(params.pig_conf_dir, "log4j.properties"),
+         mode='f',
+         owner=params.pig_user,
+         content=params.log4j_props
+    )
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py
new file mode 100644
index 0000000..6fdcef8
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py
@@ -0,0 +1,71 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import sys
+import os
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import conf_select, stack_select
+from resource_management.libraries.functions.constants import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from pig import pig
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from resource_management.libraries.functions.default import default
+from resource_management.core.exceptions import ClientComponentHasNoStatus
+
+class PigClient(Script):
+  def configure(self, env):
+    import params
+    env.set_params(params)
+    pig()
+
+  def status(self, env):
+    raise ClientComponentHasNoStatus()
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class PigClientLinux(PigClient):
+  def get_component_name(self):
+    return "hadoop-client"
+
+  def pre_upgrade_restart(self, env, upgrade_type=None):
+    import params
+    env.set_params(params)
+
+    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):

+      conf_select.select(params.stack_name, "pig", params.version)
+      conf_select.select(params.stack_name, "hadoop", params.version)
+      stack_select.select("hadoop-client", params.version) # includes pig-client
+
+  def install(self, env):
+    self.install_packages(env)
+    self.configure(env)
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class PigClientWindows(PigClient):
+
+  def install(self, env):
+    import params
+    if params.pig_home is None:
+      self.install_packages(env)
+    self.configure(env)
+
+if __name__ == "__main__":
+  PigClient().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py
new file mode 100644
index 0000000..ba4082e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py
@@ -0,0 +1,142 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import os
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import InlineTemplate, StaticFile
+from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.script.script import Script
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+class PigServiceCheck(Script):
+  pass
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class PigServiceCheckLinux(PigServiceCheck):
+  def service_check(self, env):
+    import params
+    env.set_params(params)
+
+    input_file = format('/user/{smokeuser}/passwd')
+    output_dir = format('/user/{smokeuser}/pigsmoke.out')
+
+    params.HdfsResource(format("/user/{smokeuser}"),
+                        type="directory",
+                        action="create_on_execute",
+                        owner=params.smokeuser,
+                        mode=params.smoke_hdfs_user_mode,
+                        )
+
+    params.HdfsResource(output_dir,
+                        type="directory",
+                        action="delete_on_execute",
+                        owner=params.smokeuser,
+                        )
+    params.HdfsResource(input_file,
+                        type="file",
+                        source="/etc/passwd",
+                        action="create_on_execute",
+                        owner=params.smokeuser,
+    )
+    params.HdfsResource(None, action="execute")
+ 
+    if params.security_enabled:
+      kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
+      Execute(kinit_cmd,
+        user=params.smokeuser
+      )
+
+    File( format("{tmp_dir}/pigSmoke.sh"),
+      content = StaticFile("pigSmoke.sh"),
+      mode = 0755
+    )
+
+    # check for Pig-on-M/R
+    Execute( format("pig {tmp_dir}/pigSmoke.sh"),
+      tries     = 3,
+      try_sleep = 5,
+      path      = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
+      user      = params.smokeuser,
+      logoutput = True
+    )
+
+    test_cmd = format("fs -test -e {output_dir}")
+    ExecuteHadoop( test_cmd,
+      user      = params.smokeuser,
+      conf_dir = params.hadoop_conf_dir,
+      bin_dir = params.hadoop_bin_dir
+    )
+
+    if params.stack_version_formatted and check_stack_feature(StackFeature.PIG_ON_TEZ, params.stack_version_formatted):
+      # cleanup results from previous test
+      params.HdfsResource(output_dir,
+                          type="directory",
+                          action="delete_on_execute",
+                          owner=params.smokeuser,
+      )
+      params.HdfsResource(input_file,
+                          type="file",
+                          source="/etc/passwd",
+                          action="create_on_execute",
+                          owner=params.smokeuser,
+      )
+
+      # Check for Pig-on-Tez
+      resource_created = copy_to_hdfs(
+        "tez", params.user_group,
+        params.hdfs_user,
+        skip=params.sysprep_skip_copy_tarballs_hdfs)
+      if resource_created:
+        params.HdfsResource(None, action="execute")
+
+      Execute(format("pig -x tez {tmp_dir}/pigSmoke.sh"),
+        tries     = 3,
+        try_sleep = 5,
+        path      = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
+        user      = params.smokeuser,
+        logoutput = True
+      )
+
+      ExecuteHadoop(test_cmd,
+        user      = params.smokeuser,
+        conf_dir = params.hadoop_conf_dir,
+        bin_dir = params.hadoop_bin_dir
+      )
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class PigServiceCheckWindows(PigServiceCheck):
+  def service_check(self, env):
+    import params
+    env.set_params(params)
+    smoke_cmd = os.path.join(params.stack_root,"Run-SmokeTests.cmd")
+    service = "PIG"
+    Execute(format("cmd /C {smoke_cmd} {service}", smoke_cmd=smoke_cmd, service=service),
logoutput=True, user=params.pig_user, timeout=300)
+
+if __name__ == "__main__":
+  PigServiceCheck().execute()
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py
new file mode 100644
index 0000000..8f89c5d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# Python imports
+import imp
+import os
+import traceback
+import re
+import socket
+import fnmatch
+
+
+from resource_management.core.logger import Logger
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+  with open(PARENT_FILE, 'rb') as fp:
+    service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+  traceback.print_exc()
+  print "Failed to load parent"
+
+class PigServiceAdvisor(service_advisor.ServiceAdvisor):
+
+  def __init__(self, *args, **kwargs):
+    self.as_super = super(PigServiceAdvisor, self)
+    self.as_super.__init__(*args, **kwargs)
+
+    # Always call these methods
+    self.modifyMastersWithMultipleInstances()
+    self.modifyCardinalitiesDict()
+    self.modifyHeapSizeProperties()
+    self.modifyNotValuableComponents()
+    self.modifyComponentsNotPreferableOnServer()
+    self.modifyComponentLayoutSchemes()
+
+  def modifyMastersWithMultipleInstances(self):
+    """
+    Modify the set of masters with multiple instances.
+    Must be overriden in child class.
+    """
+    # Nothing to do
+    pass
+
+  def modifyCardinalitiesDict(self):
+    """
+    Modify the dictionary of cardinalities.
+    Must be overriden in child class.
+    """
+    # Nothing to do
+    pass
+
+  def modifyHeapSizeProperties(self):
+    """
+    Modify the dictionary of heap size properties.
+    Must be overriden in child class.
+    """
+    pass
+
+  def modifyNotValuableComponents(self):
+    """
+    Modify the set of components whose host assignment is based on other services.
+    Must be overriden in child class.
+    """
+    # Nothing to do
+    pass
+
+  def modifyComponentsNotPreferableOnServer(self):
+    """
+    Modify the set of components that are not preferable on the server.
+    Must be overriden in child class.
+    """
+    # Nothing to do
+    pass
+
+  def modifyComponentLayoutSchemes(self):
+    """
+    Modify layout scheme dictionaries for components.
+    The scheme dictionary basically maps the number of hosts to
+    host index where component should exist.
+    Must be overriden in child class.
+    """
+    # Nothing to do
+    pass
+
+  def getServiceComponentLayoutValidations(self, services, hosts):
+    """
+    Get a list of errors.
+    Must be overriden in child class.
+    """
+
+    return []
+
+  def getServiceConfigurationRecommendations(self, configurations, clusterData, services,
hosts):
+    """
+    Entry point.
+    Must be overriden in child class.
+    """
+    #Logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
+    #            (self.__class__.__name__, inspect.stack()[0][3]))
+
+    recommender = PigRecommender()
+
+
+  def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults,
services, hosts):
+    """
+    Entry point.
+    Validate configurations for the service. Return a list of errors.
+    The code for this function should be the same for each Service Advisor.
+    """
+    #Logger.info("Class: %s, Method: %s. Validating Configurations." %
+    #            (self.__class__.__name__, inspect.stack()[0][3]))
+
+    validator = PigValidator()
+    # Calls the methods of the validator using arguments,
+    # method(siteProperties, siteRecommendations, configurations, services, hosts)
+    return validator.validateListOfConfigUsingMethod(configurations, recommendedDefaults,
services, hosts, validator.validators)
+
+
+
+class PigRecommender(service_advisor.ServiceAdvisor):
+  """
+  Pig Recommender suggests properties when adding the service for the first time or modifying
configs via the UI.
+  """
+
+  def __init__(self, *args, **kwargs):
+    self.as_super = super(PigRecommender, self)
+    self.as_super.__init__(*args, **kwargs)
+
+
+
+
+
+
+
+class PigValidator(service_advisor.ServiceAdvisor):
+  """
+  Pig Validator checks the correctness of properties whenever the service is first added
or the user attempts to
+  change configs via the UI.
+  """
+
+  def __init__(self, *args, **kwargs):
+    self.as_super = super(PigValidator, self)
+    self.as_super.__init__(*args, **kwargs)
+
+    self.validators = []
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml
new file mode 100644
index 0000000..fd664d0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>PIG</name>
+      <version>0.16.1.3.0</version>
+      <extends>common-services/PIG/0.16.1.3.0</extends>
+    </service>
+  </services>
+</metainfo>


Mime
View raw message