Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A589317E6B for ; Tue, 21 Oct 2014 10:21:05 +0000 (UTC) Received: (qmail 19582 invoked by uid 500); 21 Oct 2014 10:21:05 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 19479 invoked by uid 500); 21 Oct 2014 10:21:05 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 19463 invoked by uid 99); 21 Oct 2014 10:21:05 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Oct 2014 10:21:05 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 36F78935E01; Tue, 21 Oct 2014 10:21:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dmitriusan@apache.org To: commits@ambari.apache.org Date: Tue, 21 Oct 2014 10:21:06 -0000 Message-Id: <219c436993cf485984ba00a81385256b@git.apache.org> In-Reply-To: <98293c3420aa44e5be1a1156902dbc33@git.apache.org> References: <98293c3420aa44e5be1a1156902dbc33@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] git commit: AMBARI-7863. Ambari should support for Pig-on-Tez (dlysnichenko) AMBARI-7863. Ambari should support for Pig-on-Tez (dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/64016a50 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/64016a50 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/64016a50 Branch: refs/heads/branch-1.7.0 Commit: 64016a50e5d9e82ee2366e9619945950aa818d17 Parents: d4e8b56 Author: Lisnichenko Dmitro Authored: Tue Oct 21 13:19:50 2014 +0300 Committer: Lisnichenko Dmitro Committed: Tue Oct 21 13:20:49 2014 +0300 ---------------------------------------------------------------------- .../services/HDFS/package/scripts/params.py | 2 +- .../services/HDFS/package/scripts/utils.py | 6 +-- .../services/PIG/package/scripts/params.py | 3 ++ .../PIG/package/scripts/service_check.py | 29 ++++++++++++++ .../stacks/HDP/2.2/services/PIG/metainfo.xml | 3 ++ .../python/stacks/2.0.6/PIG/test_pig_client.py | 40 ++++++++++++++++++++ 6 files changed, 79 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py index 1308aa2..6b16205 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py @@ -212,7 +212,7 @@ hadoop_env_sh_template = config['configurations']['hadoop-env']['content'] java_home = config['hostLevelParams']['java_home'] stack_version = str(config['hostLevelParams']['stack_version']) -stack_is_champlain_or_further = not (stack_version.startswith('2.0') or stack_version.startswith('2.1')) +stack_is_hdp22_or_further = not (stack_version.startswith('2.0') or stack_version.startswith('2.1')) if stack_version.startswith('2.0') and System.get_instance().os_family != "suse": # deprecated rhel jsvc_path http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py index 5e1221a..14251cd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py @@ -75,14 +75,14 @@ def service(action=None, name=None, user=None, create_pid_dir=False, hadoop_secure_dn_pid_file = format("{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid") # At Champlain stack and further, we may start datanode as a non-root even in secure cluster - if not params.stack_is_champlain_or_further or secure_ports_are_in_use: + if not params.stack_is_hdp22_or_further or secure_ports_are_in_use: user = "root" pid_file = format( "{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid") - if params.stack_is_champlain_or_further: + if params.stack_is_hdp22_or_further: hadoop_env_exports.update(hadoop_secure_dn_exports) - if action == 'stop' and params.stack_is_champlain_or_further and \ + if action == 'stop' and params.stack_is_hdp22_or_further and \ os.path.isfile(hadoop_secure_dn_pid_file): # We need special handling for this case to handle the situation # when we configure non-root secure DN and then restart it http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py index 785b89d..f28e541 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py @@ -55,3 +55,6 @@ java64_home = config['hostLevelParams']['java_home'] pig_properties = config['configurations']['pig-properties']['content'] log4j_props = config['configurations']['pig-log4j']['content'] + +stack_version = str(config['hostLevelParams']['stack_version']) +stack_is_hdp22_or_further = not (stack_version.startswith('2.0') or stack_version.startswith('2.1')) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py index 7619bd6..88c46e2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py @@ -51,6 +51,7 @@ class PigServiceCheck(Script): mode = 0755 ) + # check for Pig-on-M/R Execute( format("pig {tmp_dir}/pigSmoke.sh"), tries = 3, try_sleep = 5, @@ -64,6 +65,34 @@ class PigServiceCheck(Script): bin_dir = params.hadoop_bin_dir ) + if params.stack_is_hdp22_or_further: + # cleanup results from previous test + ExecuteHadoop( create_file_cmd, + tries = 3, + try_sleep = 5, + user = params.smokeuser, + conf_dir = params.hadoop_conf_dir, + # for kinit run + keytab = params.smoke_user_keytab, + security_enabled = params.security_enabled, + kinit_path_local = params.kinit_path_local, + bin_dir = params.hadoop_bin_dir + ) + + # Check for Pig-on-Tez + Execute( format("pig -x tez {tmp_dir}/pigSmoke.sh"), + tries = 3, + try_sleep = 5, + path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'), + user = params.smokeuser + ) + + ExecuteHadoop( test_cmd, + user = params.smokeuser, + conf_dir = params.hadoop_conf_dir, + bin_dir = params.hadoop_bin_dir + ) + if __name__ == "__main__": PigServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/main/resources/stacks/HDP/2.2/services/PIG/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/PIG/metainfo.xml index f43d395..dff2ae7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/PIG/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/PIG/metainfo.xml @@ -40,6 +40,9 @@ + + TEZ + http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py index 2874614..51eabbd 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py @@ -19,6 +19,7 @@ limitations under the License. ''' from stacks.utils.RMFTestCase import * +import json class TestPigClient(RMFTestCase): @@ -52,6 +53,8 @@ class TestPigClient(RMFTestCase): ) self.assertNoMoreResources() + + def test_configure_secured(self): self.executeScript("2.0.6/services/PIG/package/scripts/pig_client.py", classname = "PigClient", @@ -81,3 +84,40 @@ class TestPigClient(RMFTestCase): content = 'log4jproperties\nline2' ) self.assertNoMoreResources() + + def test_configure_default_hdp22(self): + + config_file = "stacks/2.0.6/configs/default.json" + with open(config_file, "r") as f: + default_json = json.load(f) + + default_json['hostLevelParams']['stack_version'] = '2.2' + + self.executeScript("2.0.6/services/PIG/package/scripts/pig_client.py", + classname = "PigClient", + command = "configure", + config_dict=default_json + ) + + self.assertResourceCalled('Directory', '/etc/pig/conf', + recursive = True, + owner = 'hdfs', + group = 'hadoop' + ) + self.assertResourceCalled('File', '/etc/pig/conf/pig-env.sh', + owner = 'hdfs', + content = InlineTemplate(self.getConfig()['configurations']['pig-env']['content']) + ) + self.assertResourceCalled('File', '/etc/pig/conf/pig.properties', + owner = 'hdfs', + group = 'hadoop', + mode = 0644, + content = 'pigproperties\nline2' + ) + self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties', + owner = 'hdfs', + group = 'hadoop', + mode = 0644, + content = 'log4jproperties\nline2' + ) + self.assertNoMoreResources() \ No newline at end of file