Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8072610A23 for ; Fri, 11 Apr 2014 18:48:12 +0000 (UTC) Received: (qmail 50462 invoked by uid 500); 11 Apr 2014 18:48:12 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 50403 invoked by uid 500); 11 Apr 2014 18:48:10 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 50396 invoked by uid 99); 11 Apr 2014 18:48:08 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 11 Apr 2014 18:48:08 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 8B3F498B247; Fri, 11 Apr 2014 18:48:08 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: swagle@apache.org To: commits@ambari.apache.org Message-Id: <594a2f3c60404e1da80313ebfcc03f1a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: AMBARI-5447. WebHCat start Fail with authorization enabled. (swagle) Date: Fri, 11 Apr 2014 18:48:08 +0000 (UTC) Repository: ambari Updated Branches: refs/heads/branch-1.5.1 583f2410c -> 230dbf8f7 AMBARI-5447. WebHCat start Fail with authorization enabled. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/230dbf8f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/230dbf8f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/230dbf8f Branch: refs/heads/branch-1.5.1 Commit: 230dbf8f79344f57d00ab1c0d509af3a98bda0d1 Parents: 583f241 Author: Siddharth Wagle Authored: Fri Apr 11 11:47:28 2014 -0700 Committer: Siddharth Wagle Committed: Fri Apr 11 11:47:28 2014 -0700 ---------------------------------------------------------------------- .../libraries/providers/copy_from_local.py | 5 +- .../resource_management/TestCopyFromLocal.py | 4 +- .../services/WEBHCAT/package/scripts/webhcat.py | 54 ++++------------- .../stacks/1.3.2/WEBHCAT/test_webhcat_server.py | 62 ++++++++++++-------- 4 files changed, 54 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/230dbf8f/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py b/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py index d25dd8b..3fa8a54 100644 --- a/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py +++ b/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py @@ -37,8 +37,9 @@ class CopyFromLocalProvider(Provider): copy_cmd = format("fs -copyFromLocal {path} {dest_dir}") dest_file_name = os.path.split(path)[1] dest_path = dest_dir + dest_file_name if dest_dir.endswith(os.sep) else dest_dir + os.sep + dest_file_name - - unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {dest_path} >/dev/null 2>&1") + # Need to run unless as resource user + su_cmd = 'su - {0} -c'.format(owner) + unless_cmd = format("{su_cmd} '{kinnit_if_needed} hadoop fs -ls {dest_path}' >/dev/null 2>&1") ExecuteHadoop(copy_cmd, not_if=unless_cmd, http://git-wip-us.apache.org/repos/asf/ambari/blob/230dbf8f/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py index 7653b24..90397ae 100644 --- a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py +++ b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py @@ -36,7 +36,7 @@ class TestCopyFromLocal(TestCase): call_arg_list = execute_hadoop_mock.call_args_list self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/', call_arg_list[0][0][0].command) - self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'}, + self.assertEquals({'not_if': "su - user1 -c ' hadoop fs -ls /apps/test/*.files' >/dev/null 2>&1", 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[0][0][0].arguments) self.assertEquals('fs -chown user1 /apps/test/*.files', call_arg_list[1][0][0].command) self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments) @@ -57,7 +57,7 @@ class TestCopyFromLocal(TestCase): call_arg_list = execute_hadoop_mock.call_args_list self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/', call_arg_list[0][0][0].command) - self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1', 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'}, + self.assertEquals({'not_if': "su - user1 -c ' hadoop fs -ls /apps/test/*.files' >/dev/null 2>&1", 'user': 'user1', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[0][0][0].arguments) self.assertEquals('fs -chown user1:hdfs /apps/test/*.files', call_arg_list[1][0][0].command) self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments) http://git-wip-us.apache.org/repos/asf/ambari/blob/230dbf8f/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py index 0c75d18..be592d7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py @@ -20,7 +20,6 @@ Ambari Agent """ from resource_management import * -import sys def webhcat(): @@ -83,55 +82,26 @@ def webhcat(): path='/bin' ) - copyFromLocal(path='/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar', + CopyFromLocal('/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar', owner=params.webhcat_user, mode=0755, - dest_dir=format("{webhcat_apps_dir}/hadoop-streaming.jar"), - kinnit_if_needed=kinit_if_needed + dest_dir=params.webhcat_apps_dir, + kinnit_if_needed=kinit_if_needed, + hdfs_user=params.hdfs_user ) - copyFromLocal(path='/usr/share/HDP-webhcat/pig.tar.gz', + CopyFromLocal('/usr/share/HDP-webhcat/pig.tar.gz', owner=params.webhcat_user, mode=0755, - dest_dir=format("{webhcat_apps_dir}/pig.tar.gz"), + dest_dir=params.webhcat_apps_dir, + kinnit_if_needed=kinit_if_needed, + hdfs_user=params.hdfs_user ) - copyFromLocal(path='/usr/share/HDP-webhcat/hive.tar.gz', + CopyFromLocal('/usr/share/HDP-webhcat/hive.tar.gz', owner=params.webhcat_user, mode=0755, - dest_dir=format("{webhcat_apps_dir}/hive.tar.gz") + dest_dir=params.webhcat_apps_dir, + kinnit_if_needed=kinit_if_needed, + hdfs_user=params.hdfs_user ) - - -def copyFromLocal(path=None, owner=None, group=None, mode=None, dest_dir=None, kinnit_if_needed=""): - import params - - copy_cmd = format("fs -copyFromLocal {path} {dest_dir}") - unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {dest_dir} >/dev/null 2>&1") - - ExecuteHadoop(copy_cmd, - not_if=unless_cmd, - user=owner, - conf_dir=params.hadoop_conf_dir) - - if not owner: - chown = None - else: - if not group: - chown = owner - else: - chown = format('{owner}:{group}') - - if not chown: - chown_cmd = format("fs -chown {chown} {dest_dir}") - - ExecuteHadoop(copy_cmd, - user=owner, - conf_dir=params.hadoop_conf_dir) - - if not mode: - chmod_cmd = format('fs -chmod {mode} {dest_dir}') - - ExecuteHadoop(chmod_cmd, - user=owner, - conf_dir=params.hadoop_conf_dir) http://git-wip-us.apache.org/repos/asf/ambari/blob/230dbf8f/ambari-server/src/test/python/stacks/1.3.2/WEBHCAT/test_webhcat_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/1.3.2/WEBHCAT/test_webhcat_server.py b/ambari-server/src/test/python/stacks/1.3.2/WEBHCAT/test_webhcat_server.py index 666d693..8b363fd 100644 --- a/ambari-server/src/test/python/stacks/1.3.2/WEBHCAT/test_webhcat_server.py +++ b/ambari-server/src/test/python/stacks/1.3.2/WEBHCAT/test_webhcat_server.py @@ -151,20 +151,26 @@ class TestWebHCatServer(RMFTestCase): owner = 'hcat', group = 'hadoop', ) - self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar /apps/webhcat/hadoop-streaming.jar', - not_if = ' hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1', - user = 'hcat', - conf_dir = '/etc/hadoop/conf', + self.assertResourceCalled('CopyFromLocal', '/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar', + owner='hcat', + mode=0755, + dest_dir='/apps/webhcat', + kinnit_if_needed='', + hdfs_user='hdfs' ) - self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/pig.tar.gz /apps/webhcat/pig.tar.gz', - not_if = ' hadoop fs -ls /apps/webhcat/pig.tar.gz >/dev/null 2>&1', - user = 'hcat', - conf_dir = '/etc/hadoop/conf', + self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz', + owner='hcat', + mode=0755, + dest_dir='/apps/webhcat', + kinnit_if_needed='', + hdfs_user='hdfs' ) - self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/hive.tar.gz /apps/webhcat/hive.tar.gz', - not_if = ' hadoop fs -ls /apps/webhcat/hive.tar.gz >/dev/null 2>&1', - user = 'hcat', - conf_dir = '/etc/hadoop/conf', + self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz', + owner='hcat', + mode=0755, + dest_dir='/apps/webhcat', + kinnit_if_needed='', + hdfs_user='hdfs' ) def assert_configure_secured(self): @@ -227,18 +233,24 @@ class TestWebHCatServer(RMFTestCase): path = ['/bin'], user = 'hcat', ) - self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar /apps/webhcat/hadoop-streaming.jar', - not_if = '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; hadoop fs -ls /apps/webhcat/hadoop-streaming.jar >/dev/null 2>&1', - user = 'hcat', - conf_dir = '/etc/hadoop/conf', + self.assertResourceCalled('CopyFromLocal', '/usr/lib/hadoop/contrib/streaming/hadoop-streaming*.jar', + owner='hcat', + mode=0755, + dest_dir='/apps/webhcat', + kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;', + hdfs_user='hdfs' ) - self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/pig.tar.gz /apps/webhcat/pig.tar.gz', - not_if = ' hadoop fs -ls /apps/webhcat/pig.tar.gz >/dev/null 2>&1', - user = 'hcat', - conf_dir = '/etc/hadoop/conf', - ) - self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/share/HDP-webhcat/hive.tar.gz /apps/webhcat/hive.tar.gz', - not_if = ' hadoop fs -ls /apps/webhcat/hive.tar.gz >/dev/null 2>&1', - user = 'hcat', - conf_dir = '/etc/hadoop/conf', + self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz', + owner='hcat', + mode=0755, + dest_dir='/apps/webhcat', + kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;', + hdfs_user='hdfs' ) + self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz', + owner='hcat', + mode=0755, + dest_dir='/apps/webhcat', + kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa;', + hdfs_user='hdfs' + ) \ No newline at end of file