ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject git commit: AMBARI-4928. Cannot launch Hive CLI due to HDFS permission issues. (swagle)
Date Tue, 04 Mar 2014 02:56:13 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk b05ae04ba -> 0a6b6df9a


AMBARI-4928. Cannot launch Hive CLI due to HDFS permission issues. (swagle)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0a6b6df9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0a6b6df9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0a6b6df9

Branch: refs/heads/trunk
Commit: 0a6b6df9a0f24851d73f282bf73517883115f921
Parents: b05ae04
Author: Siddharth Wagle <swagle@hortonworks.com>
Authored: Mon Mar 3 17:20:13 2014 -0800
Committer: Siddharth Wagle <swagle@hortonworks.com>
Committed: Mon Mar 3 17:20:33 2014 -0800

----------------------------------------------------------------------
 .../libraries/providers/copy_from_local.py              | 10 +++++++---
 .../python/resource_management/TestCopyFromLocal.py     |  8 ++++----
 .../2.0.6/services/HIVE/package/scripts/hive_server.py  |  6 +++---
 .../2.0.6/services/WEBHCAT/package/scripts/webhcat.py   |  6 +++---
 .../test/python/stacks/2.0.6/HIVE/test_hive_server.py   |  6 +++---
 .../python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py  | 12 ++++++------
 6 files changed, 26 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/0a6b6df9/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
b/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
index 6d04e72..d25dd8b 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
@@ -20,6 +20,7 @@ Ambari Agent
 
 """
 
+import os
 from resource_management import *
 
 class CopyFromLocalProvider(Provider):
@@ -34,7 +35,10 @@ class CopyFromLocalProvider(Provider):
     hadoop_conf_path = self.resource.hadoop_conf_dir
 
     copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
-    unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {dest_dir} >/dev/null 2>&1")
+    dest_file_name = os.path.split(path)[1]
+    dest_path = dest_dir + dest_file_name if dest_dir.endswith(os.sep) else dest_dir + os.sep
+ dest_file_name
+
+    unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {dest_path} >/dev/null 2>&1")
 
     ExecuteHadoop(copy_cmd,
                   not_if=unless_cmd,
@@ -51,7 +55,7 @@ class CopyFromLocalProvider(Provider):
         chown = format('{owner}:{group}')
 
     if chown:
-      chown_cmd = format("fs -chown {chown} {dest_dir}")
+      chown_cmd = format("fs -chown {chown} {dest_path}")
 
       ExecuteHadoop(chown_cmd,
                     user=hdfs_usr,
@@ -60,7 +64,7 @@ class CopyFromLocalProvider(Provider):
 
     if mode:
       dir_mode = oct(mode)
-      chmod_cmd = format('fs -chmod {dir_mode} {dest_dir}')
+      chmod_cmd = format('fs -chmod {dir_mode} {dest_path}')
 
       ExecuteHadoop(chmod_cmd,
                     user=hdfs_usr,

http://git-wip-us.apache.org/repos/asf/ambari/blob/0a6b6df9/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
index 40b7fc8..7653b24 100644
--- a/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
+++ b/ambari-agent/src/test/python/resource_management/TestCopyFromLocal.py
@@ -36,9 +36,9 @@ class TestCopyFromLocal(TestCase):
       call_arg_list = execute_hadoop_mock.call_args_list
       self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
                        call_arg_list[0][0][0].command)
-      self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/ >/dev/null 2>&1',
'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+      self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1',
'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
                         call_arg_list[0][0][0].arguments)
-      self.assertEquals('fs -chown user1 /apps/test/', call_arg_list[1][0][0].command)
+      self.assertEquals('fs -chown user1 /apps/test/*.files', call_arg_list[1][0][0].command)
       self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
 
 
@@ -57,9 +57,9 @@ class TestCopyFromLocal(TestCase):
       call_arg_list = execute_hadoop_mock.call_args_list
       self.assertEqual('fs -copyFromLocal /user/testdir/*.files /apps/test/',
                        call_arg_list[0][0][0].command)
-      self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/ >/dev/null 2>&1',
'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
+      self.assertEquals({'not_if': ' hadoop fs -ls /apps/test/*.files >/dev/null 2>&1',
'user': 'user1', 'conf_dir': '/etc/hadoop/conf'},
                         call_arg_list[0][0][0].arguments)
-      self.assertEquals('fs -chown user1:hdfs /apps/test/', call_arg_list[1][0][0].command)
+      self.assertEquals('fs -chown user1:hdfs /apps/test/*.files', call_arg_list[1][0][0].command)
       self.assertEquals({'user': 'hdfs', 'conf_dir': '/etc/hadoop/conf'}, call_arg_list[1][0][0].arguments)
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/0a6b6df9/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
index c883cc1..057b8f6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
@@ -87,7 +87,7 @@ class HiveServer(Script):
         )
 
       CopyFromLocal(params.hive_exec_jar_path,
-                    mode=0655,
+                    mode=0755,
                     owner=params.hive_user,
                     dest_dir=hdfs_path,
                     kinnit_if_needed=kinit_if_needed,
@@ -136,7 +136,7 @@ class HiveServer(Script):
 
       if app_dir_path:
         CopyFromLocal(params.tez_local_api_jars,
-                      mode=0655,
+                      mode=0755,
                       owner=params.tez_user,
                       dest_dir=app_dir_path,
                       kinnit_if_needed=kinit_if_needed,
@@ -146,7 +146,7 @@ class HiveServer(Script):
 
       if lib_dir_path:
         CopyFromLocal(params.tez_local_lib_jars,
-                      mode=0655,
+                      mode=0755,
                       owner=params.tez_user,
                       dest_dir=lib_dir_path,
                       kinnit_if_needed=kinit_if_needed,

http://git-wip-us.apache.org/repos/asf/ambari/blob/0a6b6df9/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
index 67bb1a6..9b98d0d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
@@ -87,7 +87,7 @@ def webhcat():
   CopyFromLocal('/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar',
                 owner=params.webhcat_user,
                 mode=0755,
-                dest_dir=format("{webhcat_apps_dir}/hadoop-streaming.jar"),
+                dest_dir=params.webhcat_apps_dir,
                 kinnit_if_needed=kinit_if_needed,
                 hdfs_user=params.hdfs_user
   )
@@ -95,7 +95,7 @@ def webhcat():
   CopyFromLocal('/usr/share/HDP-webhcat/pig.tar.gz',
                 owner=params.webhcat_user,
                 mode=0755,
-                dest_dir=format("{webhcat_apps_dir}/pig.tar.gz"),
+                dest_dir=params.webhcat_apps_dir,
                 kinnit_if_needed=kinit_if_needed,
                 hdfs_user=params.hdfs_user
   )
@@ -103,7 +103,7 @@ def webhcat():
   CopyFromLocal('/usr/share/HDP-webhcat/hive.tar.gz',
                 owner=params.webhcat_user,
                 mode=0755,
-                dest_dir=format("{webhcat_apps_dir}/hive.tar.gz"),
+                dest_dir=params.webhcat_apps_dir,
                 kinnit_if_needed=kinit_if_needed,
                 hdfs_user=params.hdfs_user
   )
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/0a6b6df9/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index 37e073b..57e6bb8 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -99,7 +99,7 @@ class TestHiveServer(RMFTestCase):
                               )
 
     self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/tez*.jar',
-                              mode=0655,
+                              mode=0755,
                               owner='tez',
                               dest_dir='/apps/tez/',
                               kinnit_if_needed='',
@@ -107,7 +107,7 @@ class TestHiveServer(RMFTestCase):
     )
 
     self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/lib/*.jar',
-                              mode=0655,
+                              mode=0755,
                               owner='tez',
                               dest_dir='/apps/tez/lib/',
                               kinnit_if_needed='',
@@ -126,7 +126,7 @@ class TestHiveServer(RMFTestCase):
     )
 
     self.assertResourceCalled('CopyFromLocal', '/usr/lib/hive/lib/hive-exec.jar',
-                              mode=0655,
+                              mode=0755,
                               owner='hive',
                               dest_dir='/apps/hive/install',
                               kinnit_if_needed='',

http://git-wip-us.apache.org/repos/asf/ambari/blob/0a6b6df9/ambari-server/src/test/python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py b/ambari-server/src/test/python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py
index 7fb7b57..9368ad6 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/WEBHCAT/test_webhcat_server.py
@@ -157,21 +157,21 @@ class TestWebHCatServer(RMFTestCase):
     self.assertResourceCalled('CopyFromLocal', '/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar',
                               owner='hcat',
                               mode=0755,
-                              dest_dir=format('/apps/webhcat/hadoop-streaming.jar'),
+                              dest_dir='/apps/webhcat',
                               kinnit_if_needed='',
                               hdfs_user='hdfs'
     )
     self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz',
                               owner='hcat',
                               mode=0755,
-                              dest_dir=format('/apps/webhcat/pig.tar.gz'),
+                              dest_dir='/apps/webhcat',
                               kinnit_if_needed='',
                               hdfs_user='hdfs'
     )
     self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz',
                               owner='hcat',
                               mode=0755,
-                              dest_dir=format('/apps/webhcat/hive.tar.gz'),
+                              dest_dir='/apps/webhcat',
                               kinnit_if_needed='',
                               hdfs_user='hdfs'
     )
@@ -239,21 +239,21 @@ class TestWebHCatServer(RMFTestCase):
     self.assertResourceCalled('CopyFromLocal', '/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar',
                               owner='hcat',
                               mode=0755,
-                              dest_dir=format('/apps/webhcat/hadoop-streaming.jar'),
+                              dest_dir='/apps/webhcat',
                               kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab
ambari-qa;',
                               hdfs_user='hdfs'
     )
     self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/pig.tar.gz',
                               owner='hcat',
                               mode=0755,
-                              dest_dir=format('/apps/webhcat/pig.tar.gz'),
+                              dest_dir='/apps/webhcat',
                               kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab
ambari-qa;',
                               hdfs_user='hdfs'
     )
     self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/hive.tar.gz',
                               owner='hcat',
                               mode=0755,
-                              dest_dir=format('/apps/webhcat/hive.tar.gz'),
+                              dest_dir='/apps/webhcat',
                               kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab
ambari-qa;',
                               hdfs_user='hdfs'
     )
\ No newline at end of file


Mime
View raw message