ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject git commit: AMBARI-7333. Tez deployment changes for Champlain (aonishuk)
Date Wed, 17 Sep 2014 15:11:17 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 4bcf111f1 -> c09edb729


AMBARI-7333. Tez deployment changes for Champlain (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c09edb72
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c09edb72
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c09edb72

Branch: refs/heads/trunk
Commit: c09edb729fce602f0943b048df7810353ecd4975
Parents: 4bcf111
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Wed Sep 17 18:11:06 2014 +0300
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Wed Sep 17 18:11:06 2014 +0300

----------------------------------------------------------------------
 .../libraries/providers/copy_from_local.py       | 12 +++++++++---
 .../libraries/resources/copy_from_local.py       |  3 ++-
 .../HIVE/package/scripts/install_jars.py         | 19 +++++++++++++++----
 .../2.1/services/TEZ/configuration/tez-site.xml  |  2 +-
 .../python/stacks/2.0.6/HIVE/test_hive_server.py | 10 ++++++++++
 5 files changed, 37 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
index 9031a77..79d3e35 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/copy_from_local.py
@@ -27,6 +27,7 @@ class CopyFromLocalProvider(Provider):
   def action_run(self):
     path = self.resource.path
     dest_dir = self.resource.dest_dir
+    dest_file = self.resource.dest_file
     kinnit_if_needed = self.resource.kinnit_if_needed
     owner = self.resource.owner
     group = self.resource.group
@@ -34,9 +35,14 @@ class CopyFromLocalProvider(Provider):
     hdfs_usr=self.resource.hdfs_user
     hadoop_conf_path = self.resource.hadoop_conf_dir
 
-    copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
-    dest_file_name = os.path.split(path)[1]
-    dest_path = dest_dir + dest_file_name if dest_dir.endswith(os.sep) else dest_dir + os.sep
+ dest_file_name
+
+    if dest_file:
+      copy_cmd = format("fs -copyFromLocal {path} {dest_dir}/{dest_file}")
+      dest_path = dest_dir + dest_file if dest_dir.endswith(os.sep) else dest_dir + os.sep
+ dest_file
+    else:
+      dest_file_name = os.path.split(path)[1]
+      copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
+      dest_path = dest_dir + os.sep + dest_file_name
     # Need to run unless as resource user
     su_cmd = 'su - {0} -c'.format(owner)
     unless_cmd = format("{su_cmd} '{kinnit_if_needed} hadoop fs -ls {dest_path}' >/dev/null
2>&1")

http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
b/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
index 328d9c2..eaaeab5 100644
--- a/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
+++ b/ambari-common/src/main/python/resource_management/libraries/resources/copy_from_local.py
@@ -28,6 +28,7 @@ class CopyFromLocal(Resource):
 
   path = ResourceArgument(default=lambda obj: obj.name)
   dest_dir = ResourceArgument(required=True)
+  dest_file = ResourceArgument()
   owner = ResourceArgument(required=True)
   group = ResourceArgument()
   mode = ResourceArgument()
@@ -35,4 +36,4 @@ class CopyFromLocal(Resource):
   hadoop_conf_dir = ResourceArgument(default='/etc/hadoop/conf')
   hdfs_user = ResourceArgument(default='hdfs')
 
-  actions = Resource.actions + ["run"]
\ No newline at end of file
+  actions = Resource.actions + ["run"]

http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
index 3548de7..0045fee 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
@@ -72,6 +72,16 @@ def install_tez_jars():
                     hdfs_user=params.hdfs_user,
                     hadoop_conf_dir=params.hadoop_conf_dir
       )
+
+      CopyFromLocal(params.tez_tar_file,
+                    owner=params.tez_user,
+                    mode=0755,
+                    dest_dir=app_dir_path,
+                    dest_file="tez.tar.gz",
+                    kinnit_if_needed=kinit_if_needed,
+                    hdfs_user=params.hdfs_user,
+                    hadoop_conf_dir=params.hadoop_conf_dir
+      )
     pass
 
     if lib_dir_path:
@@ -92,10 +102,11 @@ def get_tez_hdfs_dir_paths(tez_lib_uris = None):
   if tez_lib_uris and tez_lib_uris.strip().find(hdfs_path_prefix, 0) != -1:
     dir_paths = tez_lib_uris.split(',')
     for path in dir_paths:
-      lib_dir_path = path.replace(hdfs_path_prefix, '')
-      lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep
-      lib_dir_paths.append(lib_dir_path)
+      if not "tez.tar.gz" in path:
+        lib_dir_path = path.replace(hdfs_path_prefix, '')
+        lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path +
os.sep
+        lib_dir_paths.append(lib_dir_path)
     pass
   pass
 
-  return lib_dir_paths
\ No newline at end of file
+  return lib_dir_paths

http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml
index 218c508..7b34d3b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/configuration/tez-site.xml
@@ -21,7 +21,7 @@
 
   <property>
     <name>tez.lib.uris</name>
-    <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/</value>
+    <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/,hdfs:///apps/tez/tez.tar.gz</value>
     <description>The location of the Tez libraries which will be localized for DAGs</description>
   </property>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/c09edb72/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index bc723ab..e0dc3d8 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -88,6 +88,16 @@ class TestHiveServer(RMFTestCase):
                               hdfs_user='hdfs'
     )
 
+    self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/tez*.tar.gz',
+                              mode=0755,
+                              owner='tez',
+                              dest_dir='/apps/tez/',
+                              dest_file='tez.tar.gz',
+                              kinnit_if_needed='',
+                              hadoop_conf_dir='/etc/hadoop/conf',
+                              hdfs_user='hdfs'
+    )
+
     self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/lib/*.jar',
                               mode=0755,
                               owner='tez',


Mime
View raw message