ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-12969. sys_prepped clusters should not have to copy tarballs again
Date Wed, 02 Sep 2015 16:38:09 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 113a0b602 -> 31edfe3ba


AMBARI-12969. sys_prepped clusters should not have to copy tarballs again


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/31edfe3b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/31edfe3b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/31edfe3b

Branch: refs/heads/branch-2.1
Commit: 31edfe3ba168673ef20d7edaa95c7026ac84aaaa
Parents: 113a0b6
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Wed Sep 2 09:35:42 2015 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Wed Sep 2 09:37:35 2015 -0700

----------------------------------------------------------------------
 .../libraries/functions/copy_tarball.py             |  9 +++++++--
 .../HIVE/0.12.0.2.0/package/scripts/hive.py         | 13 ++++++++-----
 .../HIVE/0.12.0.2.0/package/scripts/hive_server.py  | 12 ++++++++++--
 .../PIG/0.12.0.2.0/package/scripts/service_check.py |  5 ++++-
 .../1.2.0.2.2/package/scripts/job_history_server.py |  6 +++++-
 .../1.2.0.2.2/package/scripts/spark_service.py      |  2 +-
 .../TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py    |  7 ++++++-
 .../TEZ/0.4.0.2.1/package/scripts/service_check.py  |  2 +-
 .../YARN/2.1.0.2.0/package/scripts/historyserver.py | 16 ++++++++++++----
 .../python/stacks/2.0.6/HIVE/test_hive_server.py    |  8 ++++----
 .../python/stacks/2.0.6/YARN/test_historyserver.py  |  2 +-
 .../python/stacks/2.2/PIG/test_pig_service_check.py |  2 +-
 12 files changed, 60 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index 792f019..badf2fe 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -96,7 +96,7 @@ def _get_single_version_from_hdp_select():
   return hdp_version
 
 def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=None, custom_dest_file=None,
force_execute=False,
-                 use_ru_version_during_ru=True, replace_existing_files=False):
+                 use_ru_version_during_ru=True, replace_existing_files=False, host_sys_prepped=False):
   """
   :param name: Tarball name, e.g., tez, hive, pig, sqoop.
   :param user_group: Group to own the directory.
@@ -106,6 +106,7 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
   :param custom_dest_file: Override the destination file path
   :param force_execute: If true, will execute the HDFS commands immediately, otherwise, will
defer to the calling function.
   :param use_ru_version_during_ru: If true, will use the version going to during RU. Otherwise,
use the CURRENT (source) version.
+  :param host_sys_prepped: If true, tarballs will not be copied as the cluster deployment
uses prepped VMs.
   :return: Will return True if successful, otherwise, False.
   """
   import params
@@ -127,6 +128,10 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
   if custom_dest_file is not None:
     dest_file = custom_dest_file
 
+  if host_sys_prepped:
+    Logger.info("Skipping copying {0} to {1} for {2} as its a sys_prepped host.".format(str(source_file),
str(dest_file), str(name)))
+    return True
+
   upgrade_direction = default("/commandParams/upgrade_direction", None)
   is_rolling_upgrade = upgrade_direction is not None
   current_version = default("/hostLevelParams/current_version", None)
@@ -200,4 +205,4 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
   if force_execute:
     params.HdfsResource(None, action="execute")
 
-  return True
\ No newline at end of file
+  return True

http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index 1b2b155..7e35a57 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -132,8 +132,8 @@ def hive(name=None):
     # *********************************
     # HDP 2.2 or higher, copy mapreduce.tar.gz to HDFS
     if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major,
'2.2') >= 0:
-      copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
-      copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+      copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
+      copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
 
     # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode.
     # This can use a different source and dest location to account for both HDP 2.1 and 2.2
@@ -142,13 +142,15 @@ def hive(name=None):
                  params.hdfs_user,
                  file_mode=params.tarballs_mode,
                  custom_source_file=params.pig_tar_source,
-                 custom_dest_file=params.pig_tar_dest_file)
+                 custom_dest_file=params.pig_tar_dest_file,
+                 host_sys_prepped=params.host_sys_prepped)
     copy_to_hdfs("hive",
                  params.user_group,
                  params.hdfs_user,
                  file_mode=params.tarballs_mode,
                  custom_source_file=params.hive_tar_source,
-                 custom_dest_file=params.hive_tar_dest_file)
+                 custom_dest_file=params.hive_tar_dest_file,
+                 host_sys_prepped=params.host_sys_prepped)
 
     wildcard_tarballs = ["sqoop", "hadoop_streaming"]
     for tarball_name in wildcard_tarballs:
@@ -168,7 +170,8 @@ def hive(name=None):
                      params.hdfs_user,
                      file_mode=params.tarballs_mode,
                      custom_source_file=source_file,
-                     custom_dest_file=dest_file)
+                     custom_dest_file=dest_file,
+                     host_sys_prepped=params.host_sys_prepped)
     # ******* End Copy Tarballs *******
     # *********************************
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
index 3ec1747..d7542e9 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server.py
@@ -115,8 +115,16 @@ class HiveServerDefault(HiveServer):
       hdp_select.select("hive-server2", params.version)
 
       # Copy mapreduce.tar.gz and tez.tar.gz to HDFS
-      resource_created = copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
-      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) or resource_created
+      resource_created = copy_to_hdfs(
+        "mapreduce",
+        params.user_group,
+        params.hdfs_user,
+        host_sys_prepped=params.host_sys_prepped)
+      resource_created = copy_to_hdfs(
+        "tez",
+        params.user_group,
+        params.hdfs_user,
+        host_sys_prepped=params.host_sys_prepped) or resource_created
       if resource_created:
         params.HdfsResource(None, action="execute")
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
index 2f8da76..66b9772 100644
--- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
@@ -95,7 +95,10 @@ class PigServiceCheckLinux(PigServiceCheck):
       )
 
       # Check for Pig-on-Tez
-      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+      resource_created = copy_to_hdfs(
+        "tez", params.user_group,
+        params.hdfs_user,
+        host_sys_prepped=params.host_sys_prepped)
       if resource_created:
         params.HdfsResource(None, action="execute")
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
index 8cdafc4..b3999c3 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
@@ -82,7 +82,11 @@ class JobHistoryServer(Script):
       # need to copy the tarball, otherwise, copy it.
 
       if params.version and compare_versions(format_hdp_stack_version(params.version), '2.3.0.0')
< 0:
-        resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+        resource_created = copy_to_hdfs(
+          "tez",
+          params.user_group,
+          params.hdfs_user,
+          host_sys_prepped=params.host_sys_prepped)
         if resource_created:
           params.HdfsResource(None, action="execute")
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
index 8d758e3..d8b3e66 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
@@ -36,7 +36,7 @@ def spark_service(action):
     # Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency
on Tez, so it does not
     # need to copy the tarball, otherwise, copy it.
     if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.3.0.0')
< 0:
-      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
       if resource_created:
         params.HdfsResource(None, action="execute")
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py
b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py
index 7731bc7..776d185 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/pre_upgrade.py
@@ -42,7 +42,12 @@ class TezPreUpgrade(Script):
       Logger.info("Stack version {0} is sufficient to check if need to copy tez.tar.gz to
HDFS.".format(params.hdp_stack_version))
 
       # Force it to copy the current version of the tez tarball, rather than the version
the RU will go to.
-      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, use_ru_version_during_ru=False)
+      resource_created = copy_to_hdfs(
+        "tez",
+        params.user_group,
+        params.hdfs_user,
+        use_ru_version_during_ru=False,
+        host_sys_prepped=params.host_sys_prepped)
       if resource_created:
         params.HdfsResource(None, action="execute")
       else:

http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py
b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py
index 9bd366b..e0fa556 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py
@@ -63,7 +63,7 @@ class TezServiceCheckLinux(TezServiceCheck):
     )
 
     if params.hdp_stack_version and compare_versions(params.hdp_stack_version, '2.2.0.0')
>= 0:
-      copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+      copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
 
     params.HdfsResource(None, action = "execute")
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
index 16e34d4..5fe8157 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
@@ -81,8 +81,8 @@ class HistoryServerDefault(HistoryServer):
       conf_select.select(params.stack_name, "hadoop", params.version)
       hdp_select.select("hadoop-mapreduce-historyserver", params.version)
       # MC Hammer said, "Can't touch this"
-      copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
-      copy_to_hdfs("tez", params.user_group, params.hdfs_user)
+      copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
+      copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
       params.HdfsResource(None, action="execute")
 
   def start(self, env, rolling_restart=False):
@@ -92,8 +92,16 @@ class HistoryServerDefault(HistoryServer):
 
     if params.hdp_stack_version_major and compare_versions(params.hdp_stack_version_major,
'2.2.0.0') >= 0:
       # MC Hammer said, "Can't touch this"
-      resource_created = copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
-      resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user) or resource_created
+      resource_created = copy_to_hdfs(
+        "mapreduce",
+        params.user_group,
+        params.hdfs_user,
+        host_sys_prepped=params.host_sys_prepped)
+      resource_created = copy_to_hdfs(
+        "tez",
+        params.user_group,
+        params.hdfs_user,
+        host_sys_prepped=params.host_sys_prepped) or resource_created
       if resource_created:
         params.HdfsResource(None, action="execute")
     else:

http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index 42477dd..83328c1 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -867,8 +867,8 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
     self.assertResourceCalled('Execute',
                               ('hdp-select', 'set', 'hive-server2', version), sudo=True,)
 
-    copy_to_hdfs_mock.assert_any_call("mapreduce", "hadoop", "hdfs")
-    copy_to_hdfs_mock.assert_any_call("tez", "hadoop", "hdfs")
+    copy_to_hdfs_mock.assert_any_call("mapreduce", "hadoop", "hdfs", host_sys_prepped=False)
+    copy_to_hdfs_mock.assert_any_call("tez", "hadoop", "hdfs", host_sys_prepped=False)
     self.assertEquals(2, copy_to_hdfs_mock.call_count)
     self.assertResourceCalled('HdfsResource', None,
         security_enabled = False,
@@ -905,8 +905,8 @@ From source with checksum 150f554beae04f76f814f59549dead8b"""
     self.assertResourceCalled('Execute',
 
                               ('hdp-select', 'set', 'hive-server2', version), sudo=True,)
-    copy_to_hdfs_mock.assert_any_call("mapreduce", "hadoop", "hdfs")
-    copy_to_hdfs_mock.assert_any_call("tez", "hadoop", "hdfs")
+    copy_to_hdfs_mock.assert_any_call("mapreduce", "hadoop", "hdfs", host_sys_prepped=False)
+    copy_to_hdfs_mock.assert_any_call("tez", "hadoop", "hdfs", host_sys_prepped=False)
     self.assertEquals(2, copy_to_hdfs_mock.call_count)
     self.assertResourceCalled('HdfsResource', None,
         security_enabled = False,

http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
index 6cf0f88..45c7978 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
@@ -767,7 +767,7 @@ class TestHistoryServer(RMFTestCase):
                        mocks_dict = mocks_dict)
 
     self.assertResourceCalled('Execute', ('hdp-select', 'set', 'hadoop-mapreduce-historyserver',
version), sudo=True)
-    copy_to_hdfs_mock.assert_called_with("tez", "hadoop", "hdfs")
+    copy_to_hdfs_mock.assert_called_with("tez", "hadoop", "hdfs", host_sys_prepped=False)
 
     self.assertResourceCalled('HdfsResource', None,
         security_enabled = False,

http://git-wip-us.apache.org/repos/asf/ambari/blob/31edfe3b/ambari-server/src/test/python/stacks/2.2/PIG/test_pig_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/PIG/test_pig_service_check.py b/ambari-server/src/test/python/stacks/2.2/PIG/test_pig_service_check.py
index c4bfa8e..6f83106 100644
--- a/ambari-server/src/test/python/stacks/2.2/PIG/test_pig_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.2/PIG/test_pig_service_check.py
@@ -125,7 +125,7 @@ class TestPigServiceCheck(RMFTestCase):
         action = ['create_on_execute'],
     )
 
-    copy_to_hdfs_mock.assert_called_with("tez", "hadoop", "hdfs")
+    copy_to_hdfs_mock.assert_called_with("tez", "hadoop", "hdfs", host_sys_prepped=False)
     self.assertResourceCalled('HdfsResource', None,
         security_enabled = True,
         hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',


Mime
View raw message