Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E9F4B18509 for ; Tue, 16 Feb 2016 17:38:31 +0000 (UTC) Received: (qmail 72483 invoked by uid 500); 16 Feb 2016 17:38:31 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 72453 invoked by uid 500); 16 Feb 2016 17:38:31 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 72444 invoked by uid 99); 16 Feb 2016 17:38:31 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Feb 2016 17:38:31 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B1418E0534; Tue, 16 Feb 2016 17:38:31 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aonishuk@apache.org To: commits@ambari.apache.org Message-Id: <8de254d31fbc4ee3b8bdad43a7890e70@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-15057. Oozie untar and prepare-war should be only done exclusively in preupload.py (aonishuk) Date: Tue, 16 Feb 2016 17:38:31 +0000 (UTC) Repository: ambari Updated Branches: refs/heads/branch-2.2 6f4f30e39 -> b6c8e5dce AMBARI-15057. Oozie untar and prepare-war should be only done exclusively in preupload.py (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b6c8e5dc Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b6c8e5dc Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b6c8e5dc Branch: refs/heads/branch-2.2 Commit: b6c8e5dce1ce807b62fe245c3759b62de616a194 Parents: 6f4f30e Author: Andrew Onishuk Authored: Tue Feb 16 19:37:37 2016 +0200 Committer: Andrew Onishuk Committed: Tue Feb 16 19:38:22 2016 +0200 ---------------------------------------------------------------------- .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 11 +--- .../main/resources/scripts/Ambaripreupload.py | 59 ++++++++++++++++---- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 33 ++--------- 3 files changed, 57 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c8e5dc/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index be4a191..1701bbd 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -17,7 +17,6 @@ See the License for the specific language governing permissions and limitations under the License. """ -import hashlib import os from resource_management.core.resources.service import ServiceConfig @@ -228,11 +227,7 @@ def prepare_war(): if run_prepare_war: # Time-consuming to run - Execute(command, - user=params.oozie_user - ) - - return_code, output = shell.call(command, user=params.oozie_user, logoutput=False, quiet=False) + return_code, output = shell.call(command, user=params.oozie_user) if output is None: output = "" @@ -273,8 +268,7 @@ def oozie_server_specific(): ) hashcode_file = format("{oozie_home}/.hashcode") - hashcode = hashlib.md5(format('{oozie_home}/oozie-sharelib.tar.gz')).hexdigest() - skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share && [[ `cat {hashcode_file}` == '{hashcode}' ]]") + skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share") untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home) @@ -313,7 +307,6 @@ def oozie_server_specific(): prepare_war() File(hashcode_file, - content = hashcode, mode = 0644, ) http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c8e5dc/ambari-server/src/main/resources/scripts/Ambaripreupload.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py index 5a20698..61db286 100644 --- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py +++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py @@ -26,7 +26,6 @@ sys.path.append("/usr/lib/python2.6/site-packages") import glob from logging import thread import re -import hashlib import tempfile import time import functools @@ -143,6 +142,7 @@ with Environment() as env: hdfs_site = ConfigDictionary({'dfs.webhdfs.enabled':False, }) fs_default = get_fs_root() + oozie_secure = '' oozie_env_sh_template = \ ''' #!/bin/bash @@ -232,14 +232,13 @@ with Environment() as env: source_and_dest_pairs = [(component_tar_source_file, destination_file), ] return _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed) - - env.set_params(params) hadoop_conf_dir = params.hadoop_conf_dir oozie_libext_dir = format("/usr/hdp/{hdp_version}/oozie/libext") oozie_home=format("/usr/hdp/{hdp_version}/oozie") oozie_setup_sh=format("/usr/hdp/{hdp_version}/oozie/bin/oozie-setup.sh") + oozie_setup_sh_current="/usr/hdp/current/oozie-server/bin/oozie-setup.sh" oozie_tmp_dir = "/var/tmp/oozie" configure_cmds = [] configure_cmds.append(('tar','-xvf', oozie_home + '/oozie-sharelib.tar.gz','-C', oozie_home)) @@ -254,22 +253,62 @@ with Environment() as env: ) hashcode_file = format("{oozie_home}/.hashcode") - hashcode = hashlib.md5(format('{oozie_home}/oozie-sharelib.tar.gz')).hexdigest() - skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share && [[ `cat {hashcode_file}` == '{hashcode}' ]]") + skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share") Execute( configure_cmds, not_if = format("{no_op_test} || {skip_recreate_sharelib}"), sudo = True, ) - Execute(format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war"), - user = params.oozie_user, - not_if = format("{no_op_test} || {skip_recreate_sharelib}") - ) + File(hashcode_file, - content = hashcode, mode = 0644, ) + + ############################################### + # PREPARE-WAR [BEGIN] + ############################################### + prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd") + + # DON'T CHANGE THE VALUE SINCE IT'S USED TO DETERMINE WHETHER TO RUN THE COMMAND OR NOT BY READING THE MARKER FILE. + # Oozie tmp dir should be /var/tmp/oozie and is already created by a function above. + command = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure} ") + command_to_file = format("cd {oozie_tmp_dir} && {oozie_setup_sh_current} prepare-war {oozie_secure} ") + + run_prepare_war = False + if os.path.exists(prepare_war_cmd_file): + cmd = "" + with open(prepare_war_cmd_file, "r") as f: + cmd = f.readline().strip() + if command_to_file != cmd: + run_prepare_war = True + Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} has contents which differ.\n" \ + "Expected: {command_to_file}.\nActual: {cmd}.")) + else: + run_prepare_war = True + Logger.info(format("Will run prepare war cmd since marker file {prepare_war_cmd_file} is missing.")) + + if run_prepare_war: + # Time-consuming to run + return_code, output = shell.call(command, user=params.oozie_user) + if output is None: + output = "" + + if return_code != 0 or "New Oozie WAR file with added".lower() not in output.lower(): + message = "Unexpected Oozie WAR preparation output {0}".format(output) + Logger.error(message) + raise Fail(message) + + # Generate marker file + File(prepare_war_cmd_file, + content=command_to_file, + mode=0644, + ) + else: + Logger.info(format("No need to run prepare-war since marker file {prepare_war_cmd_file} already exists.")) + ############################################### + # PREPARE-WAR END [BEGIN] + ############################################### oozie_shared_lib = format("/usr/hdp/{hdp_version}/oozie/share") oozie_user = 'oozie' oozie_hdfs_user_dir = format("{hdfs_path_prefix}/user/{oozie_user}") http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c8e5dc/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py index 826a004..3bb4b30 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py +++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py @@ -25,14 +25,9 @@ from resource_management.core import shell from resource_management.core.exceptions import Fail from resource_management.libraries import functions from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil -import hashlib import tempfile -md5_mock = MagicMock() -md5_mock.hexdigest.return_value = "abc123hash" - @patch("platform.linux_distribution", new = MagicMock(return_value="Linux")) -@patch.object(hashlib, "md5", new=MagicMock(return_value=md5_mock)) @patch.object(WebHDFSUtil, "run_command", new=MagicMock(return_value={})) @patch.object(tempfile, "gettempdir", new=MagicMock(return_value="/tmp")) class TestOozieServer(RMFTestCase): @@ -219,7 +214,7 @@ class TestOozieServer(RMFTestCase): recursive = True, ) self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'), - not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]", + not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share", sudo = True, ) self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'), @@ -255,15 +250,11 @@ class TestOozieServer(RMFTestCase): not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'", ) - self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war', - user = "oozie" - ) self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd', content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war', mode = 0644, ) self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode', - content = 'abc123hash', mode = 0644, ) self.assertResourceCalled('Execute', ('chown', '-R', 'oozie:hadoop', '/var/lib/oozie/oozie-server'), @@ -435,9 +426,9 @@ class TestOozieServer(RMFTestCase): recursive = True, ) self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'), - not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]", - sudo = True, - ) + not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share", + sudo = True, + ) self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'), not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'", sudo = True, @@ -472,15 +463,11 @@ class TestOozieServer(RMFTestCase): self.assertResourceCalled('Execute', 'ambari-sudo.sh chown oozie:hadoop /usr/lib/oozie/libext/falcon-oozie-el-extension-*.jar', not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'", ) - self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war', - user = 'oozie', - ) self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd', content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war', mode = 0644, ) self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode', - content = 'abc123hash', mode = 0644, ) self.assertResourceCalled('Execute', ('chown', '-R', u'oozie:hadoop', '/var/lib/oozie/oozie-server'), @@ -804,7 +791,7 @@ class TestOozieServer(RMFTestCase): recursive = True, ) self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'), - not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]", + not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share", sudo = True, ) self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'), @@ -826,15 +813,11 @@ class TestOozieServer(RMFTestCase): not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'", ) - self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war', - user="oozie") - self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd', content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war', mode = 0644, ) self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode', - content = 'abc123hash', mode = 0644, ) self.assertResourceCalled('Execute', ('chown', '-R', 'oozie:hadoop', '/var/lib/oozie/oozie-server'), @@ -991,7 +974,7 @@ class TestOozieServer(RMFTestCase): recursive = True, ) self.assertResourceCalled('Execute', ('tar', '-xvf', '/usr/lib/oozie/oozie-sharelib.tar.gz', '-C', '/usr/lib/oozie'), - not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share && [[ `cat /usr/lib/oozie/.hashcode` == 'abc123hash' ]]", + not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1' || test -f /usr/lib/oozie/.hashcode && test -d /usr/lib/oozie/share", sudo = True, ) self.assertResourceCalled('Execute', ('cp', '/usr/share/HDP-oozie/ext-2.2.zip', '/usr/lib/oozie/libext'), @@ -1013,15 +996,11 @@ class TestOozieServer(RMFTestCase): not_if = "ambari-sudo.sh su oozie -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1'", ) - self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war -secure', - user="oozie") - self.assertResourceCalled('File', '/usr/lib/oozie/.prepare_war_cmd', content = 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-setup.sh prepare-war -secure', mode = 0644, ) self.assertResourceCalled('File', '/usr/lib/oozie/.hashcode', - content = 'abc123hash', mode = 0644, ) self.assertResourceCalled('Execute', ('chown', '-R', 'oozie:hadoop', '/var/lib/oozie/oozie-server'),