ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-11734 - Missing Falcon conf files after rolling upgrade (jonathanhurley)
Date Sat, 06 Jun 2015 02:05:40 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk fdaa98043 -> c33b8ee26


AMBARI-11734 - Missing Falcon conf files after rolling upgrade  (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c33b8ee2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c33b8ee2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c33b8ee2

Branch: refs/heads/trunk
Commit: c33b8ee26849accf516ca0afd0bb811513e65c9c
Parents: fdaa980
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Fri Jun 5 15:44:34 2015 -0400
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Fri Jun 5 22:05:20 2015 -0400

----------------------------------------------------------------------
 .../libraries/functions/tar_archive.py          | 26 ++++++++++++++++++++
 .../package/scripts/falcon_server_upgrade.py    | 10 +++-----
 .../1.4.0.2.0/package/scripts/flume_upgrade.py  | 11 +++------
 .../KNOX/0.5.0.2.2/package/scripts/upgrade.py   | 12 ++++-----
 .../package/scripts/oozie_server_upgrade.py     | 10 +++-----
 .../stacks/2.0.6/OOZIE/test_oozie_server.py     |  8 ++++++
 .../stacks/2.1/FALCON/test_falcon_server.py     |  8 ++++++
 7 files changed, 57 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
index efbf933..f1fb597 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
@@ -28,3 +28,29 @@ def archive_dir(output_filename, input_dir):
       tar.add(input_dir, arcname=os.path.basename("."))
     finally:
       tar.close()
+
+
+def archive_directory_dereference(archive, directory):
+  """
+  Creates an archive of the specified directory. This will ensure that
+  symlinks are not included, but instead are followed for recursive inclusion.
+  :param archive:   the name of the archive to create, including path
+  :param directory:   the directory to include
+  :return:  None
+  """
+  tarball = None
+  try:
+    # !!! dereference must be TRUE since the conf is a symlink and we want
+    # its contents instead of the actual symlink
+    tarball = tarfile.open(archive, mode="w", dereference=True)
+
+    # tar the files, chopping off everything in front of directory
+    # /foo/bar/conf/a, /foo/bar/conf/b, /foo/bar/conf/dir/c
+    # becomes
+    # a
+    # b
+    # dir/c
+    tarball.add(directory, arcname=os.path.relpath(directory, start=directory))
+  finally:
+    if tarball:
+      tarball.close()
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
index e6819d2..07cfbf6 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
@@ -23,6 +23,7 @@ import tempfile
 
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
+from resource_management.libraries.functions import tar_archive
 
 BACKUP_TEMP_DIR = "falcon-upgrade-backup"
 BACKUP_DATA_ARCHIVE = "falcon-local-backup.tar"
@@ -51,13 +52,8 @@ def post_stop_backup():
     if os.path.exists(archive):
       os.remove(archive)
 
-    tarball = None
-    try:
-      tarball = tarfile.open(archive, "w")
-      tarball.add(directory, arcname=os.path.basename(directory))
-    finally:
-      if tarball:
-        tarball.close()
+    # backup the directory, following symlinks instead of including them
+    tar_archive.archive_directory_dereference(archive, directory)
 
 
 def pre_start_restore():

http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
index e2028fc..265bf5e 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/package/scripts/flume_upgrade.py
@@ -23,6 +23,7 @@ import tempfile
 
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
+from resource_management.libraries.functions import tar_archive
 
 BACKUP_TEMP_DIR = "flume-upgrade-backup"
 BACKUP_CONF_DIR_ARCHIVE = "flume-conf-backup.tar"
@@ -50,13 +51,9 @@ def post_stop_backup():
     if os.path.exists(archive):
       os.remove(archive)
 
-    tarball = None
-    try:
-      tarball = tarfile.open(archive, "w")
-      tarball.add(directory, arcname=os.path.basename(directory))
-    finally:
-      if tarball:
-        tarball.close()
+    # backup the directory, following symlinks instead of including them
+    tar_archive.archive_directory_dereference(archive, directory)
+
 
 def pre_start_restore():
   """

http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
index ee2b0e6..2c805fa 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
@@ -24,6 +24,7 @@ import tempfile
 
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
+from resource_management.libraries.functions import tar_archive
 
 BACKUP_TEMP_DIR = "knox-upgrade-backup"
 BACKUP_DATA_ARCHIVE = "knox-data-backup.tar"
@@ -51,15 +52,12 @@ def backup_data():
     if os.path.exists(archive):
       os.remove(archive)
 
-    tarball = None
-    try:
-      tarball = tarfile.open(archive, "w")
-      tarball.add(directory, arcname=os.path.basename(directory))
-    finally:
-      if tarball:
-        tarball.close()
+    # backup the directory, following symlinks instead of including them
+    tar_archive.archive_directory_dereference(archive, directory)
+
   return absolute_backup_dir
 
+
 def _get_directory_mappings():
   """
   Gets a dictionary of directory to archive name that represents the

http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
index 1afae6a..9d348e1 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
@@ -30,6 +30,7 @@ from resource_management.libraries.functions import Direction
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions import compare_versions
 from resource_management.libraries.functions import format_hdp_stack_version
+from resource_management.libraries.functions import tar_archive
 from resource_management.core.resources import File
 from resource_management.core.source import DownloadSource
 
@@ -59,13 +60,8 @@ def backup_configuration():
     if os.path.exists(archive):
       os.remove(archive)
 
-    tarball = None
-    try:
-      tarball = tarfile.open(archive, "w")
-      tarball.add(directory, arcname=os.path.basename(directory))
-    finally:
-      if tarball:
-        tarball.close()
+    # backup the directory, following symlinks instead of including them
+    tar_archive.archive_directory_dereference(archive, directory)
 
 
 def restore_configuration():

http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index a75b7d0..ce0f52e 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -929,6 +929,14 @@ class TestOozieServer(RMFTestCase):
     self.assertTrue(tarfile_open_mock.called)
     self.assertEqual(tarfile_open_mock.call_count,2)
 
+    # check the call args for creation of the tarfile
+    call_args = tarfile_open_mock.call_args_list[0]
+    call_argument_tarfile = call_args[0][0]
+    call_kwargs = call_args[1]
+
+    self.assertTrue("oozie-upgrade-backup/oozie-conf-backup.tar" in call_argument_tarfile)
+    self.assertEquals(True, call_kwargs["dereference"])
+
     self.assertTrue(chmod_mock.called)
     self.assertEqual(chmod_mock.call_count,1)
     chmod_mock.assert_called_once_with('/usr/hdp/current/oozie-server/libext-customer', 511)

http://git-wip-us.apache.org/repos/asf/ambari/blob/c33b8ee2/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
index 7c79bad..e7e0345 100644
--- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
+++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
@@ -212,6 +212,14 @@ class TestFalconServer(RMFTestCase):
     # 4 calls to tarfile.open (2 directories * read + write)
     self.assertTrue(tarfile_open_mock.called)
     self.assertEqual(tarfile_open_mock.call_count,4)
+
+    # check the call args for creation of the tarfile
+    call_args = tarfile_open_mock.call_args_list[0]
+    call_argument_tarfile = call_args[0][0]
+    call_kwargs = call_args[1]
+
+    self.assertTrue("falcon-upgrade-backup/falcon-conf-backup.tar" in call_argument_tarfile)
+    self.assertEquals(True, call_kwargs["dereference"])
     
   @patch("resource_management.libraries.functions.security_commons.build_expectations")
   @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")


Mime
View raw message