ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jlun...@apache.org
Subject ambari git commit: AMBARI-12909: Existence of empty package folder in stack definition breaks the service (Nahappan Somasundaram via jluniya)
Date Tue, 01 Sep 2015 05:13:46 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 8631fdce4 -> d145dd6dc


AMBARI-12909: Existence of empty package folder in stack definition breaks the service (Nahappan
Somasundaram via jluniya)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d145dd6d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d145dd6d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d145dd6d

Branch: refs/heads/trunk
Commit: d145dd6dcaf354ba053ab36220d3271f1a9ca81e
Parents: 8631fdc
Author: Jayush Luniya <jluniya@hortonworks.com>
Authored: Mon Aug 31 22:13:40 2015 -0700
Committer: Jayush Luniya <jluniya@hortonworks.com>
Committed: Mon Aug 31 22:13:40 2015 -0700

----------------------------------------------------------------------
 .../src/main/python/ambari_agent/FileCache.py   | 12 ++++++---
 .../test/python/ambari_agent/TestFileCache.py   | 26 ++++++++++++++++++++
 .../python/ambari_server/resourceFilesKeeper.py | 12 ++++++---
 .../src/test/python/TestResourceFilesKeeper.py  | 10 ++++++++
 4 files changed, 53 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/d145dd6d/ambari-agent/src/main/python/ambari_agent/FileCache.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/FileCache.py b/ambari-agent/src/main/python/ambari_agent/FileCache.py
index fa960e1..4869e51 100644
--- a/ambari-agent/src/main/python/ambari_agent/FileCache.py
+++ b/ambari-agent/src/main/python/ambari_agent/FileCache.py
@@ -145,9 +145,15 @@ class FileCache():
           download_url = self.build_download_url(server_url_prefix,
                                                  subdirectory, self.ARCHIVE_NAME)
           membuffer = self.fetch_url(download_url)
-          self.invalidate_directory(full_path)
-          self.unpack_archive(membuffer, full_path)
-          self.write_hash_sum(full_path, remote_hash)
+          # extract only when the archive is not zero sized
+          if (membuffer.getvalue().strip()):
+            self.invalidate_directory(full_path)
+            self.unpack_archive(membuffer, full_path)
+            self.write_hash_sum(full_path, remote_hash)
+          else:
+            logger.warn("Skipping empty archive: {0}. "
+                        "Expected archive was not found. Cached copy will be used.".format(download_url))
+            pass
         # Finally consider cache directory up-to-date
         self.uptodate_paths.append(full_path)
     except CachingException, e:

http://git-wip-us.apache.org/repos/asf/ambari/blob/d145dd6d/ambari-agent/src/test/python/ambari_agent/TestFileCache.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestFileCache.py b/ambari-agent/src/test/python/ambari_agent/TestFileCache.py
index d0c987b..5933daa 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestFileCache.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestFileCache.py
@@ -233,6 +233,32 @@ class TestFileCache(TestCase):
                                   "server_url_prefix")
     self.assertEquals(res, path)
 
+    # Test empty archive
+    fetch_url_mock.reset_mock()
+    build_download_url_mock.reset_mock()
+    read_hash_sum_mock.reset_mock()
+    invalidate_directory_mock.reset_mock()
+    unpack_archive_mock.reset_mock()
+    fileCache.reset()
+
+    fetch_url_mock.side_effect = None
+    membuffer_empty = MagicMock()
+    membuffer_empty.getvalue.return_value.strip.return_value = ""
+    fetch_url_mock.return_value = membuffer_empty # Remote hash and content
+    read_hash_sum_mock.return_value = "hash2" # Local hash
+
+    res = fileCache.provide_directory("cache_path", "subdirectory",
+                                      "server_url_prefix")
+    self.assertTrue(fetch_url_mock.return_value.strip() != read_hash_sum_mock.return_value.strip())
+    self.assertEquals(build_download_url_mock.call_count, 2)
+    self.assertEquals(fetch_url_mock.call_count, 2)
+    self.assertFalse(invalidate_directory_mock.called)
+    self.assertFalse(unpack_archive_mock.called)
+    self.assertFalse(write_hash_sum_mock.called)
+    self.assertEquals(pprint.pformat(fileCache.uptodate_paths), pprint.pformat([path]))
+    self.assertEquals(res, path)
+    pass
+
 
   def test_build_download_url(self):
     fileCache = FileCache(self.config)

http://git-wip-us.apache.org/repos/asf/ambari/blob/d145dd6d/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py b/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py
index fe80723..c9bca73 100644
--- a/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py
+++ b/ambari-server/src/main/python/ambari_server/resourceFilesKeeper.py
@@ -141,17 +141,17 @@ class ResourceFilesKeeper():
     except Exception, err:
       raise KeeperException("Can not list common services: {0}".format(str(err)))
 
-
   def update_directory_archive(self, directory):
     """
     If hash sum for directory is not present or differs from saved value,
     recalculates hash sum and creates directory archive
     """
+    skip_empty_directory = True
     cur_hash = self.count_hash_sum(directory)
     saved_hash = self.read_hash_sum(directory)
     if cur_hash != saved_hash:
       if not self.nozip:
-        self.zip_directory(directory)
+        self.zip_directory(directory, skip_empty_directory)
       self.write_hash_sum(directory, cur_hash)
 
 
@@ -216,14 +216,18 @@ class ResourceFilesKeeper():
       raise KeeperException("Can not write to file {0} : {1}".format(hash_file,
                                                                    str(err)))
 
-
-  def zip_directory(self, directory):
+  def zip_directory(self, directory, skip_if_empty = False):
     """
     Packs entire directory into zip file. Hash file is also packaged
     into archive
     """
     self.dbg_out("creating archive for directory {0}".format(directory))
     try:
+      if skip_if_empty:
+        if not os.listdir(directory):
+          self.dbg_out("Empty directory. Skipping archive creation for {0}".format(directory))
+          return
+
       zf = zipfile.ZipFile(os.path.join(directory, self.ARCHIVE_NAME), "w")
       abs_src = os.path.abspath(directory)
       for root, dirs, files in os.walk(directory):

http://git-wip-us.apache.org/repos/asf/ambari/blob/d145dd6d/ambari-server/src/test/python/TestResourceFilesKeeper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestResourceFilesKeeper.py b/ambari-server/src/test/python/TestResourceFilesKeeper.py
index 6856124..1fefa7a 100644
--- a/ambari-server/src/test/python/TestResourceFilesKeeper.py
+++ b/ambari-server/src/test/python/TestResourceFilesKeeper.py
@@ -349,6 +349,16 @@ class TestResourceFilesKeeper(TestCase):
       except Exception, e:
         self.fail('Unexpected exception thrown:' + str(e))
 
+    # Test skip zipping of an empty directory
+    with patch("os.listdir") as os_listdir_mock:
+      os_listdir_mock.return_value = False # Empty dir
+      try:
+        skip_empty_directory = True
+        resource_files_keeper.zip_directory("empty-to-directory", skip_empty_directory)
+        self.assertTrue(os_listdir_mock.called)
+      except Exception, e:
+        self.fail('Unexpected exception thrown: ' + str(e))
+    pass
 
   def test_is_ignored(self):
     resource_files_keeper = ResourceFilesKeeper(self.TEST_RESOURCES_DIR, self.DUMMY_UNCHANGEABLE_PACKAGE)


Mime
View raw message