Return-Path: X-Original-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1001B108C7 for ; Thu, 22 Aug 2013 21:53:30 +0000 (UTC) Received: (qmail 2254 invoked by uid 500); 22 Aug 2013 21:53:30 -0000 Delivered-To: apmail-incubator-allura-commits-archive@incubator.apache.org Received: (qmail 2216 invoked by uid 500); 22 Aug 2013 21:53:30 -0000 Mailing-List: contact allura-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: allura-dev@incubator.apache.org Delivered-To: mailing list allura-commits@incubator.apache.org Received: (qmail 1723 invoked by uid 99); 22 Aug 2013 21:53:29 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 22 Aug 2013 21:53:29 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 700AA8C2F6D; Thu, 22 Aug 2013 21:53:29 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: brondsem@apache.org To: allura-commits@incubator.apache.org Date: Thu, 22 Aug 2013 21:54:16 -0000 Message-Id: In-Reply-To: <4368fdfbc40f4eb8b4360cb76e034d25@git.apache.org> References: <4368fdfbc40f4eb8b4360cb76e034d25@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [50/50] git commit: [#3154] simplify zipping & cleanup [#3154] simplify zipping & cleanup Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/d91d964e Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/d91d964e Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/d91d964e Branch: refs/heads/master Commit: d91d964e24f8960fe7cd7e7d94198f4632f57a09 Parents: 7a5bf4c Author: Dave Brondsema Authored: Thu Aug 22 20:04:25 2013 +0000 Committer: Dave Brondsema Committed: Thu Aug 22 21:28:23 2013 +0000 ---------------------------------------------------------------------- Allura/allura/ext/admin/admin_main.py | 2 +- Allura/allura/tasks/export_tasks.py | 29 +++++++++++------------------ Allura/allura/tests/test_tasks.py | 22 +++++++--------------- 3 files changed, 19 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d91d964e/Allura/allura/ext/admin/admin_main.py ---------------------------------------------------------------------- diff --git a/Allura/allura/ext/admin/admin_main.py b/Allura/allura/ext/admin/admin_main.py index 0e33c94..22bd228 100644 --- a/Allura/allura/ext/admin/admin_main.py +++ b/Allura/allura/ext/admin/admin_main.py @@ -653,7 +653,7 @@ class ProjectAdminController(BaseController): flash('Export for project %s already running' % c.project.shortname, 'info') redirect('export') export_tasks.bulk_export.post(tools) - flash('Export scheduled', 'ok') + flash('Export scheduled. You will recieve an email with download instructions when complete.', 'ok') redirect('export') return { http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d91d964e/Allura/allura/tasks/export_tasks.py ---------------------------------------------------------------------- diff --git a/Allura/allura/tasks/export_tasks.py b/Allura/allura/tasks/export_tasks.py index 06590021..d06105e 100644 --- a/Allura/allura/tasks/export_tasks.py +++ b/Allura/allura/tasks/export_tasks.py @@ -16,6 +16,7 @@ # under the License. import os +import os.path import logging import shutil from tempfile import mkstemp @@ -47,6 +48,7 @@ def bulk_export(tools): def _bulk_export(project, tools, user): export_filename = project.bulk_export_filename() + export_path = create_export_dir(project, export_filename) not_exported_tools = [] for tool in tools or []: app = project.app_instance(tool) @@ -60,12 +62,9 @@ def _bulk_export(project, tools, user): continue log.info('Exporting %s...' % tool) try: - path = create_export_dir(project, export_filename) - temp_name = mkstemp(dir=path)[1] - with open(temp_name, 'w') as f: - with h.push_context(project._id): - app.bulk_export(f) - os.rename(temp_name, os.path.join(path, '%s.json' % tool)) + json_file = os.path.join(export_path, '%s.json' % tool) + with open(json_file, 'w') as f: + app.bulk_export(f) except: log.error('Something went wrong during export of %s' % tool, exc_info=True) not_exported_tools.append(tool) @@ -74,7 +73,7 @@ def _bulk_export(project, tools, user): if tools and len(not_exported_tools) < len(tools): # If that fails, we need to let it fail # there won't be a valid zip file for the user to get. - zip_and_cleanup(project, export_filename) + zip_and_cleanup(export_path, export_filename) else: log.error('Nothing to export') return None @@ -106,25 +105,19 @@ def create_export_dir(project, export_filename): """Create temporary directory for export files""" # Name temporary directory after project shortname, # thus zipdir() will use proper prefix inside the archive. - tmp_dir_suffix = export_filename.split('.')[0] + tmp_dir_suffix = os.path.splitext(export_filename)[0] path = os.path.join(project.bulk_export_path(), tmp_dir_suffix) if not os.path.exists(path): os.makedirs(path) return path -def zip_and_cleanup(project, export_filename): +def zip_and_cleanup(export_path, export_filename): """ - Zip exported data for a given project and filename (no path). + Zip exported data for a given path and filename. Copy it to proper location. Remove temporary files. """ - path = project.bulk_export_path() - temp = os.path.join(path, export_filename.split('.')[0]) - zip_path_temp = os.path.join(temp, export_filename) - zip_path = os.path.join(path, export_filename) - - zipdir(temp, zip_path_temp) + zipdir(export_path, os.path.join(os.path.dirname(export_path), export_filename)) # cleanup - shutil.move(zip_path_temp, zip_path) - shutil.rmtree(temp) \ No newline at end of file + shutil.rmtree(export_path) http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d91d964e/Allura/allura/tests/test_tasks.py ---------------------------------------------------------------------- diff --git a/Allura/allura/tests/test_tasks.py b/Allura/allura/tests/test_tasks.py index 26bcb36..80ecdfa 100644 --- a/Allura/allura/tests/test_tasks.py +++ b/Allura/allura/tests/test_tasks.py @@ -344,10 +344,8 @@ class TestExportTasks(unittest.TestCase): @mock.patch('allura.tasks.export_tasks.log') def test_bulk_export_invalid_tool(self, log): export_tasks.bulk_export([u'bugs', u'blog']) - assert_equal(log.info.call_count, 2) - assert_equal(log.info.call_args_list, [ - mock.call('Can not load app for bugs mount point. Skipping.'), - mock.call('Can not load app for blog mount point. Skipping.')]) + log.info.assert_any_call('Can not load app for bugs mount point. Skipping.') + log.info.assert_any_call('Can not load app for blog mount point. Skipping.') @mock.patch('allura.tasks.export_tasks.log') @mock.patch('allura.tasks.export_tasks.M.Project.app_instance') @@ -357,10 +355,8 @@ class TestExportTasks(unittest.TestCase): def test_bulk_export_not_exportable_tool(self, mail_tasks, app, log): app.return_value.exportable = False export_tasks.bulk_export([u'bugs', u'blog']) - assert_equal(log.info.call_count, 2) - assert_equal(log.info.call_args_list, [ - mock.call('Tool bugs is not exportable. Skipping.'), - mock.call('Tool blog is not exportable. Skipping.')]) + log.info.assert_any_call('Tool bugs is not exportable. Skipping.') + log.info.assert_any_call('Tool blog is not exportable. Skipping.') @mock.patch('allura.model.project.Project.__json__') @mock.patch('allura.tasks.export_tasks.shutil') @@ -371,15 +367,12 @@ class TestExportTasks(unittest.TestCase): def test_bulk_export(self, log, wiki_bulk_export, zipdir, shutil, project_json): M.MonQTask.query.remove() export_tasks.bulk_export([u'wiki']) - assert_equal(log.info.call_count, 1) - assert_equal(log.info.call_args_list, [ - mock.call('Exporting wiki...')]) + log.info.assert_any_call('Exporting wiki...') wiki_bulk_export.assert_called_once() project_json.assert_called_once() temp = '/tmp/bulk_export/p/test/test' zipfn = '/tmp/bulk_export/p/test/test.zip' zipdir.assert_caled_with(temp, temp + '/test.zip') - shutil.move.assert_called_once_with(temp + '/test.zip', zipfn) shutil.rmtree.assert_called_once_with(temp) # check notification M.MonQTask.run_ready() @@ -403,12 +396,11 @@ class TestExportTasks(unittest.TestCase): @onlyif(os.path.exists(tg.config.get('scm.repos.tarball.zip_binary', '/usr/bin/zip')), 'zip binary is missing') def test_zip_and_cleanup(self): project = M.Project.query.get(shortname='test') - export_path = project.bulk_export_path() export_filename = project.bulk_export_filename() path = export_tasks.create_export_dir(project, export_filename) - export_tasks.zip_and_cleanup(project, export_filename) + export_tasks.zip_and_cleanup(path, export_filename) assert not os.path.exists(path) - assert os.path.exists(os.path.join(export_path, 'test.zip')) + assert os.path.exists(os.path.join(project.bulk_export_path(), 'test.zip')) def test_bulk_export_status(self): assert_equal(c.project.bulk_export_status(), None)