incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject git commit: [#6992] (Partial) refactor of bulk_export task using Boundaries concepts
Date Thu, 19 Dec 2013 20:23:17 GMT
Updated Branches:
  refs/heads/cj/6992 [created] a57573c90


[#6992] (Partial) refactor of bulk_export task using Boundaries concepts

Signed-off-by: Cory Johns <cjohns@slashdotmedia.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/a57573c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/a57573c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/a57573c9

Branch: refs/heads/cj/6992
Commit: a57573c9012b9fb3cca133f7c35ac51417ba2699
Parents: 79df9b7
Author: Cory Johns <cjohns@slashdotmedia.com>
Authored: Thu Dec 19 20:22:58 2013 +0000
Committer: Cory Johns <cjohns@slashdotmedia.com>
Committed: Thu Dec 19 20:22:58 2013 +0000

----------------------------------------------------------------------
 Allura/allura/tasks/export_tasks.py | 153 ++++++++++++++-----------------
 Allura/allura/tests/test_tasks.py   |  51 +++--------
 2 files changed, 85 insertions(+), 119 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/a57573c9/Allura/allura/tasks/export_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/export_tasks.py b/Allura/allura/tasks/export_tasks.py
index 2f286d0..3fd9499 100644
--- a/Allura/allura/tasks/export_tasks.py
+++ b/Allura/allura/tasks/export_tasks.py
@@ -44,89 +44,76 @@ def bulk_export(tools, filename=None, send_email=True):
     '''
     # it's very handy to use c.* within a @task,
     # but let's be explicit and keep it separate from the main code
-    return _bulk_export(c.project, tools, c.user, filename, send_email)
-
-
-def _bulk_export(project, tools, user, filename=None, send_email=True):
-    export_filename = filename or project.bulk_export_filename()
-    export_path = create_export_dir(project, export_filename)
-    not_exported_tools = []
-    for tool in tools or []:
-        app = project.app_instance(tool)
-        if not app:
-            log.info('Can not load app for %s mount point. Skipping.' % tool)
-            not_exported_tools.append(tool)
-            continue
-        if not app.exportable:
-            log.info('Tool %s is not exportable. Skipping.' % tool)
-            not_exported_tools.append(tool)
-            continue
-        log.info('Exporting %s...' % tool)
+    return BulkExport().process(c.project, tools, c.user, filename, send_email)
+
+
+class BulkExport(object):
+    def process(self, project, tools, user, filename=None, send_email=True):
+        export_filename = filename or project.bulk_export_filename()
+        export_path = self.get_export_path(project.bulk_export_path(), export_filename)
+        if not os.path.exists(export_path):
+            os.makedirs(export_path)
+        apps = [project.app_instance(tool) for tool in tools]
+        exportable = self.filter_exportable(apps)
+        results = [self.export(export_path, app) for app in exportable]
+        exported = self.filter_successful(results)
+        if exported:
+            zipdir(export_path, os.path.join(os.path.dirname(export_path), export_filename))
+        shutil.rmtree(export_path)
+
+        if not user:
+            log.info('No user. Skipping notification.')
+            return
+        if not send_email:
+            return
+
+        tmpl = g.jinja2_env.get_template('allura:templates/mail/bulk_export.html')
+        instructions = tg.config.get('bulk_export_download_instructions', '')
+        instructions = instructions.format(
+                project=project.shortname,
+                filename=export_filename,
+                c=c,
+            )
+        exported_names = [a.config.options.mount_point for a in exported]
+        tmpl_context = {
+            'instructions': instructions,
+            'project': project,
+            'tools': exported_names,
+            'not_exported_tools': list(set(tools) - set(exported_names)),
+        }
+        email = {
+            'sender': unicode(tg.config['forgemail.return_path']),
+            'fromaddr': unicode(tg.config['forgemail.return_path']),
+            'reply_to': unicode(tg.config['forgemail.return_path']),
+            'message_id': h.gen_message_id(),
+            'destinations': [unicode(user._id)],
+            'subject': u'Bulk export for project %s completed' % project.shortname,
+            'text': tmpl.render(tmpl_context),
+        }
+        mail_tasks.sendmail.post(**email)
+
+    def get_export_path(self, export_base_path, export_filename):
+        """Create temporary directory for export files"""
+        # Name temporary directory after project shortname,
+        # thus zipdir() will use proper prefix inside the archive.
+        tmp_dir_suffix = os.path.splitext(export_filename)[0]
+        path = os.path.join(export_base_path, tmp_dir_suffix)
+        return path
+
+    def filter_exportable(self, apps):
+        return [app for app in apps if app and app.exportable]
+
+    def export(self, export_path, app):
+        tool = app.config.options.mount_point
+        json_file = os.path.join(export_path, '%s.json' % tool)
         try:
-            json_file = os.path.join(export_path, '%s.json' % tool)
             with open(json_file, 'w') as f:
                 app.bulk_export(f)
-        except:
-            log.error('Something went wrong during export of %s' % tool, exc_info=True)
-            not_exported_tools.append(tool)
-            continue
-
-    if tools and len(not_exported_tools) < len(tools):
-        # If that fails, we need to let it fail
-        # there won't be a valid zip file for the user to get.
-        zip_and_cleanup(export_path, export_filename)
-    else:
-        log.error('Nothing to export')
-        return None
-
-    if not user:
-        log.info('No user. Skipping notification.')
-        return
-    if not send_email:
-        return
-
-    tmpl = g.jinja2_env.get_template('allura:templates/mail/bulk_export.html')
-    instructions = tg.config.get('bulk_export_download_instructions', '')
-    instructions = instructions.format(
-            project=project.shortname,
-            filename=export_filename,
-            c=c,
-        )
-    tmpl_context = {
-        'instructions': instructions,
-        'project': project,
-        'tools': list(set(tools) - set(not_exported_tools)),
-        'not_exported_tools': not_exported_tools,
-    }
-    email = {
-        'sender': unicode(tg.config['forgemail.return_path']),
-        'fromaddr': unicode(tg.config['forgemail.return_path']),
-        'reply_to': unicode(tg.config['forgemail.return_path']),
-        'message_id': h.gen_message_id(),
-        'destinations': [unicode(user._id)],
-        'subject': u'Bulk export for project %s completed' % project.shortname,
-        'text': tmpl.render(tmpl_context),
-    }
-    mail_tasks.sendmail.post(**email)
-
-
-def create_export_dir(project, export_filename):
-    """Create temporary directory for export files"""
-    # Name temporary directory after project shortname,
-    # thus zipdir() will use proper prefix inside the archive.
-    tmp_dir_suffix = os.path.splitext(export_filename)[0]
-    path = os.path.join(project.bulk_export_path(), tmp_dir_suffix)
-    if not os.path.exists(path):
-        os.makedirs(path)
-    return path
-
-
-def zip_and_cleanup(export_path, export_filename):
-    """
-    Zip exported data for a given path and filename.
-    Copy it to proper location. Remove temporary files.
-    """
-    zipdir(export_path, os.path.join(os.path.dirname(export_path), export_filename))
-
-    # cleanup
-    shutil.rmtree(export_path)
+        except Exception as e:
+            log.error('Error exporting: %s on %s', tool, app.project.shortname, exc_info=True)
+            return None
+        else:
+            return app
+
+    def filter_successful(self, results):
+        return [result for result in results if result is not None]

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/a57573c9/Allura/allura/tests/test_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_tasks.py b/Allura/allura/tests/test_tasks.py
index 13c6f7f..d908252 100644
--- a/Allura/allura/tests/test_tasks.py
+++ b/Allura/allura/tests/test_tasks.py
@@ -447,33 +447,29 @@ class TestExportTasks(unittest.TestCase):
         project = M.Project.query.get(shortname='test')
         shutil.rmtree(project.bulk_export_path(), ignore_errors=True)
 
-    @mock.patch('allura.tasks.export_tasks.log')
-    def test_bulk_export_invalid_tool(self, log):
-        export_tasks.bulk_export([u'bugs', u'blog'])
-        log.info.assert_any_call('Can not load app for bugs mount point. Skipping.')
-        log.info.assert_any_call('Can not load app for blog mount point. Skipping.')
-
-    @mock.patch('allura.tasks.export_tasks.log')
-    @mock.patch('allura.tasks.export_tasks.M.Project.app_instance')
-    @mock.patch('allura.tasks.export_tasks.mail_tasks')
-    @td.with_tool('test', 'Tickets', 'bugs')
-    @td.with_tool('test', 'Blog', 'blog')
-    def test_bulk_export_not_exportable_tool(self, mail_tasks, app, log):
-        app.return_value.exportable = False
-        export_tasks.bulk_export([u'bugs', u'blog'])
-        log.info.assert_any_call('Tool bugs is not exportable. Skipping.')
-        log.info.assert_any_call('Tool blog is not exportable. Skipping.')
+    def test_bulk_export_filter_exportable(self):
+        exportable = mock.Mock(exportable=True)
+        not_exportable = mock.Mock(exportable=False)
+        BE = export_tasks.BulkExport()
+        self.assertEqual(BE.filter_exportable([None, exportable, not_exportable]), [exportable])
+
+    def test_bulk_export_filter_successful(self):
+        BE = export_tasks.BulkExport()
+        self.assertEqual(BE.filter_successful(['foo', None, '0']), ['foo', '0'])
+
+    def test_get_export_path(self):
+        BE = export_tasks.BulkExport()
+        path = BE.get_export_path('/tmp/bulk_export/p/test/', 'test-0.zip')
+        self.assertEqual(path, '/tmp/bulk_export/p/test/test-0')
 
     @mock.patch('allura.model.project.Project.__json__')
     @mock.patch('allura.tasks.export_tasks.shutil')
     @mock.patch('allura.tasks.export_tasks.zipdir')
     @mock.patch('forgewiki.wiki_main.ForgeWikiApp.bulk_export')
-    @mock.patch('allura.tasks.export_tasks.log')
     @td.with_wiki
-    def test_bulk_export(self, log, wiki_bulk_export, zipdir, shutil, project_json):
+    def test_bulk_export(self, wiki_bulk_export, zipdir, shutil, project_json):
         M.MonQTask.query.remove()
         export_tasks.bulk_export([u'wiki'])
-        log.info.assert_any_call('Exporting wiki...')
         wiki_bulk_export.assert_called_once()
         project_json.assert_called_once()
         temp = '/tmp/bulk_export/p/test/test'
@@ -493,23 +489,6 @@ class TestExportTasks(unittest.TestCase):
         assert_in('The following tools were exported:\n- wiki', text)
         assert_in('Sample instructions for test', text)
 
-    def test_create_export_dir(self):
-        project = M.Project.query.get(shortname='test')
-        export_path = project.bulk_export_path()
-        export_filename = project.bulk_export_filename()
-        path = export_tasks.create_export_dir(project, export_filename)
-        assert_equal(path, '/tmp/bulk_export/p/test/test')
-        assert os.path.exists(os.path.join(export_path, project.shortname))
-
-    @onlyif(os.path.exists(tg.config.get('scm.repos.tarball.zip_binary', '/usr/bin/zip')),
'zip binary is missing')
-    def test_zip_and_cleanup(self):
-        project = M.Project.query.get(shortname='test')
-        export_filename = project.bulk_export_filename()
-        path = export_tasks.create_export_dir(project, export_filename)
-        export_tasks.zip_and_cleanup(path, export_filename)
-        assert not os.path.exists(path)
-        assert os.path.exists(os.path.join(project.bulk_export_path(), 'test.zip'))
-
     def test_bulk_export_status(self):
         assert_equal(c.project.bulk_export_status(), None)
         export_tasks.bulk_export.post(['wiki'])


Mime
View raw message