incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tvansteenbu...@apache.org
Subject [1/4] git commit: [#6638] Added importer for exported Allura JSON
Date Wed, 18 Sep 2013 17:02:52 GMT
Updated Branches:
  refs/heads/master 66cb620e0 -> c6f4a9d0e


[#6638] Added importer for exported Allura JSON

Signed-off-by: Cory Johns <cjohns@slashdotmedia.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/886487cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/886487cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/886487cb

Branch: refs/heads/master
Commit: 886487cb3ec764e9cb5e9b4483e438814005ce3b
Parents: b6776af
Author: Cory Johns <cjohns@slashdotmedia.com>
Authored: Mon Sep 16 19:54:55 2013 +0000
Committer: Cory Johns <cjohns@slashdotmedia.com>
Committed: Mon Sep 16 20:04:08 2013 +0000

----------------------------------------------------------------------
 Allura/allura/model/discuss.py                  |   7 +-
 Allura/development.ini                          |   2 +
 ForgeImporters/forgeimporters/base.py           |  50 +++
 ForgeImporters/forgeimporters/forge/__init__.py |  17 +
 .../forge/templates/tracker/index.html          |  29 ++
 ForgeImporters/forgeimporters/forge/tracker.py  | 208 ++++++++++++
 .../forgeimporters/google/__init__.py           |  19 +-
 .../forgeimporters/tests/forge/__init__.py      |  17 +
 .../forgeimporters/tests/forge/test_tracker.py  | 331 +++++++++++++++++++
 .../tests/google/test_extractor.py              |   4 +-
 .../forgeimporters/tests/test_base.py           |  45 ++-
 ForgeImporters/setup.py                         |   1 +
 ForgeTracker/forgetracker/tracker_main.py       |   6 +
 13 files changed, 712 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/Allura/allura/model/discuss.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/discuss.py b/Allura/allura/model/discuss.py
index 03c991e..6df982f 100644
--- a/Allura/allura/model/discuss.py
+++ b/Allura/allura/model/discuss.py
@@ -167,11 +167,14 @@ class Thread(Artifact, ActivityObject):
             posts=[dict(slug=p.slug,
                         text=p.text,
                         subject=p.subject,
+                        author=p.author().username,
+                        timestamp=p.timestamp,
                         attachments=[dict(bytes=attach.length,
                                           url=h.absurl(attach.url())) for attach in p.attachments])
                    for p in self.post_class().query.find(
-                   dict(discussion_id=self.discussion_id, thread_id=self._id, status='ok')
-                   )]
+                       dict(discussion_id=self.discussion_id, thread_id=self._id, status='ok')
+                   )
+                ]
         )
 
     @property

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index ee9911b..4fe2f65 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -146,6 +146,8 @@ bulk_export_path = /tmp/bulk_export/{nbhd}/{project}
 bulk_export_filename = {project}-backup-{date:%Y-%m-%d-%H%M%S}.zip
 bulk_export_download_instructions = Sample instructions for {project}
 
+importer_upload_path = /tmp/importer_upload/{nbhd}/{project}
+
 # space-separated list of tool names that are valid options
 # for project admins to set for their 'support_page' field
 # this field is not used by default in Allura, so this option

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index d7479b3..79b12e0 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -15,11 +15,18 @@
 #       specific language governing permissions and limitations
 #       under the License.
 
+import os
+import errno
 import logging
 import urllib
 import urllib2
 from collections import defaultdict
 import traceback
+from urlparse import urlparse, parse_qs
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
 
 from BeautifulSoup import BeautifulSoup
 from tg import expose, validate, flash, redirect, config
@@ -459,3 +466,46 @@ class ImportAdminExtension(AdminExtension):
         base_url = c.project.url() + 'admin/ext/'
         link = SitemapEntry('Import', base_url+'import/')
         sidebar_links.append(link)
+
+
+def stringio_parser(page):
+    return {
+            'content-type': page.info()['content-type'],
+            'data': StringIO(page.read()),
+        }
+
+class File(object):
+    def __init__(self, url, filename=None):
+        extractor = ProjectExtractor(None, url, parser=stringio_parser)
+        self.url = url
+        self.filename = filename or os.path.basename(urlparse(url).path)
+        self.type = extractor.page['content-type'].split(';')[0]
+        self.file = extractor.page['data']
+
+
+def get_importer_upload_path(project):
+    shortname = project.shortname
+    if project.is_nbhd_project:
+        shortname = project.url().strip('/')
+    elif project.is_user_project:
+        shortname = project.shortname.split('/')[1]
+    elif not project.is_root:
+        shortname = project.shortname.split('/')[0]
+    upload_path = config['importer_upload_path'].format(
+            nbhd=project.neighborhood.url_prefix.strip('/'),
+            project=shortname,
+            c=c,
+        )
+    return upload_path
+
+def save_importer_upload(project, filename, data):
+    dest_path = get_importer_upload_path(project)
+    dest_file = os.path.join(dest_path, filename)
+    try:
+        os.makedirs(dest_path)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+    with open(dest_file, 'w') as fp:
+        fp.write(data)
+    return dest_file

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/ForgeImporters/forgeimporters/forge/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/forge/__init__.py b/ForgeImporters/forgeimporters/forge/__init__.py
new file mode 100644
index 0000000..77505f1
--- /dev/null
+++ b/ForgeImporters/forgeimporters/forge/__init__.py
@@ -0,0 +1,17 @@
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/ForgeImporters/forgeimporters/forge/templates/tracker/index.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/forge/templates/tracker/index.html b/ForgeImporters/forgeimporters/forge/templates/tracker/index.html
new file mode 100644
index 0000000..6e3a1d0
--- /dev/null
+++ b/ForgeImporters/forgeimporters/forge/templates/tracker/index.html
@@ -0,0 +1,29 @@
+{#-
+       Licensed to the Apache Software Foundation (ASF) under one
+       or more contributor license agreements.  See the NOTICE file
+       distributed with this work for additional information
+       regarding copyright ownership.  The ASF licenses this file
+       to you under the Apache License, Version 2.0 (the
+       "License"); you may not use this file except in compliance
+       with the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+       Unless required by applicable law or agreed to in writing,
+       software distributed under the License is distributed on an
+       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+       KIND, either express or implied.  See the License for the
+       specific language governing permissions and limitations
+       under the License.
+-#}
+{% extends 'forgeimporters:templates/importer_base.html' %}
+
+{% block importer_fields %}
+  <div>
+    <label for="json">Tickets JSON
+      <br><small>JSON file exported from Allura</small>
+    </label>
+    <input name="tickets_json" type="file" />
+    {{ error('json') }}
+  </div>
+{% endblock %}

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/ForgeImporters/forgeimporters/forge/tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/forge/tracker.py b/ForgeImporters/forgeimporters/forge/tracker.py
new file mode 100644
index 0000000..07efb0e
--- /dev/null
+++ b/ForgeImporters/forgeimporters/forge/tracker.py
@@ -0,0 +1,208 @@
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+
+import os
+from collections import defaultdict
+from datetime import datetime
+import json
+
+import dateutil.parser
+from formencode import validators as fev
+from pylons import tmpl_context as c
+from pylons import app_globals as g
+from ming.orm import session, ThreadLocalORMSession
+
+from tg import (
+        expose,
+        flash,
+        redirect,
+        validate,
+        )
+from tg.decorators import (
+        with_trailing_slash,
+        without_trailing_slash,
+        )
+
+from allura.controllers import BaseController
+from allura.lib import helpers as h
+from allura.lib.plugin import ImportIdConverter
+from allura.lib.decorators import require_post, task
+from allura.lib import validators as v
+from allura import model as M
+
+from forgetracker.tracker_main import ForgeTrackerApp
+from forgetracker import model as TM
+from forgeimporters.base import (
+        ToolImporter,
+        ToolImportForm,
+        ImportErrorHandler,
+        File,
+        get_importer_upload_path,
+        save_importer_upload,
+        )
+
+
+@task(notifications_disabled=True)
+def import_tool(**kw):
+    importer = ForgeTrackerImporter()
+    with ImportErrorHandler(importer, kw.get('project_name'), c.project):
+        importer.import_tool(c.project, c.user, **kw)
+
+
+class ForgeTrackerImportForm(ToolImportForm):
+    tickets_json = v.JsonFile(required=True)
+
+
+class ForgeTrackerImportController(BaseController):
+    def __init__(self):
+        self.importer = ForgeTrackerImporter()
+
+    @property
+    def target_app(self):
+        return self.importer.target_app
+
+    @with_trailing_slash
+    @expose('jinja:forgeimporters.forge:templates/tracker/index.html')
+    def index(self, **kw):
+        return dict(importer=self.importer,
+                target_app=self.target_app)
+
+    @without_trailing_slash
+    @expose()
+    @require_post()
+    @validate(ForgeTrackerImportForm(ForgeTrackerApp), error_handler=index)
+    def create(self, tickets_json, mount_point, mount_label, **kw):
+        if ForgeTrackerImporter().enforce_limit(c.project):
+            save_importer_upload(c.project, 'tickets.json', json.dumps(tickets_json))
+            import_tool.post(
+                    mount_point=mount_point,
+                    mount_label=mount_label,
+                )
+            flash('Ticket import has begun. Your new tracker will be available '
+                    'when the import is complete.')
+            redirect(c.project.url() + 'admin/')
+        else:
+            flash('There are too many imports pending at this time.  Please wait and try
again.', 'error')
+        redirect(c.project.url() + 'admin/')
+
+
+class ForgeTrackerImporter(ToolImporter):
+    source = 'Allura'
+    target_app = ForgeTrackerApp
+    controller = ForgeTrackerImportController
+    tool_label = 'Tickets'
+
+    def __init__(self, *args, **kwargs):
+        super(ForgeTrackerImporter, self).__init__(*args, **kwargs)
+        self.max_ticket_num = 0
+
+    def _load_json(self, project):
+        upload_path = get_importer_upload_path(project)
+        full_path = os.path.join(upload_path, 'tickets.json')
+        with open(full_path) as fp:
+            return json.load(fp)
+
+    def import_tool(self, project, user, mount_point=None,
+            mount_label=None, **kw):
+        import_id_converter = ImportIdConverter.get()
+        tracker_json = self._load_json(project)
+        tracker_json['tracker_config']['options'].pop('ordinal', None)
+        tracker_json['tracker_config']['options'].pop('mount_point', None)
+        tracker_json['tracker_config']['options'].pop('mount_label', None)
+        app = project.install_app('tickets', mount_point, mount_label,
+                import_id={
+                        'source': self.source,
+                        'app_config_id': tracker_json['tracker_config']['_id'],
+                    },
+                open_status_names=tracker_json['open_status_names'],
+                closed_status_names=tracker_json['closed_status_names'],
+                **tracker_json['tracker_config']['options']
+            )
+        ThreadLocalORMSession.flush_all()
+        try:
+            M.session.artifact_orm_session._get().skip_mod_date = True
+            for ticket_json in tracker_json['tickets']:
+                reporter = self.get_user(ticket_json['reported_by'])
+                with h.push_config(c, user=reporter, app=app):
+                    self.max_ticket_num = max(ticket_json['ticket_num'], self.max_ticket_num)
+                    ticket = TM.Ticket(
+                            app_config_id=app.config._id,
+                            import_id=import_id_converter.expand(ticket_json['ticket_num'],
app),
+                            description=self.annotate(ticket_json['description'], reporter,
ticket_json['reported_by']),
+                            created_date=dateutil.parser.parse(ticket_json['created_date']),
+                            mod_date=dateutil.parser.parse(ticket_json['mod_date']),
+                            ticket_num=ticket_json['ticket_num'],
+                            summary=ticket_json['summary'],
+                            custom_fields=ticket_json['custom_fields'],
+                            status=ticket_json['status'],
+                            labels=ticket_json['labels'],
+                            votes_down=ticket_json['votes_down'],
+                            votes_up=ticket_json['votes_up'],
+                            votes = ticket_json['votes_up'] - ticket_json['votes_down'],
+                        )
+                    ticket.private = ticket_json['private']  # trigger the private property
+                    self.process_comments(ticket, ticket_json['discussion_thread']['posts'])
+                    session(ticket).flush(ticket)
+                    session(ticket).expunge(ticket)
+            app.globals.custom_fields = tracker_json['custom_fields']
+            self.process_bins(app, tracker_json['saved_bins'])
+            app.globals.last_ticket_num = self.max_ticket_num
+            M.AuditLog.log(
+                    'import tool %s from exported Allura JSON' % (
+                            app.config.options.mount_point,
+                        ),
+                    project=project,
+                    user=user,
+                    url=app.url,
+                )
+            g.post_event('project_updated')
+            app.globals.invalidate_bin_counts()
+            ThreadLocalORMSession.flush_all()
+            return app
+        except Exception as e:
+            h.make_app_admin_only(app)
+            raise
+        finally:
+            M.session.artifact_orm_session._get().skip_mod_date = False
+
+    def get_user(self, username):
+        user = M.User.by_username(username)
+        if not user:
+            user = M.User.anonymous()
+        return user
+
+    def annotate(self, text, user, username):
+        if user._id is None:
+            return '*Originally by:* %s\n\n%s' % (username, text)
+        return text
+
+    def process_comments(self, ticket, comments):
+        for comment_json in comments:
+            user = self.get_user(comment_json['author'])
+            with h.push_config(c, user=user):
+                p = ticket.discussion_thread.add_post(
+                        text = self.annotate(comment_json['text'], user, comment_json['author']),
+                        ignore_security = True,
+                        timestamp = dateutil.parser.parse(comment_json['timestamp']),
+                    )
+                p.add_multiple_attachments([File(a['url']) for a in comment_json['attachments']])
+
+    def process_bins(self, app, bins):
+        TM.Bin.query.remove({'app_config_id': app.config._id})
+        for bin_json in bins:
+            bin_json.pop('_id', None)
+            TM.Bin(app_config_id=app.config._id, **bin_json)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/ForgeImporters/forgeimporters/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/__init__.py b/ForgeImporters/forgeimporters/google/__init__.py
index 849f924..e26568c 100644
--- a/ForgeImporters/forgeimporters/google/__init__.py
+++ b/ForgeImporters/forgeimporters/google/__init__.py
@@ -20,10 +20,6 @@ import urllib
 from urlparse import urlparse, urljoin, parse_qs
 from collections import defaultdict
 from contextlib import closing
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
 import logging
 
 from BeautifulSoup import BeautifulSoup
@@ -31,6 +27,7 @@ from BeautifulSoup import BeautifulSoup
 from allura.lib import helpers as h
 from allura import model as M
 from forgeimporters.base import ProjectExtractor
+from forgeimporters.base import File
 
 
 log = logging.getLogger(__name__)
@@ -63,12 +60,6 @@ def csv_parser(page):
     # remove CSV wrapping (quotes, commas, newlines)
     return [line.strip('",\n') for line in lines]
 
-def stringio_parser(page):
-    return {
-            'content-type': page.info()['content-type'],
-            'data': StringIO(page.read()),
-        }
-
 
 class GoogleCodeProjectExtractor(ProjectExtractor):
     BASE_URL = 'http://code.google.com'
@@ -263,14 +254,6 @@ class Comment(object):
             )
         return text
 
-class File(object):
-    def __init__(self, url, filename):
-        extractor = GoogleCodeProjectExtractor(None, url, parser=stringio_parser)
-        self.url = url
-        self.filename = filename
-        self.type = extractor.page['content-type'].split(';')[0]
-        self.file = extractor.page['data']
-
 class Attachment(File):
     def __init__(self, tag):
         url = urljoin(GoogleCodeProjectExtractor.BASE_URL, tag.get('href'))

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/ForgeImporters/forgeimporters/tests/forge/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/forge/__init__.py b/ForgeImporters/forgeimporters/tests/forge/__init__.py
new file mode 100644
index 0000000..77505f1
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/forge/__init__.py
@@ -0,0 +1,17 @@
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/ForgeImporters/forgeimporters/tests/forge/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/forge/test_tracker.py b/ForgeImporters/forgeimporters/tests/forge/test_tracker.py
new file mode 100644
index 0000000..9eeaec4
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/forge/test_tracker.py
@@ -0,0 +1,331 @@
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+
+from datetime import datetime
+from unittest import TestCase
+
+import mock
+from ming.odm import ThreadLocalORMSession
+import webtest
+
+from allura.tests import TestController
+from allura.tests.decorators import with_tracker
+
+from allura import model as M
+from forgeimporters.forge import tracker
+
+
+class TestTrackerImporter(TestCase):
+    @mock.patch.object(tracker.h, 'make_app_admin_only')
+    @mock.patch.object(tracker, 'g')
+    @mock.patch.object(tracker, 'c')
+    @mock.patch.object(tracker, 'ThreadLocalORMSession')
+    @mock.patch.object(tracker, 'session')
+    @mock.patch.object(tracker, 'M')
+    @mock.patch.object(tracker, 'TM')
+    def test_import_tool(self, TM, M, session, tlos, c, g, mao):
+        importer = tracker.ForgeTrackerImporter()
+        importer._load_json = mock.Mock(return_value={
+                'tracker_config': {
+                        '_id': 'orig_id',
+                        'options': {
+                                'foo': 'bar',
+                            },
+                    },
+                'open_status_names': 'open statuses',
+                'closed_status_names': 'closed statuses',
+                'custom_fields': 'fields',
+                'saved_bins': 'bins',
+                'tickets': [
+                        {
+                                'reported_by': 'rb1',
+                                'ticket_num': 1,
+                                'description': 'd1',
+                                'created_date': '2013-09-01',
+                                'mod_date': '2013-09-02',
+                                'summary': 's1',
+                                'custom_fields': 'cf1',
+                                'status': 'st1',
+                                'labels': 'l1',
+                                'votes_down': 1,
+                                'votes_up': 2,
+                                'private': False,
+                                'discussion_thread': {'posts': 'comments1'},
+                            },
+                        {
+                                'reported_by': 'rb2',
+                                'ticket_num': 100,
+                                'description': 'd2',
+                                'created_date': '2013-09-03',
+                                'mod_date': '2013-09-04',
+                                'summary': 's2',
+                                'custom_fields': 'cf2',
+                                'status': 'st2',
+                                'labels': 'l2',
+                                'votes_down': 3,
+                                'votes_up': 5,
+                                'private': True,
+                                'discussion_thread': {'posts': 'comments2'},
+                            },
+                    ],
+            })
+        reporter = mock.Mock()
+        importer.get_user = mock.Mock(return_value=reporter)
+        importer.annotate = mock.Mock(side_effect=['ad1', 'ad2'])
+        importer.process_comments = mock.Mock()
+        importer.process_bins = mock.Mock()
+        project, user = mock.Mock(), mock.Mock()
+        app = project.install_app.return_value
+        app.config.options.mount_point = 'mount_point'
+        app.config.options.import_id = {
+                'source': 'Allura',
+                'app_config_id': 'orig_id',
+            }
+        app.config.options.get = lambda *a: getattr(app.config.options, *a)
+        app.url = 'foo'
+        tickets = TM.Ticket.side_effect = [mock.Mock(), mock.Mock()]
+
+        importer.import_tool(project, user,
+                mount_point='mount_point', mount_label='mount_label')
+
+        project.install_app.assert_called_once_with('tickets', 'mount_point', 'mount_label',
+                open_status_names='open statuses',
+                closed_status_names='closed statuses',
+                import_id={
+                        'source': 'Allura',
+                        'app_config_id': 'orig_id',
+                    },
+                foo='bar',
+            )
+        self.assertEqual(importer.annotate.call_args_list, [
+                mock.call('d1', reporter, 'rb1'),
+                mock.call('d2', reporter, 'rb2'),
+            ])
+        self.assertEqual(TM.Ticket.call_args_list, [
+                mock.call(
+                        app_config_id=app.config._id,
+                        import_id={
+                                'source': 'Allura',
+                                'app_config_id': 'orig_id',
+                                'source_id': 1,
+                            },
+                        description='ad1',
+                        created_date=datetime(2013, 9, 1),
+                        mod_date=datetime(2013, 9, 2),
+                        ticket_num=1,
+                        summary='s1',
+                        custom_fields='cf1',
+                        status='st1',
+                        labels='l1',
+                        votes_down=1,
+                        votes_up=2,
+                        votes=1,
+                    ),
+                mock.call(
+                        app_config_id=app.config._id,
+                        import_id={
+                                'source': 'Allura',
+                                'app_config_id': 'orig_id',
+                                'source_id': 100,
+                            },
+                        description='ad2',
+                        created_date=datetime(2013, 9, 3),
+                        mod_date=datetime(2013, 9, 4),
+                        ticket_num=100,
+                        summary='s2',
+                        custom_fields='cf2',
+                        status='st2',
+                        labels='l2',
+                        votes_down=3,
+                        votes_up=5,
+                        votes=2,
+                    ),
+            ])
+        self.assertEqual(tickets[0].private, False)
+        self.assertEqual(tickets[1].private, True)
+        self.assertEqual(importer.process_comments.call_args_list, [
+                mock.call(tickets[0], 'comments1'),
+                mock.call(tickets[1], 'comments2'),
+            ])
+        self.assertEqual(tlos.flush_all.call_args_list, [
+                mock.call(),
+                mock.call(),
+            ])
+        self.assertEqual(session.return_value.flush.call_args_list, [
+                mock.call(tickets[0]),
+                mock.call(tickets[1]),
+            ])
+        self.assertEqual(session.return_value.expunge.call_args_list, [
+                mock.call(tickets[0]),
+                mock.call(tickets[1]),
+            ])
+        self.assertEqual(app.globals.custom_fields, 'fields')
+        importer.process_bins.assert_called_once_with(app, 'bins')
+        self.assertEqual(app.globals.last_ticket_num, 100)
+        M.AuditLog.log.assert_called_once_with(
+                'import tool mount_point from exported Allura JSON',
+                project=project, user=user, url='foo')
+        g.post_event.assert_called_once_with('project_updated')
+        app.globals.invalidate_bin_counts.assert_called_once_with()
+
+    @mock.patch.object(tracker, 'ThreadLocalORMSession')
+    @mock.patch.object(tracker, 'M')
+    @mock.patch.object(tracker, 'h')
+    def test_import_tool_failure(self, h, M, ThreadLocalORMSession):
+        M.session.artifact_orm_session._get.side_effect = ValueError
+        project = mock.Mock()
+        user = mock.Mock()
+        tracker_json = {
+                'tracker_config': {'_id': 'orig_id', 'options': {}},
+                'open_status_names': 'os',
+                'closed_status_names': 'cs',
+            }
+
+        importer = tracker.ForgeTrackerImporter()
+        importer._load_json = mock.Mock(return_value=tracker_json)
+        self.assertRaises(ValueError, importer.import_tool, project, user, project_name='project_name',
+                mount_point='mount_point', mount_label='mount_label')
+
+        h.make_app_admin_only.assert_called_once_with(project.install_app.return_value)
+
+    @mock.patch.object(tracker, 'M')
+    def test_get_user(self, M):
+        importer = tracker.ForgeTrackerImporter()
+        M.User.anonymous.return_value = 'anon'
+
+        M.User.by_username.return_value = 'bar'
+        self.assertEqual(importer.get_user('foo'), 'bar')
+        self.assertEqual(M.User.anonymous.call_count, 0)
+
+        M.User.by_username.return_value = None
+        self.assertEqual(importer.get_user('foo'), 'anon')
+        self.assertEqual(M.User.anonymous.call_count, 1)
+
+    def test_annotate(self):
+        importer = tracker.ForgeTrackerImporter()
+        user = mock.Mock(_id=1)
+        self.assertEqual(importer.annotate('foo', user, 'bar'), 'foo')
+        user._id = None
+        self.assertEqual(importer.annotate('foo', user, 'bar'), '*Originally by:* bar\n\nfoo')
+
+    @mock.patch.object(tracker, 'File')
+    @mock.patch.object(tracker, 'c')
+    def test_process_comments(self, c, File):
+        importer = tracker.ForgeTrackerImporter()
+        author = mock.Mock()
+        importer.get_user = mock.Mock(return_value=author)
+        importer.annotate = mock.Mock(side_effect=['at1', 'at2'])
+        ticket = mock.Mock()
+        add_post = ticket.discussion_thread.add_post
+        ama = add_post.return_value.add_multiple_attachments
+        File.side_effect = ['f1', 'f2', 'f3', 'f4']
+        comments = [
+                {
+                    'author': 'a1',
+                    'text': 't1',
+                    'timestamp': '2013-09-01',
+                    'attachments': [{'url': 'u1'}, {'url': 'u2'}],
+                },
+                {
+                    'author': 'a2',
+                    'text': 't2',
+                    'timestamp': '2013-09-02',
+                    'attachments': [{'url': 'u3'}, {'url': 'u4'}],
+                },
+            ]
+
+        importer.process_comments(ticket, comments)
+
+        self.assertEqual(importer.get_user.call_args_list, [mock.call('a1'), mock.call('a2')])
+        self.assertEqual(importer.annotate.call_args_list, [
+                mock.call('t1', author, 'a1'),
+                mock.call('t2', author, 'a2'),
+            ])
+        self.assertEqual(add_post.call_args_list, [
+                mock.call(text='at1', ignore_security=True, timestamp=datetime(2013, 9, 1)),
+                mock.call(text='at2', ignore_security=True, timestamp=datetime(2013, 9, 2)),
+            ])
+        self.assertEqual(File.call_args_list, [
+                mock.call('u1'),
+                mock.call('u2'),
+                mock.call('u3'),
+                mock.call('u4'),
+            ])
+        self.assertEqual(ama.call_args_list, [
+                mock.call(['f1', 'f2']),
+                mock.call(['f3', 'f4']),
+            ])
+
+    @mock.patch.object(tracker, 'TM')
+    def test_process_bins(self, TM):
+        app = mock.Mock()
+        app.config._id = 1
+        importer = tracker.ForgeTrackerImporter()
+        importer.process_bins(app, [{'_id': 1, 'b': 1}, {'b': 2}])
+        TM.Bin.query.remove.assert_called_once_with({'app_config_id': 1})
+        self.assertEqual(TM.Bin.call_args_list, [
+                mock.call(app_config_id=1, b=1),
+                mock.call(app_config_id=1, b=2),
+            ])
+
+
+class TestForgeTrackerImportController(TestController, TestCase):
+    def setUp(self):
+        """Mount Allura importer on the Tracker admin controller"""
+        super(TestForgeTrackerImportController, self).setUp()
+        from forgetracker.tracker_main import TrackerAdminController
+        TrackerAdminController._importer = tracker.ForgeTrackerImportController()
+
+    @with_tracker
+    def test_index(self):
+        r = self.app.get('/p/test/admin/bugs/_importer/')
+        self.assertIsNotNone(r.html.find(attrs=dict(name="tickets_json")))
+        self.assertIsNotNone(r.html.find(attrs=dict(name="mount_label")))
+        self.assertIsNotNone(r.html.find(attrs=dict(name="mount_point")))
+
+    @with_tracker
+    @mock.patch('forgeimporters.forge.tracker.save_importer_upload')
+    @mock.patch('forgeimporters.forge.tracker.import_tool')
+    def test_create(self, import_tool, sui):
+        project = M.Project.query.get(shortname='test')
+        params = {
+                'tickets_json': webtest.Upload('tickets.json', '{"key": "val"}'),
+                'mount_label': 'mylabel',
+                'mount_point': 'mymount',
+            }
+        r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+                status=302)
+        self.assertEqual(r.location, 'http://localhost/p/test/admin/')
+        sui.assert_called_once_with(project, 'tickets.json', '{"key": "val"}')
+        import_tool.post.assert_called_once_with(mount_point='mymount', mount_label='mylabel')
+
+    @with_tracker
+    @mock.patch('forgeimporters.forge.tracker.save_importer_upload')
+    @mock.patch('forgeimporters.forge.tracker.import_tool')
+    def test_create_limit(self, import_tool, sui):
+        project = M.Project.query.get(shortname='test')
+        project.set_tool_data('ForgeTrackerImporter', pending=1)
+        ThreadLocalORMSession.flush_all()
+        params = {
+                'tickets_json': webtest.Upload('tickets.json', '{"key": "val"}'),
+                'mount_label': 'mylabel',
+                'mount_point': 'mymount',
+            }
+        r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+                status=302).follow()
+        self.assertIn('Please wait and try again', r)
+        self.assertEqual(import_tool.post.call_count, 0)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/ForgeImporters/forgeimporters/tests/google/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_extractor.py b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
index 668662e..f305776 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
@@ -211,7 +211,7 @@ class TestGoogleCodeProjectExtractor(TestCase):
                 'OpSys-OSX',
             ])
 
-    @mock.patch.object(google, 'StringIO')
+    @mock.patch.object(base, 'StringIO')
     def test_get_issue_attachments(self, StringIO):
         self.urlopen.return_value.info.return_value = {'content-type': 'text/plain; foo'}
         test_issue = open(pkg_resources.resource_filename('forgeimporters', 'tests/data/google/test-issue.html')).read()
@@ -225,7 +225,7 @@ class TestGoogleCodeProjectExtractor(TestCase):
         self.assertEqual(attachments[1].url, 'http://allura-google-importer.googlecode.com/issues/attachment?aid=70000001&name=at2.txt&token=C9Hn4s1-g38hlSggRGo65VZM1ys%3A1376059941255')
         self.assertEqual(attachments[1].type, 'text/plain')
 
-    @mock.patch.object(google, 'StringIO')
+    @mock.patch.object(base, 'StringIO')
     def test_iter_comments(self, StringIO):
         test_issue = open(pkg_resources.resource_filename('forgeimporters', 'tests/data/google/test-issue.html')).read()
         gpe = self._make_extractor(test_issue)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/ForgeImporters/forgeimporters/tests/test_base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/test_base.py b/ForgeImporters/forgeimporters/tests/test_base.py
index 4aefc35..710d301 100644
--- a/ForgeImporters/forgeimporters/tests/test_base.py
+++ b/ForgeImporters/forgeimporters/tests/test_base.py
@@ -16,15 +16,18 @@
 #       under the License.
 
 from unittest import TestCase
+import errno
 
 from formencode import Invalid
 import mock
-from tg import expose
+from tg import expose, config
 from nose.tools import assert_equal, assert_raises
 
 from alluratest.controller import TestController
+from allura.tests import decorators as td
+from allura.lib import helpers as h
 
-from .. import base
+from forgeimporters import base
 
 
 class TestProjectExtractor(TestCase):
@@ -255,3 +258,41 @@ class TestProjectToolsImportController(TestController):
         url = import1_page.environ['PATH_INFO']
         assert url.endswith('/admin/ext/import/importer1'), url
         assert_equal(import1_page.body, 'test importer 1 controller webpage')
+
+
+def test_get_importer_upload_path():
+    project = mock.Mock(
+            shortname='prefix/shortname',
+            is_nbhd_project=False,
+            is_user_project=False,
+            is_root=False,
+            url=lambda: 'n_url/',
+            neighborhood=mock.Mock(url_prefix='p/'),
+        )
+    with h.push_config(config, importer_upload_path='path/{nbhd}/{project}'):
+        assert_equal(base.get_importer_upload_path(project), 'path/p/prefix')
+        project.is_nbhd_project = True
+        assert_equal(base.get_importer_upload_path(project), 'path/p/n_url')
+        project.is_nbhd_project = False
+        project.is_user_project = True
+        assert_equal(base.get_importer_upload_path(project), 'path/p/shortname')
+        project.is_user_project = False
+        project.is_root = True
+        assert_equal(base.get_importer_upload_path(project), 'path/p/prefix/shortname')
+
+@mock.patch.object(base, 'os')
+@mock.patch.object(base, 'get_importer_upload_path')
+def test_save_importer_upload(giup, os):
+    os.path.join = lambda *a: '/'.join(a)
+    giup.return_value = 'path'
+    os.makedirs.side_effect = OSError(errno.EEXIST, 'foo')
+    _open = mock.MagicMock()
+    fp = _open.return_value.__enter__.return_value
+    with mock.patch('__builtin__.open', _open):
+        base.save_importer_upload('project', 'file', 'data')
+    os.makedirs.assert_called_once_with('path')
+    _open.assert_called_once_with('path/file', 'w')
+    fp.write.assert_called_once_with('data')
+
+    os.makedirs.side_effect = OSError(errno.EACCES, 'foo')
+    assert_raises(OSError, base.save_importer_upload, 'project', 'file', 'data')

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/ForgeImporters/setup.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/setup.py b/ForgeImporters/setup.py
index bb8148b..eaeb7df 100644
--- a/ForgeImporters/setup.py
+++ b/ForgeImporters/setup.py
@@ -43,6 +43,7 @@ setup(name='ForgeImporters',
       google-code-repo = forgeimporters.google.code:GoogleRepoImporter
       github-repo = forgeimporters.github.code:GitHubRepoImporter
       trac-tickets = forgeimporters.trac.tickets:TracTicketImporter
+      forge-tracker = forgeimporters.forge.tracker:ForgeTrackerImporter
 
       [allura.admin]
       importers = forgeimporters.base:ImportAdminExtension

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/886487cb/ForgeTracker/forgetracker/tracker_main.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/tracker_main.py b/ForgeTracker/forgetracker/tracker_main.py
index 28bd4aa..1656c5b 100644
--- a/ForgeTracker/forgetracker/tracker_main.py
+++ b/ForgeTracker/forgetracker/tracker_main.py
@@ -422,6 +422,12 @@ class ForgeTrackerApp(Application):
         f.write(',\n"milestones":')
         milestones = self.milestones
         json.dump(milestones, f, cls=jsonify.GenericJSON, indent=2)
+        f.write(',\n"custom_fields":')
+        json.dump(self.globals.custom_fields, f, cls=jsonify.GenericJSON, indent=2)
+        f.write(',\n"open_status_names":')
+        json.dump(self.globals.open_status_names, f, cls=jsonify.GenericJSON, indent=2)
+        f.write(',\n"closed_status_names":')
+        json.dump(self.globals.closed_status_names, f, cls=jsonify.GenericJSON, indent=2)
         f.write(',\n"saved_bins":')
         bins = self.bins
         json.dump(bins, f, cls=jsonify.GenericJSON, indent=2)


Mime
View raw message