incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject [1/6] git commit: [#6480] Improvements to Trac importer
Date Fri, 16 Aug 2013 00:30:04 GMT
Updated Branches:
  refs/heads/master 6657808db -> 3e67aa667


[#6480] Improvements to Trac importer

- Disable email notifications during import
- Retry time-out HTTP requests
- Add option to supply user-map file

Signed-off-by: Tim Van Steenburgh <tvansteenburgh@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/675495e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/675495e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/675495e3

Branch: refs/heads/master
Commit: 675495e39723e593cba00cdcf84e3cb02d00e4b0
Parents: 6657808
Author: Tim Van Steenburgh <tvansteenburgh@gmail.com>
Authored: Mon Aug 12 18:06:43 2013 +0000
Committer: Cory Johns <cjohns@slashdotmedia.com>
Committed: Thu Aug 15 16:08:41 2013 +0000

----------------------------------------------------------------------
 Allura/allura/lib/decorators.py                 | 75 +++++++++++---------
 Allura/allura/lib/helpers.py                    | 71 +++++++++++++++++-
 Allura/allura/lib/import_api.py                 |  2 +-
 Allura/allura/lib/spam/__init__.py              |  2 +-
 Allura/allura/lib/validators.py                 | 30 ++++++++
 Allura/allura/scripts/trac_export.py            |  9 ++-
 Allura/allura/tests/test_decorators.py          | 39 ++++++++++
 Allura/allura/tests/test_helpers.py             | 42 +++++++++++
 Allura/allura/tests/test_validators.py          | 36 ++++++++++
 ForgeBlog/forgeblog/command/rssfeeds.py         |  2 +-
 ForgeImporters/forgeimporters/base.py           |  5 +-
 .../forgeimporters/templates/project_base.html  | 11 ++-
 ForgeImporters/forgeimporters/trac/project.py   |  2 +
 .../forgeimporters/trac/templates/project.html  | 14 +++-
 .../trac/templates/tickets/index.html           |  5 +-
 .../forgeimporters/trac/tests/test_tickets.py   | 12 +++-
 ForgeImporters/forgeimporters/trac/tickets.py   | 12 ++--
 ForgeTracker/forgetracker/import_support.py     | 24 +++++--
 ForgeTracker/forgetracker/tracker_main.py       | 23 +++---
 ForgeWiki/forgewiki/wiki_main.py                | 23 +++---
 20 files changed, 354 insertions(+), 85 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/decorators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/decorators.py b/Allura/allura/lib/decorators.py
index 36fb2de..137152e 100644
--- a/Allura/allura/lib/decorators.py
+++ b/Allura/allura/lib/decorators.py
@@ -28,16 +28,49 @@ from tg import request, redirect
 
 from webob import exc
 
-def task(func):
-    '''Decorator to add some methods to task functions'''
-    def post(*args, **kwargs):
-        from allura import model as M
-        delay = kwargs.pop('delay', 0)
-        return M.MonQTask.post(func, args, kwargs, delay=delay)
-    # if decorating a class, have to make it a staticmethod
-    # or it gets a spurious cls argument
-    func.post = staticmethod(post) if inspect.isclass(func) else post
-    return func
+from pylons import tmpl_context as c
+from allura.lib import helpers as h
+
+
+def _get_model():
+    from allura import model as M
+    return M
+
+def task(*args, **kw):
+    """Decorator that adds a ``.post()`` function to the decorated callable.
+
+    Calling <original_callable>.post(*args, **kw) queues the callable for
+    execution by a background worker process. All parameters must be
+    BSON-serializable.
+
+    Example usage:
+
+    @task
+    def myfunc():
+        pass
+
+    @task(notifications_disabled=True)
+    def myotherfunc():
+        # No email notifications will be sent for c.project during this task
+        pass
+
+    """
+    def task_(func):
+        def post(*args, **kwargs):
+            delay = kwargs.pop('delay', 0)
+            project = getattr(c, 'project', None)
+            cm = (h.notifications_disabled if project and
+                    kw.get('notifications_disabled') else h.null_contextmanager)
+            with cm(project):
+                M = _get_model()
+                return M.MonQTask.post(func, args, kwargs, delay=delay)
+        # if decorating a class, have to make it a staticmethod
+        # or it gets a spurious cls argument
+        func.post = staticmethod(post) if inspect.isclass(func) else post
+        return func
+    if len(args) == 1 and callable(args[0]):
+        return task_(args[0])
+    return task_
 
 class event_handler(object):
     '''Decorator to register event handlers'''
@@ -155,28 +188,6 @@ class log_action(object): # pragma no cover
         extra['referer_link'] = referer_link
         return extra
 
-class exceptionless(object):
-    '''Decorator making the decorated function return 'error_result' on any
-    exceptions rather than propagating exceptions up the stack
-    '''
-
-    def __init__(self, error_result, log=None):
-        self.error_result = error_result
-        self.log = log
-
-    def __call__(self, fun):
-        fname = 'exceptionless(%s)' % fun.__name__
-        def inner(*args, **kwargs):
-            try:
-                return fun(*args, **kwargs)
-            except Exception as e:
-                if self.log:
-                    self.log.exception('Error calling %s(args=%s, kwargs=%s): %s',
-                            fname, args, kwargs, str(e))
-                return self.error_result
-        inner.__name__ = fname
-        return inner
-
 def Property(function):
     '''Decorator to easily assign descriptors based on sub-function names
     See <http://code.activestate.com/recipes/410698-property-decorator-for-python-24/>

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/helpers.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index 1604aa9..0511dc7 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -22,6 +22,7 @@ import os
 import os.path
 import difflib
 import urllib
+import urllib2
 import re
 import json
 import logging
@@ -30,6 +31,7 @@ from hashlib import sha1
 from datetime import datetime, timedelta
 from collections import defaultdict
 import shlex
+import socket
 
 import tg
 import genshi.template
@@ -53,10 +55,10 @@ from webhelpers import date, feedgenerator, html, number, misc, text
 
 from allura.lib import exceptions as exc
 # Reimport to make available to templates
-from allura.lib.decorators import exceptionless
 from allura.lib import AsciiDammit
 from .security import has_access
 
+log = logging.getLogger(__name__)
 
 # validates project, subproject, and user names
 re_project_name = re.compile(r'^[a-z][-a-z0-9]{2,14}$')
@@ -857,3 +859,70 @@ def split_select_field_options(field_options):
         # so we're getting rid of those.
         field_options = [o.replace('"', '') for o in field_options]
     return field_options
+
+
+@contextmanager
+def notifications_disabled(project):
+    """Temporarily disable email notifications on a project.
+
+    """
+    orig = project.notifications_disabled
+    try:
+        project.notifications_disabled = True
+        yield
+    finally:
+        project.notifications_disabled = orig
+
+
+@contextmanager
+def null_contextmanager(*args, **kw):
+    """A no-op contextmanager.
+
+    """
+    yield
+
+
+class exceptionless(object):
+    '''Decorator making the decorated function return 'error_result' on any
+    exceptions rather than propagating exceptions up the stack
+    '''
+
+    def __init__(self, error_result, log=None):
+        self.error_result = error_result
+        self.log = log
+
+    def __call__(self, fun):
+        fname = 'exceptionless(%s)' % fun.__name__
+        def inner(*args, **kwargs):
+            try:
+                return fun(*args, **kwargs)
+            except Exception as e:
+                if self.log:
+                    self.log.exception('Error calling %s(args=%s, kwargs=%s): %s',
+                            fname, args, kwargs, str(e))
+                return self.error_result
+        inner.__name__ = fname
+        return inner
+
+
+def urlopen(url, retries=3, codes=(408,)):
+    """Open url, optionally retrying if an error is encountered.
+
+    Socket timeouts will always be retried if retries > 0.
+    HTTP errors are retried if the error code is passed in ``codes``.
+
+    :param retries: Number of time to retry.
+    :param codes: HTTP error codes that should be retried.
+
+    """
+    while True:
+        try:
+            return urllib2.urlopen(url)
+        except (urllib2.HTTPError, socket.timeout) as e:
+            if retries and (isinstance(e, socket.timeout) or
+                    e.code in codes):
+                retries -= 1
+                continue
+            else:
+                log.exception('Failed after %s retries: %s', retries, e)
+                raise

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/import_api.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/import_api.py b/Allura/allura/lib/import_api.py
index 1f3f98d..8eeb8a7 100644
--- a/Allura/allura/lib/import_api.py
+++ b/Allura/allura/lib/import_api.py
@@ -49,7 +49,7 @@ class AlluraImportApiClient(object):
         url = urlparse.urljoin(self.base_url, url)
         if self.verbose:
             print "Using URL '%s'" % (url)
-        
+
         params = self.sign(urlparse.urlparse(url).path, params.items())
 
         while True:

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/spam/__init__.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/spam/__init__.py b/Allura/allura/lib/spam/__init__.py
index 8e6607a..cfb5c41 100644
--- a/Allura/allura/lib/spam/__init__.py
+++ b/Allura/allura/lib/spam/__init__.py
@@ -17,7 +17,7 @@
 
 import logging
 
-from allura.lib.decorators import exceptionless
+from allura.lib.helpers import exceptionless
 
 log = logging.getLogger(__name__)
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/validators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/validators.py b/Allura/allura/lib/validators.py
index ae863e8..9b44d53 100644
--- a/Allura/allura/lib/validators.py
+++ b/Allura/allura/lib/validators.py
@@ -151,6 +151,36 @@ class JsonConverter(fev.FancyValidator):
             raise fe.Invalid('Invalid JSON: ' + str(e), value, state)
         return obj
 
+class JsonFile(fev.FieldStorageUploadConverter):
+    """Validates that a file is JSON and returns the deserialized Python object
+
+    """
+    def _to_python(self, value, state):
+        return JsonConverter.to_python(value.value)
+
+class UserMapJsonFile(JsonFile):
+    """Validates that a JSON file conforms to this format:
+
+    {str:str, ...}
+
+    and returns a deserialized or stringified copy of it.
+
+    """
+    def __init__(self, as_string=False):
+        self.as_string = as_string
+
+    def _to_python(self, value, state):
+        value = super(self.__class__, self)._to_python(value, state)
+        try:
+            for k, v in value.iteritems():
+                if not(isinstance(k, basestring) and isinstance(v, basestring)):
+                    raise
+            return json.dumps(value) if self.as_string else value
+        except:
+            raise fe.Invalid(
+                    'User map file must contain mapping of {str:str, ...}',
+                    value, state)
+
 class CreateTaskSchema(fe.Schema):
     task = TaskValidator(not_empty=True, strip=True)
     task_args = JsonConverter(if_missing=dict(args=[], kwargs={}))

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/scripts/trac_export.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/trac_export.py b/Allura/allura/scripts/trac_export.py
index e45175b..d53afbc 100644
--- a/Allura/allura/scripts/trac_export.py
+++ b/Allura/allura/scripts/trac_export.py
@@ -18,6 +18,7 @@
 #       under the License.
 
 import logging
+import socket
 import sys
 import csv
 import urlparse
@@ -34,6 +35,8 @@ from BeautifulSoup import BeautifulSoup, NavigableString
 import dateutil.parser
 import pytz
 
+from allura.lib import helpers as h
+
 log = logging.getLogger(__name__)
 
 
@@ -121,7 +124,7 @@ class TracExport(object):
 
     def csvopen(self, url):
         self.log_url(url)
-        f = urllib2.urlopen(url)
+        f = h.urlopen(url)
         # Trac doesn't throw 403 error, just shows normal 200 HTML page
         # telling that access denied. So, we'll emulate 403 ourselves.
         # TODO: currently, any non-csv result treated as 403.
@@ -143,7 +146,7 @@ class TracExport(object):
         from html2text import html2text
         url = self.full_url(self.TICKET_URL % id, 'rss')
         self.log_url(url)
-        d = feedparser.parse(url)
+        d = feedparser.parse(h.urlopen(url))
         res = []
         for comment in d['entries']:
             c = {}
@@ -160,7 +163,7 @@ class TracExport(object):
         # Scrape HTML to get ticket attachments
         url = self.full_url(self.ATTACHMENT_LIST_URL % id)
         self.log_url(url)
-        f = urllib2.urlopen(url)
+        f = h.urlopen(url)
         soup = BeautifulSoup(f)
         attach = soup.find('div', id='attachments')
         list = []

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/tests/test_decorators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_decorators.py b/Allura/allura/tests/test_decorators.py
new file mode 100644
index 0000000..893f327
--- /dev/null
+++ b/Allura/allura/tests/test_decorators.py
@@ -0,0 +1,39 @@
+from unittest import TestCase
+
+from mock import patch
+
+from allura.lib.decorators import task
+
+
+class TestTask(TestCase):
+
+    def test_no_params(self):
+        @task
+        def func():
+            pass
+        self.assertTrue(hasattr(func, 'post'))
+
+    def test_with_params(self):
+        @task(disable_notifications=True)
+        def func():
+            pass
+        self.assertTrue(hasattr(func, 'post'))
+
+    @patch('allura.lib.decorators.c')
+    @patch('allura.lib.decorators._get_model')
+    def test_post(self, c, _get_model):
+        @task(disable_notifications=True)
+        def func(s, foo=None, **kw):
+            pass
+        def mock_post(f, args, kw, delay=None):
+            self.assertTrue(c.project.notifications_disabled)
+            self.assertFalse('delay' in kw)
+            self.assertEqual(delay, 1)
+            self.assertEqual(kw, dict(foo=2))
+            self.assertEqual(args, ('test',))
+            self.assertEqual(f, func)
+
+        c.project.notifications_disabled = False
+        M = _get_model.return_value
+        M.MonQTask.post.side_effect = mock_post
+        func.post('test', foo=2, delay=1)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/tests/test_helpers.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_helpers.py b/Allura/allura/tests/test_helpers.py
index 42fb963..4ec91df 100644
--- a/Allura/allura/tests/test_helpers.py
+++ b/Allura/allura/tests/test_helpers.py
@@ -254,3 +254,45 @@ def test_datetimeformat():
 def test_split_select_field_options():
     assert_equals(h.split_select_field_options('"test message" test2'), ['test message',
'test2'])
     assert_equals(h.split_select_field_options('"test message test2'), ['test', 'message',
'test2'])
+
+
+def test_notifications_disabled():
+    project = Mock(notifications_disabled=False)
+    with h.notifications_disabled(project):
+        assert_equals(project.notifications_disabled, True)
+    assert_equals(project.notifications_disabled, False)
+
+
+class TestUrlOpen(TestCase):
+    @patch('allura.lib.helpers.urllib2')
+    def test_no_error(self, urllib2):
+        r = h.urlopen('myurl')
+        self.assertEqual(r, urllib2.urlopen.return_value)
+        urllib2.urlopen.assert_called_once_with('myurl')
+
+    @patch('allura.lib.helpers.urllib2.urlopen')
+    def test_socket_timeout(self, urlopen):
+        import socket
+        def side_effect(url):
+            raise socket.timeout()
+        urlopen.side_effect = side_effect
+        self.assertRaises(socket.timeout, h.urlopen, 'myurl')
+        self.assertEqual(urlopen.call_count, 4)
+
+    @patch('allura.lib.helpers.urllib2.urlopen')
+    def test_handled_http_error(self, urlopen):
+        from urllib2 import HTTPError
+        def side_effect(url):
+            raise HTTPError('url', 408, 'timeout', None, None)
+        urlopen.side_effect = side_effect
+        self.assertRaises(HTTPError, h.urlopen, 'myurl')
+        self.assertEqual(urlopen.call_count, 4)
+
+    @patch('allura.lib.helpers.urllib2.urlopen')
+    def test_unhandled_http_error(self, urlopen):
+        from urllib2 import HTTPError
+        def side_effect(url):
+            raise HTTPError('url', 404, 'timeout', None, None)
+        urlopen.side_effect = side_effect
+        self.assertRaises(HTTPError, h.urlopen, 'myurl')
+        self.assertEqual(urlopen.call_count, 1)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/tests/test_validators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_validators.py b/Allura/allura/tests/test_validators.py
index 8a40610..b0d165c 100644
--- a/Allura/allura/tests/test_validators.py
+++ b/Allura/allura/tests/test_validators.py
@@ -44,6 +44,42 @@ class TestJsonConverter(unittest.TestCase):
             self.val.to_python('{')
 
 
+class TestJsonFile(unittest.TestCase):
+    val = v.JsonFile
+
+    class FieldStorage(object):
+        def __init__(self, content):
+            self.value = content
+
+    def test_valid(self):
+        self.assertEqual({}, self.val.to_python(self.FieldStorage('{}')))
+
+    def test_invalid(self):
+        with self.assertRaises(fe.Invalid):
+            self.val.to_python(self.FieldStorage('{'))
+
+
+class TestUserMapFile(unittest.TestCase):
+    val = v.UserMapJsonFile()
+
+    class FieldStorage(object):
+        def __init__(self, content):
+            self.value = content
+
+    def test_valid(self):
+        self.assertEqual({"user_old": "user_new"}, self.val.to_python(
+            self.FieldStorage('{"user_old": "user_new"}')))
+
+    def test_invalid(self):
+        with self.assertRaises(fe.Invalid):
+            self.val.to_python(self.FieldStorage('{"user_old": 1}'))
+
+    def test_as_string(self):
+        val = v.UserMapJsonFile(as_string=True)
+        self.assertEqual('{"user_old": "user_new"}', val.to_python(
+            self.FieldStorage('{"user_old": "user_new"}')))
+
+
 class TestUserValidator(unittest.TestCase):
     val = v.UserValidator
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeBlog/forgeblog/command/rssfeeds.py
----------------------------------------------------------------------
diff --git a/ForgeBlog/forgeblog/command/rssfeeds.py b/ForgeBlog/forgeblog/command/rssfeeds.py
index 9f44fd4..305bc5a 100644
--- a/ForgeBlog/forgeblog/command/rssfeeds.py
+++ b/ForgeBlog/forgeblog/command/rssfeeds.py
@@ -33,7 +33,7 @@ from forgeblog import model as BM
 from forgeblog import version
 from forgeblog.main import ForgeBlogApp
 from allura.lib import exceptions
-from allura.lib.decorators import exceptionless
+from allura.lib.helpers import exceptionless
 
 ## Everything in this file depends on html2text,
 ## so import attempt is placed in global scope.

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 8cc3b52..e1af4f7 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -50,7 +50,7 @@ class ProjectImportForm(schema.Schema):
     project_name = fev.UnicodeString(not_empty=True, max=40)
 
 
-@task
+@task(notifications_disabled=True)
 def import_tool(importer_name, project_name=None, mount_point=None, mount_label=None, **kw):
     importer = ToolImporter.by_name(importer_name)
     importer.import_tool(c.project, c.user, project_name=project_name,
@@ -61,8 +61,9 @@ class ProjectImporter(BaseController):
     """
     Base class for project importers.
 
-    Subclases are required to implement the :meth:`index()` and
+    Subclasses are required to implement the :meth:`index()` and
     :meth:`process()` views described below.
+
     """
     source = None
     process_validator = None

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/templates/project_base.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/templates/project_base.html b/ForgeImporters/forgeimporters/templates/project_base.html
index 6f1b683..9453e65 100644
--- a/ForgeImporters/forgeimporters/templates/project_base.html
+++ b/ForgeImporters/forgeimporters/templates/project_base.html
@@ -53,11 +53,10 @@
         }
 
         function check_names() {
-            var data = {
-                'neighborhood': $('#neighborhood').val(),
-                'project_name': $('#project_name').val(),
-                'project_shortname': $('#project_shortname').val()
-            };
+            var data = {};
+            $('#project-import-form input').each(function() {
+              data[$(this).attr('name')] = $(this).val();
+            });
             $.getJSON('check_names', data, function(result) {
                 $('#project_name_error').addClass('hidden');
                 $('#project_shortname_error').addClass('hidden');
@@ -86,7 +85,7 @@
 {% endblock %}
 
 {% block content %}
-<form id="project-import-form" method="POST" action="process">
+<form id="project-import-form" method="POST" action="process" enctype="multipart/form-data">
     <input type="hidden" id="neighborhood" name="neighborhood" value="{{importer.neighborhood.name}}"/>
 
     <fieldset id="project-fields">

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/project.py b/ForgeImporters/forgeimporters/trac/project.py
index 66e8326..71ef5f4 100644
--- a/ForgeImporters/forgeimporters/trac/project.py
+++ b/ForgeImporters/forgeimporters/trac/project.py
@@ -23,6 +23,7 @@ from tg import expose, validate
 from tg.decorators import with_trailing_slash
 
 from allura.lib.decorators import require_post
+from allura.lib.validators import UserMapJsonFile
 
 from .. import base
 
@@ -32,6 +33,7 @@ log = logging.getLogger(__name__)
 
 class TracProjectForm(base.ProjectImportForm):
     trac_url = fev.URL(not_empty=True)
+    user_map = UserMapJsonFile(as_string=True)
 
 
 class TracProjectImporter(base.ProjectImporter):

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/templates/project.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/templates/project.html b/ForgeImporters/forgeimporters/trac/templates/project.html
index 869e290..a9af734 100644
--- a/ForgeImporters/forgeimporters/trac/templates/project.html
+++ b/ForgeImporters/forgeimporters/trac/templates/project.html
@@ -24,9 +24,21 @@
     </div>
     <div class="grid-10">
         <input id="trac_url" name="trac_url" value="{{c.form_values['trac_url']}}" autofocus/>
-        <div id="trac_ur_errorl" class="error{% if not c.form_errors['trac_url'] %} hidden{%
endif %}">
+        <div id="trac_url_error" class="error{% if not c.form_errors['trac_url'] %} hidden{%
endif %}">
             {{c.form_errors['trac_url']}}
         </div>
     </div>
+
+    <div class="grid-6" style="clear:left">
+        <label>User Map (optional)</label>
+    </div>
+    <div class="grid-10">
+        <input id="user_map" name="user_map" value="{{c.form_values['user_map']}}" type="file"/>
+        <br><small>JSON file mapping Trac usernames to Allura usernames</small>
+        <div id="user_map_error" class="error{% if not c.form_errors['user_map'] %} hidden{%
endif %}">
+            {{c.form_errors['user_map']}}
+        </div>
+    </div>
+
     {{ super() }}
 {% endblock %}

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/templates/tickets/index.html b/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
index eaf9aac..7c278b0 100644
--- a/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
+++ b/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
@@ -27,10 +27,13 @@ Import tickets from Trac
 {% endblock %}
 
 {% block content %}
-<form action="create" method="post" class="pad">
+<form action="create" method="post" enctype="multipart/form-data" class="pad">
   <label for="trac_url">URL of the Trac instance</label>
   <input name="trac_url" />
 
+  <label for="user_map">JSON file mapping Trac usernames to Allura usernames (optional)</label>
+  <input name="user_map" type="file" />
+
   <label for="mount_label">Label</label>
   <input name="mount_label" value="Source" />
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
index 5f88eef..2a539df 100644
--- a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
@@ -15,6 +15,7 @@
 #       specific language governing permissions and limitations
 #       under the License.
 
+import json
 from unittest import TestCase
 from mock import Mock, patch
 
@@ -39,7 +40,7 @@ class TestTracTicketImporter(TestCase):
         from datetime import datetime, timedelta
         now = datetime.utcnow()
         dt.utcnow.return_value = now
-
+        user_map = {"orig_user":"new_user"}
         importer = TracTicketImporter()
         app = Mock(name='ForgeTrackerApp')
         project = Mock(name='Project', shortname='myproject')
@@ -48,7 +49,9 @@ class TestTracTicketImporter(TestCase):
         res = importer.import_tool(project, user,
                 mount_point='bugs',
                 mount_label='Bugs',
-                trac_url='http://example.com/trac/url')
+                trac_url='http://example.com/trac/url',
+                user_map=json.dumps(user_map),
+                )
         self.assertEqual(res, app)
         project.install_app.assert_called_once_with(
                 'Tickets', mount_point='bugs', mount_label='Bugs')
@@ -60,7 +63,8 @@ class TestTracTicketImporter(TestCase):
                 expires=now + timedelta(minutes=60))
         api_client = ApiClient.return_value
         import_tracker.assert_called_once_with(
-                api_client, 'myproject', 'bugs', {}, '[]',
+                api_client, 'myproject', 'bugs',
+                {"user_map": user_map}, '[]',
                 validate=False)
         g.post_event.assert_called_once_with('project_updated')
 
@@ -91,10 +95,12 @@ class TestTracTicketImportController(TestController, TestCase):
                 mount_point='mymount',
                 )
         r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+                upload_files=[('user_map', 'myfile', '{"orig_user": "new_user"}')],
                 status=302)
         project = M.Project.query.get(shortname='test')
         self.assertEqual(r.location, 'http://localhost/p/test/mymount')
         self.assertEqual(project._id, importer.import_tool.call_args[0][0]._id)
         self.assertEqual(u'mymount', importer.import_tool.call_args[1]['mount_point'])
         self.assertEqual(u'mylabel', importer.import_tool.call_args[1]['mount_label'])
+        self.assertEqual('{"orig_user": "new_user"}', importer.import_tool.call_args[1]['user_map'])
         self.assertEqual(u'http://example.com/trac/url', importer.import_tool.call_args[1]['trac_url'])

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index f7d50b4..00bca2a 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -41,6 +41,7 @@ from tg.decorators import (
 from allura.controllers import BaseController
 from allura.lib.decorators import require_post
 from allura.lib.import_api import AlluraImportApiClient
+from allura.lib.validators import UserMapJsonFile
 from allura.model import ApiTicket
 from allura.scripts.trac_export import (
         TracExport,
@@ -54,6 +55,7 @@ from forgetracker.scripts.import_tracker import import_tracker
 
 class TracTicketImportSchema(fe.Schema):
     trac_url = fev.URL(not_empty=True)
+    user_map = UserMapJsonFile(as_string=True)
     mount_point = fev.UnicodeString()
     mount_label = fev.UnicodeString()
 
@@ -68,11 +70,12 @@ class TracTicketImportController(BaseController):
     @expose()
     @require_post()
     @validate(TracTicketImportSchema(), error_handler=index)
-    def create(self, trac_url, mount_point, mount_label, **kw):
+    def create(self, trac_url, mount_point, mount_label, user_map=None, **kw):
         app = TracTicketImporter().import_tool(c.project, c.user,
                 mount_point=mount_point,
                 mount_label=mount_label,
-                trac_url=trac_url)
+                trac_url=trac_url,
+                user_map=user_map)
         redirect(app.url())
 
 
@@ -84,7 +87,7 @@ class TracTicketImporter(ToolImporter):
     tool_description = 'Import your tickets from Trac'
 
     def import_tool(self, project, user, project_name=None, mount_point=None,
-            mount_label=None, trac_url=None, **kw):
+            mount_label=None, trac_url=None, user_map=None, **kw):
         """ Import Trac tickets into a new Allura Tracker tool.
 
         """
@@ -105,7 +108,8 @@ class TracTicketImporter(ToolImporter):
         session(api_ticket).flush(api_ticket)
         cli = AlluraImportApiClient(config['base_url'], api_ticket.api_key,
                 api_ticket.secret_key, False)
-        import_tracker(cli, project.shortname, mount_point, {},
+        import_tracker(cli, project.shortname, mount_point,
+                {'user_map': json.loads(user_map) if user_map else {}},
                 export_string, validate=False)
         g.post_event('project_updated')
         return app

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeTracker/forgetracker/import_support.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/import_support.py b/ForgeTracker/forgetracker/import_support.py
index 26e182e..04d453a 100644
--- a/ForgeTracker/forgetracker/import_support.py
+++ b/ForgeTracker/forgetracker/import_support.py
@@ -136,11 +136,11 @@ class ImportSupport(object):
         return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')
 
     def get_user_id(self, username):
-        username = self.options['user_map'].get(username, username)
+        username = self.options['user_map'].get(username)
+        if not username:
+            return None
         u = M.User.by_username(username)
-        if u:
-            return u._id
-        return None
+        return u._id if u else None
 
     def check_custom_field(self, field, value):
         field = c.app.globals.get_custom_field(field)
@@ -193,7 +193,12 @@ class ImportSupport(object):
                 new_f, conv = transform
                 remapped[new_f] = conv(v)
 
-        remapped['description'] = self.link_processing(remapped['description'])
+        description = self.link_processing(remapped['description'])
+        if ticket_dict['submitter'] and not remapped['reported_by_id']:
+            description = 'Originally created by: {0}\n\n{1}'.format(
+                    ticket_dict['submitter'], description)
+        remapped['description'] = description
+
         ticket_num = ticket_dict['id']
         existing_ticket = TM.Ticket.query.get(app_config_id=c.app.config._id,
                                           ticket_num=ticket_num)
@@ -261,8 +266,13 @@ class ImportSupport(object):
 
     def make_comment(self, thread, comment_dict):
         ts = self.parse_date(comment_dict['date'])
-        comment = thread.post(text=self.link_processing(comment_dict['comment']), timestamp=ts)
-        comment.author_id = self.get_user_id(comment_dict['submitter'])
+        author_id = self.get_user_id(comment_dict['submitter'])
+        text = self.link_processing(comment_dict['comment'])
+        if not author_id and comment_dict['submitter']:
+            text = 'Originally posted by: {0}\n\n{1}'.format(
+                    comment_dict['submitter'], text)
+        comment = thread.post(text=text, timestamp=ts)
+        comment.author_id = author_id
         comment.import_id = c.api_token.api_key
 
     def make_attachment(self, org_ticket_id, ticket_id, att_dict):

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeTracker/forgetracker/tracker_main.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/tracker_main.py b/ForgeTracker/forgetracker/tracker_main.py
index a1363dc..2b432df 100644
--- a/ForgeTracker/forgetracker/tracker_main.py
+++ b/ForgeTracker/forgetracker/tracker_main.py
@@ -1608,18 +1608,19 @@ class RootRestController(BaseController):
 
     @expose('json:')
     def perform_import(self, doc=None, options=None, **post_data):
-        require_access(c.project, 'admin')
-        if c.api_token.get_capability('import') != [c.project.neighborhood.name, c.project.shortname]:
-            log.error('Import capability is not enabled for %s', c.project.shortname)
-            raise exc.HTTPForbidden(detail='Import is not allowed')
+        with h.notifications_disabled(c.project):
+            require_access(c.project, 'admin')
+            if c.api_token.get_capability('import') != [c.project.neighborhood.name, c.project.shortname]:
+                log.error('Import capability is not enabled for %s', c.project.shortname)
+                raise exc.HTTPForbidden(detail='Import is not allowed')
 
-        migrator = ImportSupport()
-        try:
-            status = migrator.perform_import(doc, options, **post_data)
-            return status
-        except Exception, e:
-            log.exception(e)
-            return dict(status=False, errors=[str(e)])
+            migrator = ImportSupport()
+            try:
+                status = migrator.perform_import(doc, options, **post_data)
+                return status
+            except Exception, e:
+                log.exception(e)
+                return dict(status=False, errors=[str(e)])
 
     @expose('json:')
     def search(self, q=None, limit=100, page=0, sort=None, **kw):

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeWiki/forgewiki/wiki_main.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/wiki_main.py b/ForgeWiki/forgewiki/wiki_main.py
index 816db78..e059038 100644
--- a/ForgeWiki/forgewiki/wiki_main.py
+++ b/ForgeWiki/forgewiki/wiki_main.py
@@ -730,17 +730,18 @@ class PageRestController(BaseController):
         return self.page.__json__()
 
     def _update_page(self, title, **post_data):
-        if not self.page:
-            require_access(c.app, 'create')
-            self.page = WM.Page.upsert(title)
-            self.page.viewable_by = ['all']
-        else:
-            require_access(self.page, 'edit')
-        self.page.text = post_data['text']
-        if 'labels' in post_data:
-            self.page.labels = post_data['labels'].split(',')
-        self.page.commit()
-        return {}
+        with h.notifications_disabled(c.project):
+            if not self.page:
+                require_access(c.app, 'create')
+                self.page = WM.Page.upsert(title)
+                self.page.viewable_by = ['all']
+            else:
+                require_access(self.page, 'edit')
+            self.page.text = post_data['text']
+            if 'labels' in post_data:
+                self.page.labels = post_data['labels'].split(',')
+            self.page.commit()
+            return {}
 
 
 class WikiAdminController(DefaultAdminController):


Mime
View raw message