Return-Path: X-Original-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4DF1710155 for ; Mon, 3 Feb 2014 22:43:54 +0000 (UTC) Received: (qmail 73293 invoked by uid 500); 3 Feb 2014 22:43:53 -0000 Delivered-To: apmail-incubator-allura-commits-archive@incubator.apache.org Received: (qmail 73274 invoked by uid 500); 3 Feb 2014 22:43:53 -0000 Mailing-List: contact allura-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: allura-dev@incubator.apache.org Delivered-To: mailing list allura-commits@incubator.apache.org Received: (qmail 73266 invoked by uid 99); 3 Feb 2014 22:43:53 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 03 Feb 2014 22:43:53 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 22F89918BEA; Mon, 3 Feb 2014 22:43:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: johnsca@apache.org To: allura-commits@incubator.apache.org Message-Id: <3da7c372b24546ef8cbf38c62aca3b16@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: [#7124] Added URL availability validation for Trac Date: Mon, 3 Feb 2014 22:43:53 +0000 (UTC) Updated Branches: refs/heads/cj/7124 [created] 83606c46d [#7124] Added URL availability validation for Trac Signed-off-by: Cory Johns Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/83606c46 Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/83606c46 Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/83606c46 Branch: refs/heads/cj/7124 Commit: 83606c46d0ed65659ff81d54678b47082fe98980 Parents: bfee3bc Author: Cory Johns Authored: Mon Feb 3 22:42:13 2014 +0000 Committer: Cory Johns Committed: Mon Feb 3 22:43:35 2014 +0000 ---------------------------------------------------------------------- ForgeImporters/forgeimporters/trac/__init__.py | 28 ++++++++++++ ForgeImporters/forgeimporters/trac/project.py | 7 ++- .../forgeimporters/trac/tests/test_tickets.py | 46 +++++++++++++++----- ForgeImporters/forgeimporters/trac/tickets.py | 11 ++--- requirements-sf.txt | 2 +- 5 files changed, 73 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/83606c46/ForgeImporters/forgeimporters/trac/__init__.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/trac/__init__.py b/ForgeImporters/forgeimporters/trac/__init__.py index 144e298..c861469 100644 --- a/ForgeImporters/forgeimporters/trac/__init__.py +++ b/ForgeImporters/forgeimporters/trac/__init__.py @@ -14,3 +14,31 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +from formencode import validators as fev +import requests + + +class TracURLValidator(fev.URL): + not_empty = True + messages = { + 'unavailable': 'This project is unavailable for import' + } + + def _to_python(self, value, state=None): + value = super(TracURLValidator, self)._to_python(value, state) + # remove extraneous /wiki/[PageName] from the end of the URL + url_parts = value.split('/') + try: + wiki_in_url = url_parts.index('wiki') + except ValueError: + wiki_in_url = -1 + if wiki_in_url >= len(url_parts) - 2: + value = '/'.join(url_parts[:wiki_in_url]) + # normalize trailing slash + value = value.rstrip('/') + '/' + + resp = requests.head(value, allow_redirects=True) + if resp.status_code != 200: + raise fev.Invalid(self.message('unavailable', state), value, state) + return value http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/83606c46/ForgeImporters/forgeimporters/trac/project.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/trac/project.py b/ForgeImporters/forgeimporters/trac/project.py index 1dba980..0f44259 100644 --- a/ForgeImporters/forgeimporters/trac/project.py +++ b/ForgeImporters/forgeimporters/trac/project.py @@ -17,22 +17,21 @@ import logging -from formencode import validators as fev - from tg import expose, validate from tg.decorators import with_trailing_slash from allura.lib.decorators import require_post from allura.lib.validators import UserMapJsonFile -from .. import base +from forgeimporters import base +from forgeimporters.trac import TracURLValidator log = logging.getLogger(__name__) class TracProjectForm(base.ProjectImportForm): - trac_url = fev.URL(not_empty=True) + trac_url = TracURLValidator() user_map = UserMapJsonFile(as_string=True) http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/83606c46/ForgeImporters/forgeimporters/trac/tests/test_tickets.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py index e9366f2..ce2eccd 100644 --- a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py +++ b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py @@ -26,7 +26,7 @@ from IPython.testing.decorators import module_not_available, skipif from allura.tests import TestController from allura.tests.decorators import with_tracker -from alluratest.controller import TestRestApiBase +from alluratest.controller import TestRestApiBase, setup_unit_test from allura import model as M from forgetracker import model as TM @@ -40,6 +40,9 @@ from forgeimporters.trac.tickets import ( class TestTracTicketImporter(TestCase): + def setUp(self): + setup_unit_test() + @patch('forgeimporters.trac.tickets.session') @patch('forgeimporters.trac.tickets.g') @patch('forgeimporters.trac.tickets.AuditLog') @@ -69,9 +72,9 @@ class TestTracTicketImporter(TestCase): closed_status_names='closed', import_id={ 'source': 'Trac', - 'trac_url': 'http://example.com/trac/url/', + 'trac_url': 'http://example.com/trac/url', }) - export.assert_called_once_with('http://example.com/trac/url/') + export.assert_called_once_with('http://example.com/trac/url') ImportSupport.return_value.perform_import.assert_called_once_with( json.dumps(export.return_value), json.dumps({ @@ -80,7 +83,7 @@ class TestTracTicketImporter(TestCase): }), ) AuditLog.log.assert_called_once_with( - 'import tool bugs from http://example.com/trac/url/', + 'import tool bugs from http://example.com/trac/url', project=project, user=user, url='foo') g.post_event.assert_called_once_with('project_updated') @@ -122,15 +125,18 @@ class TestTracTicketImportController(TestController, TestCase): self.assertIsNotNone(r.html.find(attrs=dict(name="mount_point"))) @with_tracker + @patch('forgeimporters.trac.requests.head') @patch('forgeimporters.base.import_tool') - def test_create(self, import_tool): + def test_create(self, import_tool, head): + head.return_value.status_code = 200 params = dict(trac_url='http://example.com/trac/url', mount_label='mylabel', mount_point='mymount', ) r = self.app.post('/p/test/admin/bugs/_importer/create', params, - upload_files=[ - ('user_map', 'myfile', '{"orig_user": "new_user"}')], + upload_files=[( + 'user_map', 'myfile', '{"orig_user": "new_user"}' + )], status=302) self.assertEqual(r.location, 'http://localhost/p/test/admin/') self.assertEqual( @@ -139,12 +145,14 @@ class TestTracTicketImportController(TestController, TestCase): u'mylabel', import_tool.post.call_args[1]['mount_label']) self.assertEqual('{"orig_user": "new_user"}', import_tool.post.call_args[1]['user_map']) - self.assertEqual(u'http://example.com/trac/url', + self.assertEqual(u'http://example.com/trac/url/', import_tool.post.call_args[1]['trac_url']) @with_tracker + @patch('forgeimporters.trac.requests.head') @patch('forgeimporters.base.import_tool') - def test_create_limit(self, import_tool): + def test_create_limit(self, import_tool, head): + head.return_value.status_code = 200 project = M.Project.query.get(shortname='test') project.set_tool_data('TracTicketImporter', pending=1) ThreadLocalORMSession.flush_all() @@ -153,12 +161,28 @@ class TestTracTicketImportController(TestController, TestCase): mount_point='mymount', ) r = self.app.post('/p/test/admin/bugs/_importer/create', params, - upload_files=[ - ('user_map', 'myfile', '{"orig_user": "new_user"}')], + upload_files=[( + 'user_map', 'myfile', '{"orig_user": "new_user"}' + )], status=302).follow() self.assertIn('Please wait and try again', r) self.assertEqual(import_tool.post.call_count, 0) + @with_tracker + @patch('forgeimporters.trac.requests.head') + @patch('forgeimporters.base.import_tool') + def test_create_not_found(self, import_tool, head): + head.return_value.status_code = 404 + params = dict(trac_url='http://example.com/trac/url', + mount_label='mylabel', + mount_point='mymount', + ) + r = self.app.post('/p/test/admin/bugs/_importer/create', params, + upload_files=[( + 'user_map', 'myfile', '{"orig_user": "new_user"}' + )]) + self.assertEqual(import_tool.post.call_count, 0) + class TestTracImportSupport(TestCase): http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/83606c46/ForgeImporters/forgeimporters/trac/tickets.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py index c2c28c1..a8372ed 100644 --- a/ForgeImporters/forgeimporters/trac/tickets.py +++ b/ForgeImporters/forgeimporters/trac/tickets.py @@ -48,13 +48,14 @@ from forgeimporters.base import ( ToolImporter, ToolImportForm, ) +from forgeimporters.trac import TracURLValidator from forgetracker.tracker_main import ForgeTrackerApp from forgetracker.import_support import ImportSupport from forgetracker import model as TM class TracTicketImportForm(ToolImportForm): - trac_url = fev.URL(not_empty=True) + trac_url = TracURLValidator() user_map = v.UserMapJsonFile(as_string=True) @@ -85,11 +86,12 @@ class TracTicketImportController(BaseController): mount_label=mount_label, trac_url=trac_url, user_map=user_map) - flash('Ticket import has begun. Your new tracker will be available ' - 'when the import is complete.') + flash('Ticket import has begun. Your new tracker will be ' + 'available when the import is complete.') else: flash( - 'There are too many imports pending at this time. Please wait and try again.', 'error') + 'There are too many imports pending at this time. Please ' + 'wait and try again.', 'error') redirect(c.project.url() + 'admin/') @@ -105,7 +107,6 @@ class TracTicketImporter(ToolImporter): """ Import Trac tickets into a new Allura Tracker tool. """ - trac_url = trac_url.rstrip('/') + '/' mount_point = mount_point or 'tickets' app = project.install_app( 'Tickets', http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/83606c46/requirements-sf.txt ---------------------------------------------------------------------- diff --git a/requirements-sf.txt b/requirements-sf.txt index b54c0c0..61c12ac 100644 --- a/requirements-sf.txt +++ b/requirements-sf.txt @@ -20,7 +20,7 @@ wsgipreload==1.2 pyzmq==2.1.7 html2text==3.200.3dev-20121112 PyMollom==0.1 -TracWikiImporter==0.3.6 +TracWikiImporter==0.3.7 MediawikiImporter==0.0.2 Unidecode==0.04.14