Return-Path: X-Original-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B47F0108BB for ; Fri, 10 Jan 2014 18:26:37 +0000 (UTC) Received: (qmail 1804 invoked by uid 500); 10 Jan 2014 18:20:41 -0000 Delivered-To: apmail-incubator-allura-commits-archive@incubator.apache.org Received: (qmail 1559 invoked by uid 500); 10 Jan 2014 18:20:12 -0000 Mailing-List: contact allura-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: allura-dev@incubator.apache.org Delivered-To: mailing list allura-commits@incubator.apache.org Received: (qmail 840 invoked by uid 99); 10 Jan 2014 18:19:26 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Jan 2014 18:19:26 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 6D92D32A2FC; Fri, 10 Jan 2014 18:19:25 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: tvansteenburgh@apache.org To: allura-commits@incubator.apache.org Date: Fri, 10 Jan 2014 18:19:35 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [11/32] PEP8 cleanup http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/forge/tracker.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/forge/tracker.py b/ForgeImporters/forgeimporters/forge/tracker.py index 0fe3fea..c06db73 100644 --- a/ForgeImporters/forgeimporters/forge/tracker.py +++ b/ForgeImporters/forgeimporters/forge/tracker.py @@ -24,15 +24,15 @@ from pylons import app_globals as g from ming.orm import session, ThreadLocalORMSession from tg import ( - expose, - flash, - redirect, - validate, - ) + expose, + flash, + redirect, + validate, +) from tg.decorators import ( - with_trailing_slash, - without_trailing_slash, - ) + with_trailing_slash, + without_trailing_slash, +) from allura.controllers import BaseController from allura.lib import helpers as h @@ -44,12 +44,12 @@ from allura import model as M from forgetracker.tracker_main import ForgeTrackerApp from forgetracker import model as TM from forgeimporters.base import ( - ToolImporter, - ToolImportForm, - File, - get_importer_upload_path, - save_importer_upload, - ) + ToolImporter, + ToolImportForm, + File, + get_importer_upload_path, + save_importer_upload, +) class ForgeTrackerImportForm(ToolImportForm): @@ -57,6 +57,7 @@ class ForgeTrackerImportForm(ToolImportForm): class ForgeTrackerImportController(BaseController): + def __init__(self): self.importer = ForgeTrackerImporter() @@ -68,7 +69,7 @@ class ForgeTrackerImportController(BaseController): @expose('jinja:forgeimporters.forge:templates/tracker/index.html') def index(self, **kw): return dict(importer=self.importer, - target_app=self.target_app) + target_app=self.target_app) @without_trailing_slash @expose() @@ -76,16 +77,18 @@ class ForgeTrackerImportController(BaseController): @validate(ForgeTrackerImportForm(ForgeTrackerApp), error_handler=index) def create(self, tickets_json, mount_point, mount_label, **kw): if self.importer.enforce_limit(c.project): - save_importer_upload(c.project, 'tickets.json', json.dumps(tickets_json)) + save_importer_upload( + c.project, 'tickets.json', json.dumps(tickets_json)) self.importer.post( - mount_point=mount_point, - mount_label=mount_label, - ) + mount_point=mount_point, + mount_label=mount_label, + ) flash('Ticket import has begun. Your new tracker will be available ' - 'when the import is complete.') + 'when the import is complete.') redirect(c.project.url() + 'admin/') else: - flash('There are too many imports pending at this time. Please wait and try again.', 'error') + flash( + 'There are too many imports pending at this time. Please wait and try again.', 'error') redirect(c.project.url() + 'admin/') @@ -106,7 +109,7 @@ class ForgeTrackerImporter(ToolImporter): return json.load(fp) def import_tool(self, project, user, mount_point=None, - mount_label=None, **kw): + mount_label=None, **kw): import_id_converter = ImportIdConverter.get() tracker_json = self._load_json(project) tracker_json['tracker_config']['options'].pop('ordinal', None) @@ -114,14 +117,16 @@ class ForgeTrackerImporter(ToolImporter): tracker_json['tracker_config']['options'].pop('mount_label', None) tracker_json['tracker_config']['options'].pop('import_id', None) app = project.install_app('tickets', mount_point, mount_label, - import_id={ - 'source': self.source, - 'app_config_id': tracker_json['tracker_config']['_id'], - }, - open_status_names=tracker_json['open_status_names'], - closed_status_names=tracker_json['closed_status_names'], - **tracker_json['tracker_config']['options'] - ) + import_id={ + 'source': self.source, + 'app_config_id': tracker_json['tracker_config']['_id'], + }, + open_status_names=tracker_json[ + 'open_status_names'], + closed_status_names=tracker_json[ + 'closed_status_names'], + **tracker_json['tracker_config']['options'] + ) ThreadLocalORMSession.flush_all() try: M.session.artifact_orm_session._get().skip_mod_date = True @@ -129,42 +134,51 @@ class ForgeTrackerImporter(ToolImporter): reporter = self.get_user(ticket_json['reported_by']) owner = self.get_user(ticket_json['assigned_to']) with h.push_config(c, user=reporter, app=app): - self.max_ticket_num = max(ticket_json['ticket_num'], self.max_ticket_num) + self.max_ticket_num = max( + ticket_json['ticket_num'], self.max_ticket_num) ticket = TM.Ticket( - app_config_id=app.config._id, - import_id=import_id_converter.expand(ticket_json['ticket_num'], app), - description=self.annotate( - self.annotate( - ticket_json['description'], - owner, ticket_json['assigned_to'], label=' owned'), - reporter, ticket_json['reported_by'], label=' created'), - created_date=dateutil.parser.parse(ticket_json['created_date']), - mod_date=dateutil.parser.parse(ticket_json['mod_date']), - ticket_num=ticket_json['ticket_num'], - summary=ticket_json['summary'], - custom_fields=ticket_json['custom_fields'], - status=ticket_json['status'], - labels=ticket_json['labels'], - votes_down=ticket_json['votes_down'], - votes_up=ticket_json['votes_up'], - votes=ticket_json['votes_up'] - ticket_json['votes_down'], - assigned_to_id=owner._id, - ) - ticket.private = ticket_json['private'] # trigger the private property - self.process_comments(ticket, ticket_json['discussion_thread']['posts']) + app_config_id=app.config._id, + import_id=import_id_converter.expand( + ticket_json['ticket_num'], app), + description=self.annotate( + self.annotate( + ticket_json['description'], + owner, ticket_json[ + 'assigned_to'], label=' owned'), + reporter, ticket_json[ + 'reported_by'], label=' created'), + created_date=dateutil.parser.parse( + ticket_json['created_date']), + mod_date=dateutil.parser.parse( + ticket_json['mod_date']), + ticket_num=ticket_json['ticket_num'], + summary=ticket_json['summary'], + custom_fields=ticket_json['custom_fields'], + status=ticket_json['status'], + labels=ticket_json['labels'], + votes_down=ticket_json['votes_down'], + votes_up=ticket_json['votes_up'], + votes=ticket_json['votes_up'] - + ticket_json['votes_down'], + assigned_to_id=owner._id, + ) + # trigger the private property + ticket.private = ticket_json['private'] + self.process_comments( + ticket, ticket_json['discussion_thread']['posts']) session(ticket).flush(ticket) session(ticket).expunge(ticket) app.globals.custom_fields = tracker_json['custom_fields'] self.process_bins(app, tracker_json['saved_bins']) app.globals.last_ticket_num = self.max_ticket_num M.AuditLog.log( - 'import tool %s from exported Allura JSON' % ( - app.config.options.mount_point, - ), - project=project, - user=user, - url=app.url, - ) + 'import tool %s from exported Allura JSON' % ( + app.config.options.mount_point, + ), + project=project, + user=user, + url=app.url, + ) g.post_event('project_updated') app.globals.invalidate_bin_counts() ThreadLocalORMSession.flush_all() @@ -193,11 +207,15 @@ class ForgeTrackerImporter(ToolImporter): user = self.get_user(comment_json['author']) with h.push_config(c, user=user): p = ticket.discussion_thread.add_post( - text = self.annotate(comment_json['text'], user, comment_json['author']), - ignore_security = True, - timestamp = dateutil.parser.parse(comment_json['timestamp']), - ) - p.add_multiple_attachments([File(a['url']) for a in comment_json['attachments']]) + text=self.annotate( + comment_json[ + 'text'], user, comment_json['author']), + ignore_security=True, + timestamp=dateutil.parser.parse( + comment_json['timestamp']), + ) + p.add_multiple_attachments([File(a['url']) + for a in comment_json['attachments']]) def process_bins(self, app, bins): TM.Bin.query.remove({'app_config_id': app.config._id}) http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/github/__init__.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py index 58afba3..2d4e85c 100644 --- a/ForgeImporters/forgeimporters/github/__init__.py +++ b/ForgeImporters/forgeimporters/github/__init__.py @@ -34,10 +34,10 @@ log = logging.getLogger(__name__) class GitHubProjectExtractor(base.ProjectExtractor): PAGE_MAP = { - 'project_info': 'https://api.github.com/repos/{project_name}', - 'issues': 'https://api.github.com/repos/{project_name}/issues', - 'wiki_url': 'https://github.com/{project_name}.wiki', - } + 'project_info': 'https://api.github.com/repos/{project_name}', + 'issues': 'https://api.github.com/repos/{project_name}/issues', + 'wiki_url': 'https://github.com/{project_name}.wiki', + } POSSIBLE_STATES = ('opened', 'closed') SUPPORTED_ISSUE_EVENTS = ('closed', 'reopened', 'assigned') NEXT_PAGE_URL_RE = re.compile(r'<([^>]*)>; rel="next"') @@ -66,7 +66,8 @@ class GitHubProjectExtractor(base.ProjectExtractor): def urlopen(self, url, **kw): try: - resp = super(GitHubProjectExtractor, self).urlopen(self.add_token(url), **kw) + resp = super(GitHubProjectExtractor, self).urlopen( + self.add_token(url), **kw) except urllib2.HTTPError as e: # GitHub will return 403 if rate limit exceeded. # We're checking for limit on every request below, but we still @@ -95,10 +96,12 @@ class GitHubProjectExtractor(base.ProjectExtractor): return json.loads(page.read().decode('utf8')), next_page_url def get_page(self, page_name_or_url, **kw): - page = super(GitHubProjectExtractor, self).get_page(page_name_or_url, **kw) + page = super(GitHubProjectExtractor, self).get_page( + page_name_or_url, **kw) page, next_page_url = page while next_page_url: - p = super(GitHubProjectExtractor, self).get_page(next_page_url, **kw) + p = super(GitHubProjectExtractor, + self).get_page(next_page_url, **kw) p, next_page_url = p page += p self.page = page @@ -114,7 +117,8 @@ class GitHubProjectExtractor(base.ProjectExtractor): return self.get_page('project_info').get('clone_url') def iter_issues(self): - # github api doesn't allow getting closed and opened tickets in one query + # github api doesn't allow getting closed and opened tickets in one + # query issues = [] url = self.get_page_url('issues') + '?state={state}' for state in self.POSSIBLE_STATES: @@ -144,6 +148,7 @@ class GitHubProjectExtractor(base.ProjectExtractor): class GitHubOAuthMixin(object): + '''Support for github oauth web application flow.''' def oauth_begin(self): @@ -155,8 +160,10 @@ class GitHubOAuthMixin(object): return # token already exists, nothing to do redirect_uri = request.url.rstrip('/') + '/oauth_callback' oauth = OAuth2Session(client_id, redirect_uri=redirect_uri) - auth_url, state = oauth.authorization_url('https://github.com/login/oauth/authorize') - session['github.oauth.state'] = state # Used in callback to prevent CSRF + auth_url, state = oauth.authorization_url( + 'https://github.com/login/oauth/authorize') + # Used in callback to prevent CSRF + session['github.oauth.state'] = state session['github.oauth.redirect'] = request.url session.save() redirect(auth_url) @@ -168,11 +175,13 @@ class GitHubOAuthMixin(object): secret = config.get('github_importer.client_secret') if not client_id or not secret: return # GitHub app is not configured - oauth = OAuth2Session(client_id, state=session.get('github.oauth.state')) + oauth = OAuth2Session( + client_id, state=session.get('github.oauth.state')) token = oauth.fetch_token( 'https://github.com/login/oauth/access_token', client_secret=secret, authorization_response=request.url ) - c.user.set_tool_data('GitHubProjectImport', token=token['access_token']) + c.user.set_tool_data('GitHubProjectImport', + token=token['access_token']) redirect(session.get('github.oauth.redirect', '/')) http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/github/code.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/code.py b/ForgeImporters/forgeimporters/github/code.py index 6fdd0b4..83095ff 100644 --- a/ForgeImporters/forgeimporters/github/code.py +++ b/ForgeImporters/forgeimporters/github/code.py @@ -19,15 +19,15 @@ from pylons import tmpl_context as c from pylons import app_globals as g from formencode import validators as fev from tg import ( - expose, - flash, - redirect, - validate, - ) + expose, + flash, + redirect, + validate, +) from tg.decorators import ( - with_trailing_slash, - without_trailing_slash, - ) + with_trailing_slash, + without_trailing_slash, +) from allura.lib.decorators import require_post from allura.lib import helpers as h @@ -37,9 +37,9 @@ from allura import model as M from forgegit.git_main import ForgeGitApp from forgeimporters.base import ( - ToolImporter, - ToolImportForm, - ) + ToolImporter, + ToolImportForm, +) from forgeimporters.github import GitHubProjectExtractor, GitHubOAuthMixin @@ -49,6 +49,7 @@ class GitHubRepoImportForm(ToolImportForm): class GitHubRepoImportController(BaseController, GitHubOAuthMixin): + def __init__(self): self.importer = GitHubRepoImporter() @@ -61,7 +62,7 @@ class GitHubRepoImportController(BaseController, GitHubOAuthMixin): def index(self, **kw): self.oauth_begin() return dict(importer=self.importer, - target_app=self.target_app) + target_app=self.target_app) @without_trailing_slash @expose() @@ -70,14 +71,15 @@ class GitHubRepoImportController(BaseController, GitHubOAuthMixin): def create(self, gh_project_name, gh_user_name, mount_point, mount_label, **kw): if self.importer.enforce_limit(c.project): self.importer.post( - project_name=gh_project_name, - user_name=gh_user_name, - mount_point=mount_point, - mount_label=mount_label) + project_name=gh_project_name, + user_name=gh_user_name, + mount_point=mount_point, + mount_label=mount_label) flash('Repo import has begun. Your new repo will be available ' - 'when the import is complete.') + 'when the import is complete.') else: - flash('There are too many imports pending at this time. Please wait and try again.', 'error') + flash( + 'There are too many imports pending at this time. Please wait and try again.', 'error') redirect(c.project.url() + 'admin/') @@ -89,7 +91,7 @@ class GitHubRepoImporter(ToolImporter): tool_description = 'Import your repo from GitHub' def import_tool(self, project, user, project_name=None, mount_point=None, - mount_label=None, user_name=None, **kw): + mount_label=None, user_name=None, **kw): """ Import a GitHub repo into a new Git Allura tool. """ @@ -107,9 +109,9 @@ class GitHubRepoImporter(ToolImporter): } ) M.AuditLog.log( - 'import tool %s from %s on %s' % ( - app.config.options.mount_point, - project_name, self.source, - ), project=project, user=user, url=app.url) + 'import tool %s from %s on %s' % ( + app.config.options.mount_point, + project_name, self.source, + ), project=project, user=user, url=app.url) g.post_event('project_updated') return app http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/github/project.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/project.py b/ForgeImporters/forgeimporters/github/project.py index 2462617..9c8e549 100644 --- a/ForgeImporters/forgeimporters/github/project.py +++ b/ForgeImporters/forgeimporters/github/project.py @@ -32,12 +32,14 @@ from . import GitHubOAuthMixin log = logging.getLogger(__name__) + class GitHubProjectForm(base.ProjectImportForm): project_name = fev.Regex(r'^[a-zA-Z0-9-_.]+$', - not_empty=True, - messages={ - 'invalid': 'Valid symbols are: letters, numbers, dashes, underscores and periods', - }) + not_empty=True, + messages={ + 'invalid': 'Valid symbols are: letters, numbers, dashes, underscores and periods', + }) + class GitHubProjectImporter(base.ProjectImporter, GitHubOAuthMixin): http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/github/tests/test_code.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/tests/test_code.py b/ForgeImporters/forgeimporters/github/tests/test_code.py index 7c85058..8a6f95b 100644 --- a/ForgeImporters/forgeimporters/github/tests/test_code.py +++ b/ForgeImporters/forgeimporters/github/tests/test_code.py @@ -26,7 +26,8 @@ from forgeimporters.github.code import GitHubRepoImporter from forgeimporters.github import GitHubOAuthMixin -# important to be distinct from 'test' which ForgeGit uses, so that the tests can run in parallel and not clobber each other +# important to be distinct from 'test' which ForgeGit uses, so that the +# tests can run in parallel and not clobber each other test_project_with_repo = 'test2' with_git = with_tool(test_project_with_repo, 'git', 'src', 'git') @@ -48,7 +49,8 @@ class TestGitHubRepoImporter(TestCase): app = p.install_app.return_value app.config.options.mount_point = 'code' app.url = 'foo' - GitHubRepoImporter().import_tool(p, u, project_name='project_name', user_name='testuser') + GitHubRepoImporter().import_tool( + p, u, project_name='project_name', user_name='testuser') p.install_app.assert_called_once_with( 'Git', mount_point='code', @@ -56,8 +58,8 @@ class TestGitHubRepoImporter(TestCase): init_from_url='http://remote/clone/url/', import_id={'source': 'GitHub', 'project_name': 'testuser/project_name'}) M.AuditLog.log.assert_called_once_with( - 'import tool code from testuser/project_name on GitHub', - project=p, user=u, url='foo') + 'import tool code from testuser/project_name on GitHub', + project=p, user=u, url='foo') g.post_event.assert_called_once_with('project_updated') @@ -65,7 +67,8 @@ class TestGitHubImportController(TestController, TestCase): @with_git def test_index(self): - r = self.app.get('/p/{}/admin/ext/import/github-repo/'.format(test_project_with_repo)) + r = self.app.get( + '/p/{}/admin/ext/import/github-repo/'.format(test_project_with_repo)) self.assertIsNotNone(r.html.find(attrs=dict(name="gh_user_name"))) self.assertIsNotNone(r.html.find(attrs=dict(name="gh_project_name"))) self.assertIsNotNone(r.html.find(attrs=dict(name="mount_label"))) @@ -75,18 +78,23 @@ class TestGitHubImportController(TestController, TestCase): @patch('forgeimporters.base.import_tool') def test_create(self, import_tool): params = dict( - gh_user_name='spooky', - gh_project_name='poop', - mount_label='mylabel', - mount_point='mymount', - ) - r = self.app.post('/p/{}/admin/ext/import/github-repo/create'.format(test_project_with_repo), - params, - status=302) - self.assertEqual(r.location, 'http://localhost/p/{}/admin/'.format(test_project_with_repo)) - self.assertEqual(u'mymount', import_tool.post.call_args[1]['mount_point']) - self.assertEqual(u'mylabel', import_tool.post.call_args[1]['mount_label']) - self.assertEqual(u'poop', import_tool.post.call_args[1]['project_name']) + gh_user_name='spooky', + gh_project_name='poop', + mount_label='mylabel', + mount_point='mymount', + ) + r = self.app.post( + '/p/{}/admin/ext/import/github-repo/create'.format(test_project_with_repo), + params, + status=302) + self.assertEqual( + r.location, 'http://localhost/p/{}/admin/'.format(test_project_with_repo)) + self.assertEqual( + u'mymount', import_tool.post.call_args[1]['mount_point']) + self.assertEqual( + u'mylabel', import_tool.post.call_args[1]['mount_label']) + self.assertEqual( + u'poop', import_tool.post.call_args[1]['project_name']) self.assertEqual(u'spooky', import_tool.post.call_args[1]['user_name']) @with_git @@ -96,19 +104,21 @@ class TestGitHubImportController(TestController, TestCase): project.set_tool_data('GitHubRepoImporter', pending=1) ThreadLocalORMSession.flush_all() params = dict( - gh_user_name='spooky', - gh_project_name='poop', - mount_label='mylabel', - mount_point='mymount', - ) - r = self.app.post('/p/{}/admin/ext/import/github-repo/create'.format(test_project_with_repo), - params, - status=302).follow() + gh_user_name='spooky', + gh_project_name='poop', + mount_label='mylabel', + mount_point='mymount', + ) + r = self.app.post( + '/p/{}/admin/ext/import/github-repo/create'.format(test_project_with_repo), + params, + status=302).follow() self.assertIn('Please wait and try again', r) self.assertEqual(import_tool.post.call_count, 0) @with_git @patch.object(GitHubOAuthMixin, 'oauth_begin') def test_oauth(self, oauth_begin): - r = self.app.get('/p/{}/admin/ext/import/github-repo/'.format(test_project_with_repo)) + r = self.app.get( + '/p/{}/admin/ext/import/github-repo/'.format(test_project_with_repo)) oauth_begin.assert_called_once() http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/github/tests/test_tracker.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/tests/test_tracker.py b/ForgeImporters/forgeimporters/github/tests/test_tracker.py index d63b4d6..c494453 100644 --- a/ForgeImporters/forgeimporters/github/tests/test_tracker.py +++ b/ForgeImporters/forgeimporters/github/tests/test_tracker.py @@ -25,9 +25,11 @@ from allura import model as M from forgeimporters.github import GitHubOAuthMixin -# important to be distinct from 'test' which ForgeTracker uses, so that the tests can run in parallel and not clobber each other +# important to be distinct from 'test' which ForgeTracker uses, so that +# the tests can run in parallel and not clobber each other test_project_with_tracker = 'test2' -with_tracker = with_tool(test_project_with_tracker, 'tickets', 'spooky-issues', 'tickets') +with_tracker = with_tool(test_project_with_tracker, + 'tickets', 'spooky-issues', 'tickets') class TestGitHubTrackerImportController(TestController, TestCase): @@ -51,10 +53,14 @@ class TestGitHubTrackerImportController(TestController, TestCase): mount_point='issues', mount_label='Issues') r = self.app.post(self.url + 'create', params, status=302) - self.assertEqual(r.location, 'http://localhost/p/%s/admin/' % test_project_with_tracker) - self.assertEqual(u'Issues', import_tool.post.call_args[1]['mount_label']) - self.assertEqual(u'issues', import_tool.post.call_args[1]['mount_point']) - self.assertEqual(u'mulder', import_tool.post.call_args[1]['project_name']) + self.assertEqual(r.location, 'http://localhost/p/%s/admin/' % + test_project_with_tracker) + self.assertEqual( + u'Issues', import_tool.post.call_args[1]['mount_label']) + self.assertEqual( + u'issues', import_tool.post.call_args[1]['mount_point']) + self.assertEqual( + u'mulder', import_tool.post.call_args[1]['project_name']) self.assertEqual(u'spooky', import_tool.post.call_args[1]['user_name']) @with_tracker http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/github/tests/test_wiki.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/tests/test_wiki.py b/ForgeImporters/forgeimporters/github/tests/test_wiki.py index 70ea74d..29e9757 100644 --- a/ForgeImporters/forgeimporters/github/tests/test_wiki.py +++ b/ForgeImporters/forgeimporters/github/tests/test_wiki.py @@ -32,7 +32,8 @@ from forgeimporters.github.utils import GitHubMarkdownConverter from forgeimporters.github import GitHubOAuthMixin -# important to be distinct from 'test' which ForgeWiki uses, so that the tests can run in parallel and not clobber each other +# important to be distinct from 'test' which ForgeWiki uses, so that the +# tests can run in parallel and not clobber each other test_project_with_wiki = 'test2' with_wiki = with_tool(test_project_with_wiki, 'wiki', 'w', 'wiki') @@ -44,7 +45,6 @@ class TestGitHubWikiImporter(TestCase): project.get_tool_data.side_effect = lambda *args: gh_proj_name return project - @patch('forgeimporters.github.wiki.M') @patch('forgeimporters.github.wiki.ThreadLocalORMSession') @patch('forgeimporters.github.wiki.g') @@ -57,7 +57,8 @@ class TestGitHubWikiImporter(TestCase): app = p.install_app.return_value app.config.options.mount_point = 'wiki' app.url = 'foo' - GitHubWikiImporter().import_tool(p, u, project_name='project_name', user_name='testuser') + GitHubWikiImporter().import_tool( + p, u, project_name='project_name', user_name='testuser') p.install_app.assert_called_once_with( 'Wiki', mount_point='wiki', @@ -96,7 +97,8 @@ class TestGitHubWikiImporter(TestCase): self.commit2 = Mock() blobs = [self.blob1, self.blob2, self.blob3] self.commit2.tree.blobs = blobs - self.commit2.tree.__contains__ = lambda _, item: item in [self.blob1.name, self.blob2.name, self.blob3.name] + self.commit2.tree.__contains__ = lambda _, item: item in [ + self.blob1.name, self.blob2.name, self.blob3.name] self.commit2.tree.traverse.return_value = blobs self.commit2.committed_date = 1256291446 @@ -144,7 +146,8 @@ class TestGitHubWikiImporter(TestCase): with patch('forgeimporters.github.wiki.rmtree'): path.return_value = 'temp_path' GitHubWikiImporter().import_pages('wiki_url') - repo.clone_from.assert_called_with('wiki_url', to_path='temp_path', bare=True) + repo.clone_from.assert_called_with( + 'wiki_url', to_path='temp_path', bare=True) @patch('forgeimporters.github.wiki.git.Repo._clone') @patch('forgeimporters.github.wiki.GitHubWikiImporter._with_history') @@ -178,7 +181,8 @@ class TestGitHubWikiImporter(TestCase): importer.rewrite_links = Mock(return_value='') importer._with_history(self.commit2) assert_equal(upsert.call_args_list, [call('Home')]) - assert_equal(render.call_args_list, [call('Home.rst', u'# test message')]) + assert_equal(render.call_args_list, + [call('Home.rst', u'# test message')]) @skipif(module_not_available('html2text')) @patch('forgeimporters.github.wiki.WM.Page.upsert') @@ -230,12 +234,15 @@ class TestGitHubWikiImporter(TestCase): assert_equal(f(u'[[Pagê Nâme]]'), u'[Pagê Nâme]') # Github always converts spaces and slashes in links to hyphens, # to lookup page in the filesystem. During import we're converting - # all hyphens in page name to spaces, but still supporting both link formats. + # all hyphens in page name to spaces, but still supporting both link + # formats. assert_equal(f(u'[[Page With Spaces]]'), u'[Page With Spaces]') assert_equal(f(u'[[Page-With-Spaces]]'), u'[Page With Spaces]') assert_equal(f(u'[[Page / 1]]'), u'[Page 1]') - assert_equal(f(u'[[Title|Page With Spaces]]'), u'[Title](Page With Spaces)') - assert_equal(f(u'[[Title|Page-With-Spaces]]'), u'[Title](Page With Spaces)') + assert_equal(f(u'[[Title|Page With Spaces]]'), + u'[Title](Page With Spaces)') + assert_equal(f(u'[[Title|Page-With-Spaces]]'), + u'[Title](Page With Spaces)') assert_equal(f(u'[[go here|Page / 1]]'), u'[go here](Page 1)') def test_convert_gollum_page_links_escaped(self): @@ -245,21 +252,25 @@ class TestGitHubWikiImporter(TestCase): assert_equal(f(u"'[[Page With Spaces]]"), u'[[Page With Spaces]]') assert_equal(f(u"'[[Page-With-Spaces]]"), u'[[Page-With-Spaces]]') assert_equal(f(u"'[[Page / 1]]"), u'[[Page / 1]]') - assert_equal(f(u"'[[Title|Page With Spaces]]"), u'[[Title|Page With Spaces]]') - assert_equal(f(u"'[[Title|Page-With-Spaces]]"), u'[[Title|Page-With-Spaces]]') + assert_equal(f(u"'[[Title|Page With Spaces]]"), + u'[[Title|Page With Spaces]]') + assert_equal(f(u"'[[Title|Page-With-Spaces]]"), + u'[[Title|Page-With-Spaces]]') assert_equal(f(u"'[[go here|Page / 1]]"), u'[[go here|Page / 1]]') def test_convert_gollum_external_links(self): f = GitHubWikiImporter().convert_gollum_tags assert_equal(f(u'[[http://sf.net]]'), u'') assert_equal(f(u'[[https://sf.net]]'), u'') - assert_equal(f(u'[[SourceForge|http://sf.net]]'), u'[SourceForge](http://sf.net)') + assert_equal(f(u'[[SourceForge|http://sf.net]]'), + u'[SourceForge](http://sf.net)') def test_convert_gollum_external_links_escaped(self): f = GitHubWikiImporter().convert_gollum_tags assert_equal(f(u"'[[http://sf.net]]"), u'[[http://sf.net]]') assert_equal(f(u"'[[https://sf.net]]"), u'[[https://sf.net]]') - assert_equal(f(u"'[[SourceForge|http://sf.net]]"), u'[[SourceForge|http://sf.net]]') + assert_equal(f(u"'[[SourceForge|http://sf.net]]"), + u'[[SourceForge|http://sf.net]]') def test_convert_gollum_toc(self): f = GitHubWikiImporter().convert_gollum_tags @@ -292,7 +303,8 @@ Our website is . importer.github_wiki_url = 'https://github.com/a/b/wiki' importer.app = Mock() importer.app.url = '/p/test/wiki/' - importer.github_markdown_converter = GitHubMarkdownConverter('user', 'proj') + importer.github_markdown_converter = GitHubMarkdownConverter( + 'user', 'proj') f = importer.convert_markup source = u'''Look at [[this page|Some Page]] @@ -363,16 +375,26 @@ Our website is [[http://sf.net]]. f = GitHubWikiImporter().rewrite_links prefix = 'https://github/a/b/wiki' new = '/p/test/wiki/' - assert_equal(f(u'Test Page', prefix, new), - u'Test Page') - assert_equal(f(u'Test-Page', prefix, new), - u'Test Page') - assert_equal(f(u'', prefix, new), - u'') - assert_equal(f(u'https://github/a/b/wiki/Test Page', prefix, new), - u'/p/test/wiki/Test Page') - assert_equal(f(u'Test Page', prefix, new), - u'Test Page') + assert_equal( + f(u'Test Page', + prefix, new), + u'Test Page') + assert_equal( + f(u'Test-Page', + prefix, new), + u'Test Page') + assert_equal( + f(u'', + prefix, new), + u'') + assert_equal( + f(u'https://github/a/b/wiki/Test Page', + prefix, new), + u'/p/test/wiki/Test Page') + assert_equal( + f(u'Test Page', + prefix, new), + u'Test Page') @skipif(module_not_available('html2text')) def test_convert_markup_with_mediawiki2markdown(self): @@ -489,7 +511,8 @@ some text and **[Tips n' Tricks]** importer.app.url = '/p/test/wiki/' f = importer.convert_markup source = u'*[[this checklist|Troubleshooting]]*' - assert_equal(f(source, 't.textile').strip(), u'**[this checklist](Troubleshooting)**') + assert_equal(f(source, 't.textile').strip(), + u'**[this checklist](Troubleshooting)**') @without_module('html2text') def test_convert_textile_special_tag_without_html2text(self): @@ -514,7 +537,8 @@ class TestGitHubWikiImportController(TestController, TestCase): self.assertIsNotNone(r.html.find(attrs=dict(name='gh_project_name'))) self.assertIsNotNone(r.html.find(attrs=dict(name='mount_label'))) self.assertIsNotNone(r.html.find(attrs=dict(name='mount_point'))) - self.assertIsNotNone(r.html.find(attrs=dict(name='tool_option', value='import_history'))) + self.assertIsNotNone( + r.html.find(attrs=dict(name='tool_option', value='import_history'))) @with_wiki @patch('forgeimporters.base.import_tool') @@ -526,7 +550,8 @@ class TestGitHubWikiImportController(TestController, TestCase): mount_label='GitHub Wiki', tool_option='import_history') r = self.app.post(self.url + 'create', params, status=302) - self.assertEqual(r.location, 'http://localhost/p/%s/admin/' % test_project_with_wiki) + self.assertEqual(r.location, 'http://localhost/p/%s/admin/' % + test_project_with_wiki) args = import_tool.post.call_args[1] self.assertEqual(u'GitHub Wiki', args['mount_label']) self.assertEqual(u'gh-wiki', args['mount_point']) @@ -537,7 +562,8 @@ class TestGitHubWikiImportController(TestController, TestCase): # without history params.pop('tool_option') r = self.app.post(self.url + 'create', params, status=302) - self.assertEqual(r.location, 'http://localhost/p/%s/admin/' % test_project_with_wiki) + self.assertEqual(r.location, 'http://localhost/p/%s/admin/' % + test_project_with_wiki) args = import_tool.post.call_args[1] self.assertEqual(u'GitHub Wiki', args['mount_label']) self.assertEqual(u'gh-wiki', args['mount_point']) http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/github/tracker.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/tracker.py b/ForgeImporters/forgeimporters/github/tracker.py index db16c83..3af1517 100644 --- a/ForgeImporters/forgeimporters/github/tracker.py +++ b/ForgeImporters/forgeimporters/github/tracker.py @@ -27,15 +27,15 @@ except ImportError: from formencode import validators as fev from tg import ( - expose, - validate, - flash, - redirect - ) + expose, + validate, + flash, + redirect +) from tg.decorators import ( - with_trailing_slash, - without_trailing_slash - ) + with_trailing_slash, + without_trailing_slash +) from allura import model as M from allura.controllers import BaseController @@ -85,14 +85,15 @@ class GitHubTrackerImportController(BaseController, GitHubOAuthMixin): def create(self, gh_project_name, gh_user_name, mount_point, mount_label, **kw): if self.importer.enforce_limit(c.project): self.importer.post( - project_name=gh_project_name, - user_name=gh_user_name, - mount_point=mount_point, - mount_label=mount_label) + project_name=gh_project_name, + user_name=gh_user_name, + mount_point=mount_point, + mount_label=mount_label) flash('Ticket import has begun. Your new tracker will be available ' - 'when the import is complete.') + 'when the import is complete.') else: - flash('There are too many imports pending at this time. Please wait and try again.', 'error') + flash( + 'There are too many imports pending at this time. Please wait and try again.', 'error') redirect(c.project.url() + 'admin/') @@ -105,18 +106,18 @@ class GitHubTrackerImporter(ToolImporter): open_milestones = set() def import_tool(self, project, user, project_name, mount_point=None, - mount_label=None, **kw): + mount_label=None, **kw): import_id_converter = ImportIdConverter.get() project_name = '%s/%s' % (kw['user_name'], project_name) app = project.install_app('tickets', mount_point, mount_label, - EnableVoting=False, - open_status_names='open', - closed_status_names='closed', - import_id={ - 'source': self.source, - 'project_name': project_name, - } - ) + EnableVoting=False, + open_status_names='open', + closed_status_names='closed', + import_id={ + 'source': self.source, + 'project_name': project_name, + } + ) self.github_markdown_converter = GitHubMarkdownConverter( kw['user_name'], project_name) ThreadLocalORMSession.flush_all() @@ -142,10 +143,10 @@ class GitHubTrackerImporter(ToolImporter): app.globals.last_ticket_num = self.max_ticket_num ThreadLocalORMSession.flush_all() M.AuditLog.log( - 'import tool %s from %s on %s' % ( - app.config.options.mount_point, - project_name, self.source), - project=project, user=user, url=app.url) + 'import tool %s from %s on %s' % ( + app.config.options.mount_point, + project_name, self.source), + project=project, user=user, url=app.url) g.post_event('project_updated') app.globals.invalidate_bin_counts() return app @@ -165,35 +166,36 @@ class GitHubTrackerImporter(ToolImporter): ticket.mod_date = self.parse_datetime(issue['updated_at']) if issue['assignee']: owner_line = '*Originally owned by:* {}\n'.format( - self.get_user_link(issue['assignee']['login'])) + self.get_user_link(issue['assignee']['login'])) else: owner_line = '' # body processing happens here body, attachments = self._get_attachments(extractor, issue['body']) ticket.add_multiple_attachments(attachments) ticket.description = ( - u'*Originally created by:* {creator}\n' - u'{owner}' - u'\n' - u'{body}').format( - creator=self.get_user_link(issue['user']['login']), - owner=owner_line, - body=self.github_markdown_converter.convert(body), - ) + u'*Originally created by:* {creator}\n' + u'{owner}' + u'\n' + u'{body}').format( + creator=self.get_user_link(issue['user']['login']), + owner=owner_line, + body=self.github_markdown_converter.convert(body), + ) ticket.labels = [label['name'] for label in issue['labels']] def process_comments(self, extractor, ticket, issue): for comment in extractor.iter_comments(issue): - body, attachments = self._get_attachments(extractor, comment['body']) + body, attachments = self._get_attachments( + extractor, comment['body']) if comment['user']: posted_by = u'*Originally posted by:* {}\n\n'.format( self.get_user_link(comment['user']['login'])) body = posted_by + body p = ticket.discussion_thread.add_post( - text = self.github_markdown_converter.convert(body), - ignore_security = True, - timestamp = self.parse_datetime(comment['created_at']), - ) + text=self.github_markdown_converter.convert(body), + ignore_security=True, + timestamp=self.parse_datetime(comment['created_at']), + ) p.add_multiple_attachments(attachments) def process_events(self, extractor, ticket, issue): @@ -201,22 +203,22 @@ class GitHubTrackerImporter(ToolImporter): prefix = text = '' if event['event'] in ('reopened', 'closed'): prefix = '*Ticket changed by:* {}\n\n'.format( - self.get_user_link(event['actor']['login'])) + self.get_user_link(event['actor']['login'])) if event['event'] == 'reopened': text = '- **status**: closed --> open' elif event['event'] == 'closed': text = '- **status**: open --> closed' elif event['event'] == 'assigned': text = '- **assigned_to**: {}'.format( - self.get_user_link(event['actor']['login'])) + self.get_user_link(event['actor']['login'])) text = prefix + text if not text: continue ticket.discussion_thread.add_post( - text = text, - ignore_security = True, - timestamp = self.parse_datetime(event['created_at']) + text=text, + ignore_security=True, + timestamp=self.parse_datetime(event['created_at']) ) def process_milestones(self, ticket, issue): @@ -265,11 +267,13 @@ class GitHubTrackerImporter(ToolImporter): attachments.append(Attachment( extractor, match.group(1), # url - 'attach{}.{}'.format(i + 1, match.group(2)) # extension + 'attach{}.{}'.format(i + 1, match.group(2)) # extension )) return (body, attachments) + class Attachment(object): + def __init__(self, extractor, url, filename): self.url = url self.filename = filename http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/github/utils.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/utils.py b/ForgeImporters/forgeimporters/github/utils.py index 0fb650b..3f00310 100644 --- a/ForgeImporters/forgeimporters/github/utils.py +++ b/ForgeImporters/forgeimporters/github/utils.py @@ -38,7 +38,7 @@ class GitHubMarkdownConverter(object): nextline = False for p in self.code_patterns: if line.startswith(p): - prev_line = lines[i-1].strip() if (i-1) >= 0 else '' + prev_line = lines[i - 1].strip() if (i - 1) >= 0 else '' if len(prev_line) > 0 and not in_block: new_lines.append('') if p == '```': @@ -62,10 +62,12 @@ class GitHubMarkdownConverter(object): if inline_matches: # need to not handle inline blocks as a text for i, m in enumerate(inline_matches): - line = line.replace(m, '%s' % i) + line = line.replace( + m, '%s' % i) line = self._handle_non_code(line) for i, m in enumerate(inline_matches): - line = line.replace('%s' % i, m) + line = line.replace( + '%s' % i, m) new_lines.append(line) else: new_lines.append(self._handle_non_code(line)) http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/github/wiki.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/wiki.py b/ForgeImporters/forgeimporters/github/wiki.py index 33801df..74789d2 100644 --- a/ForgeImporters/forgeimporters/github/wiki.py +++ b/ForgeImporters/forgeimporters/github/wiki.py @@ -28,28 +28,28 @@ from pylons import tmpl_context as c from ming.orm import ThreadLocalORMSession from formencode import validators as fev from tg import ( - expose, - validate, - flash, - redirect, - ) + expose, + validate, + flash, + redirect, +) from tg.decorators import ( - with_trailing_slash, - without_trailing_slash, - ) + with_trailing_slash, + without_trailing_slash, +) from allura.controllers import BaseController from allura.lib import helpers as h from allura.lib import utils from allura.lib.plugin import ImportIdConverter from allura.lib.decorators import ( - require_post, - ) + require_post, +) from allura import model as M from forgeimporters.base import ( - ToolImporter, - ToolImportForm, - ) + ToolImporter, + ToolImportForm, +) from forgeimporters.github import GitHubProjectExtractor, GitHubOAuthMixin from forgeimporters.github.utils import GitHubMarkdownConverter from forgewiki import model as WM @@ -105,7 +105,8 @@ class GitHubWikiImportController(BaseController, GitHubOAuthMixin): flash('Wiki import has begun. Your new wiki will be available ' 'when the import is complete.') else: - flash('There are too many imports pending at this time. Please wait and try again.', 'error') + flash( + 'There are too many imports pending at this time. Please wait and try again.', 'error') redirect(c.project.url() + 'admin/') @@ -120,22 +121,24 @@ class GitHubWikiImporter(ToolImporter): mediawiki_exts = ['.wiki', '.mediawiki'] markdown_exts = utils.MARKDOWN_EXTENSIONS textile_exts = ['.textile'] - # List of supported formats https://github.com/gollum/gollum/wiki#page-files + # List of supported formats + # https://github.com/gollum/gollum/wiki#page-files supported_formats = [ - '.asciidoc', - '.creole', - '.org', - '.pod', - '.rdoc', - '.rest.txt', - '.rst.txt', - '.rest', - '.rst', + '.asciidoc', + '.creole', + '.org', + '.pod', + '.rdoc', + '.rest.txt', + '.rst.txt', + '.rest', + '.rst', ] + mediawiki_exts + markdown_exts + textile_exts available_pages = [] - def import_tool(self, project, user, project_name=None, mount_point=None, mount_label=None, user_name=None, - tool_option=None, **kw): + def import_tool( + self, project, user, project_name=None, mount_point=None, mount_label=None, user_name=None, + tool_option=None, **kw): """ Import a GitHub wiki into a new Wiki Allura tool. """ @@ -144,7 +147,8 @@ class GitHubWikiImporter(ToolImporter): if not extractor.has_wiki(): return - self.github_wiki_url = extractor.get_page_url('wiki_url').replace('.wiki', '/wiki') + self.github_wiki_url = extractor.get_page_url( + 'wiki_url').replace('.wiki', '/wiki') self.app = project.install_app( "Wiki", mount_point=mount_point or 'wiki', @@ -161,7 +165,8 @@ class GitHubWikiImporter(ToolImporter): try: M.session.artifact_orm_session._get().skip_mod_date = True with h.push_config(c, app=self.app): - self.import_pages(extractor.get_page_url('wiki_url'), history=with_history) + self.import_pages( + extractor.get_page_url('wiki_url'), history=with_history) ThreadLocalORMSession.flush_all() M.AuditLog.log( 'import tool %s from %s on %s' % ( @@ -183,7 +188,7 @@ class GitHubWikiImporter(ToolImporter): pages = [blob.name for blob in commit.tree.traverse()] pages = map(os.path.splitext, pages) pages = [self._convert_page_name(name) for name, ext in pages - if ext in self.supported_formats] + if ext in self.supported_formats] self.available_pages = pages def _without_history(self, commit): @@ -196,7 +201,8 @@ class GitHubWikiImporter(ToolImporter): self._set_available_pages(commit) renamed_to = None if '=>' in filename: - # File renamed. Stats contains entry like 'Page.md => NewPage.md' + # File renamed. Stats contains entry like 'Page.md => + # NewPage.md' filename, renamed_to = filename.split(' => ') if renamed_to and renamed_to in commit.tree: text = commit.tree[renamed_to].data_stream.read() @@ -209,7 +215,8 @@ class GitHubWikiImporter(ToolImporter): def _make_page(self, text, filename, commit, renamed_to=None): orig_name = self._format_supported(filename) - renamed_orig_name = self._format_supported(renamed_to) if renamed_to else None + renamed_orig_name = self._format_supported( + renamed_to) if renamed_to else None if not orig_name: return if renamed_to and not renamed_orig_name: @@ -220,13 +227,16 @@ class GitHubWikiImporter(ToolImporter): wiki_page.viewable_by = ['all'] if renamed_orig_name and renamed_to in commit.tree: wiki_page.title = self._convert_page_name(renamed_orig_name) - wiki_page.text = self.convert_markup(h.really_unicode(text), renamed_to) + wiki_page.text = self.convert_markup( + h.really_unicode(text), renamed_to) elif filename in commit.tree: - wiki_page.text = self.convert_markup(h.really_unicode(text), filename) + wiki_page.text = self.convert_markup( + h.really_unicode(text), filename) else: wiki_page.delete() import_id_name = renamed_orig_name if renamed_orig_name else orig_name - wiki_page.import_id = ImportIdConverter.get().expand(import_id_name, self.app) + wiki_page.import_id = ImportIdConverter.get().expand( + import_id_name, self.app) wiki_page.commit() return wiki_page @@ -290,12 +300,14 @@ class GitHubWikiImporter(ToolImporter): if not new_prefix.endswith('/'): new_prefix += '/' _re = re.compile(r'%s(\S*)' % prefix) + def repl(m): return new_prefix + self._convert_page_name(m.group(1)) text = _re.sub(repl, text) else: text = h.render_any_markup(filename, text) - text = self.rewrite_links(text, self.github_wiki_url, self.app.url) + text = self.rewrite_links( + text, self.github_wiki_url, self.app.url) return text elif ext and ext in self.textile_exts: text = self._prepare_textile_text(text) @@ -306,8 +318,10 @@ class GitHubWikiImporter(ToolImporter): text = html2text.html2text(text) text = self.convert_gollum_tags(text) text = text.replace('', '').replace('', '') - text = text.replace('<notextile>', '').replace('</notextile>', '') - text = text.replace('<notextile>', '').replace('</notextile>', '') + text = text.replace('<notextile>', '').replace( + '</notextile>', '') + text = text.replace('<notextile>', '').replace( + '</notextile>', '') return text else: text = h.render_any_markup(filename, text) @@ -372,7 +386,8 @@ class GitHubWikiImporter(ToolImporter): # E.g. if you have two pages: a.md and A.md both [[a]] and [[A]] will refer a.md. # We're emulating this behavior using list of all available pages try: - idx = map(lambda p: p.lower(), self.available_pages).index(page.lower()) + idx = map(lambda p: p.lower(), + self.available_pages).index(page.lower()) except ValueError: idx = None if idx is not None: @@ -409,6 +424,6 @@ class GitHubWikiImporter(ToolImporter): # to convert gollum tags properly used tag, # so these tags will not be affected by converter - text = text.replace('[[', '[[').replace(']]', ']]') + text = text.replace( + '[[', '[[').replace(']]', ']]') return text - http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/google/__init__.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/google/__init__.py b/ForgeImporters/forgeimporters/google/__init__.py index 87990d0..e2ff4df 100644 --- a/ForgeImporters/forgeimporters/google/__init__.py +++ b/ForgeImporters/forgeimporters/google/__init__.py @@ -36,6 +36,7 @@ from forgeimporters.base import File log = logging.getLogger(__name__) + def _as_text(node, chunks=None): """ Similar to node.text, but preserves whitespace around tags, @@ -52,6 +53,7 @@ def _as_text(node, chunks=None): _as_text(n, chunks) return ''.join(chunks) + def _as_markdown(tag, project_name): fragments = [] for fragment in tag: @@ -60,7 +62,8 @@ def _as_markdown(tag, project_name): qs = parse_qs(href.query) gc_link = not href.netloc or href.netloc == 'code.google.com' path_parts = href.path.split('/') - target_project = path_parts[2] if gc_link and len(path_parts) >= 3 else '' + target_project = path_parts[ + 2] if gc_link and len(path_parts) >= 3 else '' internal_link = target_project == project_name if gc_link and internal_link and 'id' in qs: # rewrite issue 123 project-internal issue links @@ -69,29 +72,37 @@ def _as_markdown(tag, project_name): # rewrite r123 project-internal revision links fragment = '[r%s]' % qs['r'][0] elif gc_link: - # preserve GC-internal links (probably issue PROJECT:123 inter-project issue links) + # preserve GC-internal links (probably issue PROJECT:123 + # inter-project issue links) fragment = '[%s](%s)' % ( - h.plain2markdown(fragment.text, preserve_multiple_spaces=True, has_html_entities=True), - urljoin('https://code.google.com/p/%s/issues/' % project_name, fragment['href']), - ) + h.plain2markdown( + fragment.text, preserve_multiple_spaces=True, has_html_entities=True), + urljoin('https://code.google.com/p/%s/issues/' % + project_name, fragment['href']), + ) else: # convert all other links to Markdown syntax fragment = '[%s](%s)' % (fragment.text, fragment['href']) elif getattr(fragment, 'name', None) == 'i': - # preserve styling of "(No comment was entered for this change.)" messages - fragment = '*%s*' % h.plain2markdown(fragment.text, preserve_multiple_spaces=True, has_html_entities=True) + # preserve styling of "(No comment was entered for this change.)" + # messages + fragment = '*%s*' % h.plain2markdown(fragment.text, + preserve_multiple_spaces=True, has_html_entities=True) elif getattr(fragment, 'name', None) == 'b': # preserve styling of issue template - fragment = '**%s**' % h.plain2markdown(fragment.text, preserve_multiple_spaces=True, has_html_entities=True) + fragment = '**%s**' % h.plain2markdown(fragment.text, + preserve_multiple_spaces=True, has_html_entities=True) elif getattr(fragment, 'name', None) == 'br': # preserve forced line-breaks fragment = '\n' else: # convert all others to plain MD - fragment = h.plain2markdown(unicode(fragment), preserve_multiple_spaces=True, has_html_entities=True) + fragment = h.plain2markdown( + unicode(fragment), preserve_multiple_spaces=True, has_html_entities=True) fragments.append(fragment) return ''.join(fragments).strip() + def csv_parser(page): lines = page.readlines() if not lines: @@ -107,9 +118,9 @@ def csv_parser(page): class GoogleCodeProjectNameValidator(fev.FancyValidator): not_empty = True - messages={ - 'invalid': 'Please enter a project URL, or a project name containing only letters, numbers, and dashes.', - } + messages = { + 'invalid': 'Please enter a project URL, or a project name containing only letters, numbers, and dashes.', + } def _to_python(self, value, state=None): url = urlparse(value.strip()) @@ -127,30 +138,31 @@ class GoogleCodeProjectExtractor(ProjectExtractor): RE_REPO_TYPE = re.compile(r'(svn|hg|git)') PAGE_MAP = { - 'project_info': BASE_URL + '/p/{project_name}/', - 'source_browse': BASE_URL + '/p/{project_name}/source/browse/', - 'issues_csv': BASE_URL + '/p/{project_name}/issues/csv?can=1&colspec=ID&sort=ID&start={start}', - 'issue': BASE_URL + '/p/{project_name}/issues/detail?id={issue_id}', - } - - LICENSE_MAP = defaultdict(lambda:'Other/Proprietary License', { - 'Apache License 2.0': 'Apache License V2.0', - 'Artistic License/GPL': 'Artistic License', - 'Eclipse Public License 1.0': 'Eclipse Public License', - 'GNU GPL v2': 'GNU General Public License version 2.0 (GPLv2)', - 'GNU GPL v3': 'GNU General Public License version 3.0 (GPLv3)', - 'GNU Lesser GPL': 'GNU Library or Lesser General Public License version 2.0 (LGPLv2)', - 'MIT License': 'MIT License', - 'Mozilla Public License 1.1': 'Mozilla Public License 1.1 (MPL 1.1)', - 'New BSD License': 'BSD License', - 'Other Open Source': 'Open Software License', - }) + 'project_info': BASE_URL + '/p/{project_name}/', + 'source_browse': BASE_URL + '/p/{project_name}/source/browse/', + 'issues_csv': BASE_URL + '/p/{project_name}/issues/csv?can=1&colspec=ID&sort=ID&start={start}', + 'issue': BASE_URL + '/p/{project_name}/issues/detail?id={issue_id}', + } + + LICENSE_MAP = defaultdict(lambda: 'Other/Proprietary License', { + 'Apache License 2.0': 'Apache License V2.0', + 'Artistic License/GPL': 'Artistic License', + 'Eclipse Public License 1.0': 'Eclipse Public License', + 'GNU GPL v2': 'GNU General Public License version 2.0 (GPLv2)', + 'GNU GPL v3': 'GNU General Public License version 3.0 (GPLv3)', + 'GNU Lesser GPL': 'GNU Library or Lesser General Public License version 2.0 (LGPLv2)', + 'MIT License': 'MIT License', + 'Mozilla Public License 1.1': 'Mozilla Public License 1.1 (MPL 1.1)', + 'New BSD License': 'BSD License', + 'Other Open Source': 'Open Software License', + }) DEFAULT_ICON = 'http://www.gstatic.com/codesite/ph/images/defaultlogo.png' def get_short_description(self, project): page = self.get_page('project_info') - project.short_description = page.find(itemprop='description').text.strip() + project.short_description = page.find( + itemprop='description').text.strip() def get_icon(self, project): page = self.get_page('project_info') @@ -165,12 +177,13 @@ class GoogleCodeProjectExtractor(ProjectExtractor): filetype = 'image/png' M.ProjectFile.save_image( icon_name, icon.file, filetype, - square=True, thumbnail_size=(48,48), + square=True, thumbnail_size=(48, 48), thumbnail_meta={'project_id': project._id, 'category': 'icon'}) def get_license(self, project): page = self.get_page('project_info') - license = page.find(text='Code license').findNext().find('a').text.strip() + license = page.find(text='Code license').findNext().find( + 'a').text.strip() trove = M.TroveCategory.query.get(fullname=self.LICENSE_MAP[license]) project.trove_license.append(trove._id) @@ -179,7 +192,7 @@ class GoogleCodeProjectExtractor(ProjectExtractor): repo_type = page.find(id="crumb_root") if not repo_type: raise Exception("Couldn't detect repo type: no #crumb_root in " - "{0}".format(self.url)) + "{0}".format(self.url)) re_match = self.RE_REPO_TYPE.match(repo_type.text.lower()) if re_match: return re_match.group(0) @@ -200,13 +213,14 @@ class GoogleCodeProjectExtractor(ProjectExtractor): yield (int(issue_id), cls(project_name, 'issue', issue_id=issue_id)) except HTTPError as e: if e.code == 404: - log.warn('Unable to load GC issue: %s #%s: %s: %s', project_name, issue_id, e, e.url) + log.warn('Unable to load GC issue: %s #%s: %s: %s', + project_name, issue_id, e, e.url) continue else: raise # get any new issues that were created while importing # (jumping back a few in case some were deleted and new ones added) - new_ids = extractor.get_issue_ids(start=len(issue_ids)-10) + new_ids = extractor.get_issue_ids(start=len(issue_ids) - 10) issue_ids = new_ids - issue_ids def get_issue_ids(self, start=0): @@ -223,7 +237,8 @@ class GoogleCodeProjectExtractor(ProjectExtractor): return issue_ids def get_issue_summary(self): - text = self.page.find(id='issueheader').findAll('td', limit=2)[1].span.text.strip() + text = self.page.find(id='issueheader').findAll( + 'td', limit=2)[1].span.text.strip() bs = BeautifulSoup(text, convertEntities=BeautifulSoup.HTML_ENTITIES) return bs.text @@ -246,14 +261,16 @@ class GoogleCodeProjectExtractor(ProjectExtractor): return UserLink(a) def get_issue_status(self): - tag = self.page.find(id='issuemeta').find('th', text=re.compile('Status:')).findNext().span + tag = self.page.find(id='issuemeta').find( + 'th', text=re.compile('Status:')).findNext().span if tag: return tag.text.strip() else: return '' def get_issue_owner(self): - tag = self.page.find(id='issuemeta').find('th', text=re.compile('Owner:')).findNext().find(True, 'userlink') + tag = self.page.find(id='issuemeta').find( + 'th', text=re.compile('Owner:')).findNext().find(True, 'userlink') if tag: return UserLink(tag) else: @@ -277,20 +294,24 @@ class GoogleCodeProjectExtractor(ProjectExtractor): for comment in self.page.findAll('div', 'issuecomment'): yield Comment(comment, self.project_name) + class UserLink(object): + def __init__(self, tag): self.name = tag.text.strip() if tag.get('href'): - self.url = urljoin(GoogleCodeProjectExtractor.BASE_URL, tag.get('href')) + self.url = urljoin( + GoogleCodeProjectExtractor.BASE_URL, tag.get('href')) else: self.url = None def __str__(self): if self.url: - return '[{name}]({url})'.format(name = self.name, url = self.url) + return '[{name}]({url})'.format(name=self.name, url=self.url) else: return self.name + def _get_attachments(tag): attachment_links = tag.find('div', 'attachments') if attachment_links: @@ -307,9 +328,12 @@ def _get_attachments(tag): else: return [] + class Comment(object): + def __init__(self, tag, project_name): - self.author = UserLink(tag.find('span', 'author').find(True, 'userlink')) + self.author = UserLink( + tag.find('span', 'author').find(True, 'userlink')) self.created_date = tag.find('span', 'date').get('title') self.body = _as_markdown(tag.find('pre'), project_name) self._get_updates(tag) @@ -318,28 +342,30 @@ class Comment(object): def _get_updates(self, tag): _updates = tag.find('div', 'updates') self.updates = { - b.text: b.nextSibling.strip() - for b in _updates.findAll('b')} if _updates else {} + b.text: b.nextSibling.strip() + for b in _updates.findAll('b')} if _updates else {} @property def annotated_text(self): text = ( - u'*Originally posted by:* {author}\n' - u'\n' - u'{body}\n' - u'\n' - u'{updates}' - ).format( - author=self.author, - body=self.body, - updates='\n'.join( - '**%s** %s' % (k,v) - for k,v in self.updates.items() - ), - ) + u'*Originally posted by:* {author}\n' + u'\n' + u'{body}\n' + u'\n' + u'{updates}' + ).format( + author=self.author, + body=self.body, + updates='\n'.join( + '**%s** %s' % (k, v) + for k, v in self.updates.items() + ), + ) return text + class Attachment(File): + def __init__(self, url): url = urljoin(GoogleCodeProjectExtractor.BASE_URL, url) filename = parse_qs(urlparse(url).query)['name'][0] http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/google/code.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/google/code.py b/ForgeImporters/forgeimporters/google/code.py index bcb379e..8181c87 100644 --- a/ForgeImporters/forgeimporters/google/code.py +++ b/ForgeImporters/forgeimporters/google/code.py @@ -23,15 +23,15 @@ from formencode import validators as fev from pylons import tmpl_context as c from pylons import app_globals as g from tg import ( - expose, - flash, - redirect, - validate, - ) + expose, + flash, + redirect, + validate, +) from tg.decorators import ( - with_trailing_slash, - without_trailing_slash, - ) + with_trailing_slash, + without_trailing_slash, +) from allura.controllers import BaseController from allura.lib import validators as v @@ -39,8 +39,8 @@ from allura.lib.decorators import require_post from allura import model as M from forgeimporters.base import ( - ToolImporter, - ) + ToolImporter, +) from forgeimporters.google import GoogleCodeProjectExtractor from forgeimporters.google import GoogleCodeProjectNameValidator @@ -96,7 +96,8 @@ class GoogleRepoImportForm(fe.schema.Schema): gc_project_name = value['gc_project_name'] mount_point = value['mount_point'] try: - repo_type = GoogleCodeProjectExtractor(gc_project_name).get_repo_type() + repo_type = GoogleCodeProjectExtractor( + gc_project_name).get_repo_type() except urllib2.HTTPError as e: if e.code == 404: msg = 'No such project' @@ -108,13 +109,15 @@ class GoogleRepoImportForm(fe.schema.Schema): raise tool_class = REPO_APPS[repo_type] try: - value['mount_point'] = v.MountPointValidator(tool_class).to_python(mount_point) + value['mount_point'] = v.MountPointValidator( + tool_class).to_python(mount_point) except fe.Invalid as e: raise fe.Invalid('mount_point:' + str(e), value, state) return value class GoogleRepoImportController(BaseController): + def __init__(self): self.importer = GoogleRepoImporter() @@ -126,7 +129,7 @@ class GoogleRepoImportController(BaseController): @expose('jinja:forgeimporters.google:templates/code/index.html') def index(self, **kw): return dict(importer=self.importer, - target_app=self.target_app) + target_app=self.target_app) @without_trailing_slash @expose() @@ -135,13 +138,14 @@ class GoogleRepoImportController(BaseController): def create(self, gc_project_name, mount_point, mount_label, **kw): if self.importer.enforce_limit(c.project): self.importer.post( - project_name=gc_project_name, - mount_point=mount_point, - mount_label=mount_label) + project_name=gc_project_name, + mount_point=mount_point, + mount_label=mount_label) flash('Repo import has begun. Your new repo will be available ' - 'when the import is complete.') + 'when the import is complete.') else: - flash('There are too many imports pending at this time. Please wait and try again.', 'error') + flash( + 'There are too many imports pending at this time. Please wait and try again.', 'error') redirect(c.project.url() + 'admin/') @@ -153,7 +157,7 @@ class GoogleRepoImporter(ToolImporter): tool_description = 'Import your primary SVN, Git, or Hg repo from Google Code' def import_tool(self, project, user, project_name=None, mount_point=None, - mount_label=None, **kw): + mount_label=None, **kw): """ Import a Google Code repo into a new SVN, Git, or Hg Allura tool. """ @@ -161,19 +165,19 @@ class GoogleRepoImporter(ToolImporter): repo_type = extractor.get_repo_type() repo_url = get_repo_url(project_name, repo_type) app = project.install_app( - REPO_ENTRY_POINTS[repo_type], - mount_point=mount_point or 'code', - mount_label=mount_label or 'Code', - init_from_url=repo_url, - import_id={ - 'source': self.source, - 'project_name': project_name, - } - ) + REPO_ENTRY_POINTS[repo_type], + mount_point=mount_point or 'code', + mount_label=mount_label or 'Code', + init_from_url=repo_url, + import_id={ + 'source': self.source, + 'project_name': project_name, + } + ) M.AuditLog.log( - 'import tool %s from %s on %s' % ( - app.config.options.mount_point, - project_name, self.source, - ), project=project, user=user, url=app.url) + 'import tool %s from %s on %s' % ( + app.config.options.mount_point, + project_name, self.source, + ), project=project, user=user, url=app.url) g.post_event('project_updated') return app http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/google/project.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/google/project.py b/ForgeImporters/forgeimporters/google/project.py index 4bd6181..bb74c80 100644 --- a/ForgeImporters/forgeimporters/google/project.py +++ b/ForgeImporters/forgeimporters/google/project.py @@ -37,6 +37,7 @@ class GoogleCodeProjectForm(base.ProjectImportForm): class GoogleCodeProjectImporter(base.ProjectImporter): + """ Project importer for Google Code. http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/google/tests/__init__.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/google/tests/__init__.py b/ForgeImporters/forgeimporters/google/tests/__init__.py index 77505f1..144e298 100644 --- a/ForgeImporters/forgeimporters/google/tests/__init__.py +++ b/ForgeImporters/forgeimporters/google/tests/__init__.py @@ -14,4 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c93733ac/ForgeImporters/forgeimporters/google/tests/test_code.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/google/tests/test_code.py b/ForgeImporters/forgeimporters/google/tests/test_code.py index 7c79e36..e8f6e35 100644 --- a/ForgeImporters/forgeimporters/google/tests/test_code.py +++ b/ForgeImporters/forgeimporters/google/tests/test_code.py @@ -24,15 +24,16 @@ from allura.tests.decorators import with_tool from allura import model as M -# important to be distinct from 'test' which ForgeSVN uses, so that the tests can run in parallel and not clobber each other +# important to be distinct from 'test' which ForgeSVN uses, so that the +# tests can run in parallel and not clobber each other test_project_with_repo = 'test2' from forgeimporters.google.code import ( - get_repo_url, - GoogleRepoImporter, - GoogleRepoImportController, - ) + get_repo_url, + GoogleRepoImporter, + GoogleRepoImportController, +) class TestGetRepoUrl(TestCase): @@ -72,24 +73,25 @@ class TestGoogleRepoImporter(TestCase): GoogleRepoImporter().import_tool(p, u, project_name='project_name') get_repo_url.assert_called_once_with('project_name', 'git') p.install_app.assert_called_once_with('Git', - mount_point='code', - mount_label='Code', - init_from_url='http://remote/clone/url/', - import_id={ - 'source': 'Google Code', - 'project_name': 'project_name', - }, - ) + mount_point='code', + mount_label='Code', + init_from_url='http://remote/clone/url/', + import_id={ + 'source': 'Google Code', + 'project_name': 'project_name', + }, + ) M.AuditLog.log.assert_called_once_with( - 'import tool code from project_name on Google Code', - project=p, user=u, url='foo') + 'import tool code from project_name on Google Code', + project=p, user=u, url='foo') g.post_event.assert_called_once_with('project_updated') class TestGoogleRepoImportController(TestController, TestCase): def test_index(self): - r = self.app.get('/p/{}/admin/ext/import/google-code-repo/'.format(test_project_with_repo)) + r = self.app.get( + '/p/{}/admin/ext/import/google-code-repo/'.format(test_project_with_repo)) self.assertIsNotNone(r.html.find(attrs=dict(name="gc_project_name"))) self.assertIsNotNone(r.html.find(attrs=dict(name="mount_label"))) self.assertIsNotNone(r.html.find(attrs=dict(name="mount_point"))) @@ -99,16 +101,21 @@ class TestGoogleRepoImportController(TestController, TestCase): def test_create(self, import_tool, extractor): extractor.return_value.get_repo_type.return_value = 'git' params = dict(gc_project_name='poop', - mount_label='mylabel', - mount_point='mymount', - ) - r = self.app.post('/p/{}/admin/ext/import/google-code-repo/create'.format(test_project_with_repo), - params, - status=302) - self.assertEqual(r.location, 'http://localhost/p/{}/admin/'.format(test_project_with_repo)) - self.assertEqual(u'mymount', import_tool.post.call_args[1]['mount_point']) - self.assertEqual(u'mylabel', import_tool.post.call_args[1]['mount_label']) - self.assertEqual(u'poop', import_tool.post.call_args[1]['project_name']) + mount_label='mylabel', + mount_point='mymount', + ) + r = self.app.post( + '/p/{}/admin/ext/import/google-code-repo/create'.format(test_project_with_repo), + params, + status=302) + self.assertEqual( + r.location, 'http://localhost/p/{}/admin/'.format(test_project_with_repo)) + self.assertEqual( + u'mymount', import_tool.post.call_args[1]['mount_point']) + self.assertEqual( + u'mylabel', import_tool.post.call_args[1]['mount_label']) + self.assertEqual( + u'poop', import_tool.post.call_args[1]['project_name']) @patch('forgeimporters.google.code.GoogleCodeProjectExtractor') @patch('forgeimporters.base.import_tool') @@ -118,11 +125,12 @@ class TestGoogleRepoImportController(TestController, TestCase): project.set_tool_data('GoogleRepoImporter', pending=1) ThreadLocalORMSession.flush_all() params = dict(gc_project_name='poop', - mount_label='mylabel', - mount_point='mymount', - ) - r = self.app.post('/p/{}/admin/ext/import/google-code-repo/create'.format(test_project_with_repo), - params, - status=302).follow() + mount_label='mylabel', + mount_point='mymount', + ) + r = self.app.post( + '/p/{}/admin/ext/import/google-code-repo/create'.format(test_project_with_repo), + params, + status=302).follow() self.assertIn('Please wait and try again', r) self.assertEqual(import_tool.post.call_count, 0)