Return-Path: X-Original-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5879110351 for ; Fri, 27 Sep 2013 16:43:10 +0000 (UTC) Received: (qmail 11538 invoked by uid 500); 27 Sep 2013 16:43:08 -0000 Delivered-To: apmail-incubator-allura-commits-archive@incubator.apache.org Received: (qmail 11384 invoked by uid 500); 27 Sep 2013 16:43:07 -0000 Mailing-List: contact allura-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: allura-dev@incubator.apache.org Delivered-To: mailing list allura-commits@incubator.apache.org Received: (qmail 11258 invoked by uid 99); 27 Sep 2013 16:43:06 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 27 Sep 2013 16:43:06 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 4971790C2C0; Fri, 27 Sep 2013 16:43:06 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: brondsem@apache.org To: allura-commits@incubator.apache.org Date: Fri, 27 Sep 2013 16:43:06 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [01/36] git commit: [#6534] ticket:415 Wiki importer for github Updated Branches: refs/heads/db/6534 [created] bbaf6e2e6 [#6534] ticket:415 Wiki importer for github Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/9f088874 Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/9f088874 Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/9f088874 Branch: refs/heads/db/6534 Commit: 9f08887450bf02c938ab86002a100d53c86b11bd Parents: 7fc8966 Author: Yuriy Arhipov Authored: Tue Sep 3 08:43:12 2013 +0400 Committer: Dave Brondsema Committed: Fri Sep 27 16:28:41 2013 +0000 ---------------------------------------------------------------------- .../forgeimporters/github/__init__.py | 7 ++ .../forgeimporters/github/tests/test_wiki.py | 64 ++++++++++++++ ForgeImporters/forgeimporters/github/wiki.py | 87 ++++++++++++++++++++ .../tests/github/test_extractor.py | 10 ++- ForgeImporters/setup.py | 1 + 5 files changed, 167 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9f088874/ForgeImporters/forgeimporters/github/__init__.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py index 3d3caac..cf843c2 100644 --- a/ForgeImporters/forgeimporters/github/__init__.py +++ b/ForgeImporters/forgeimporters/github/__init__.py @@ -30,6 +30,7 @@ class GitHubProjectExtractor(base.ProjectExtractor): PAGE_MAP = { 'project_info': 'https://api.github.com/repos/{project_name}', 'issues': 'https://api.github.com/repos/{project_name}/issues', + 'wiki_url': 'https://github.com/{project_name}.wiki', } POSSIBLE_STATES = ('opened', 'closed') SUPPORTED_ISSUE_EVENTS = ('closed', 'reopened', 'assigned') @@ -91,3 +92,9 @@ class GitHubProjectExtractor(base.ProjectExtractor): for event in events: if event.get('event') in self.SUPPORTED_ISSUE_EVENTS: yield event + + def has_wiki(self): + return self.get_page('project_info').get('has_wiki') + + def get_wiki_url(self): + return self.get_page_url('wiki_url') http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9f088874/ForgeImporters/forgeimporters/github/tests/test_wiki.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/tests/test_wiki.py b/ForgeImporters/forgeimporters/github/tests/test_wiki.py new file mode 100644 index 0000000..fe716e9 --- /dev/null +++ b/ForgeImporters/forgeimporters/github/tests/test_wiki.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from unittest import TestCase +from nose.tools import assert_equal +from mock import Mock, patch + +from forgeimporters.github.wiki import GitHubWikiImporter +from alluratest.controller import setup_basic_test + + +class TestGitHubRepoImporter(TestCase): + + def _make_project(self, gh_proj_name=None): + project = Mock() + project.get_tool_data.side_effect = lambda *args: gh_proj_name + return project + + @patch('forgeimporters.github.wiki.g') + @patch('forgeimporters.github.wiki.GitHubProjectExtractor') + def test_import_tool_happy_path(self, ghpe, g): + with patch('forgeimporters.github.wiki.GitHubWikiImporter.get_wiki_pages'), patch('forgeimporters.github.wiki.c'): + ghpe.return_value.has_wiki.return_value = True + ghpe.return_value.get_wiki_url.return_value = "http://testwiki.com" + p = self._make_project(gh_proj_name='myproject') + GitHubWikiImporter().import_tool(p, Mock(name='c.user'), project_name='project_name', user_name='testuser') + p.install_app.assert_called_once_with( + 'Wiki', + mount_point='wiki', + mount_label='Wiki') + g.post_event.assert_called_once_with('project_updated') + + +class TestGitHubWikiImporter(TestCase): + + def setUp(self): + setup_basic_test() + + @patch('forgeimporters.github.wiki.GitHubWikiImporter.get_wiki_pages_form_git') + def test_get_wiki_pages(self, get_wiki_pages): + get_wiki_pages.return_value = { + "Home_creole.creole": "**TEST**", + "Home_md.md": "# TEST", + "Home.rest": "test" + } + data = GitHubWikiImporter().get_wiki_pages("http://test.git") + assert_equal(data['Home_creole'], '

TEST

\n') + assert_equal(data['Home_md'], '

TEST

') + assert_equal(data['Home'], '
\n

test

\n
\n') http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9f088874/ForgeImporters/forgeimporters/github/wiki.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/wiki.py b/ForgeImporters/forgeimporters/github/wiki.py new file mode 100644 index 0000000..562d6b5 --- /dev/null +++ b/ForgeImporters/forgeimporters/github/wiki.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import git +from tempfile import mkdtemp +from shutil import rmtree + +from pylons import app_globals as g +from pylons import tmpl_context as c + + +from allura.lib import helpers as h +from forgeimporters.base import ToolImporter +from forgeimporters.github import GitHubProjectExtractor +from forgewiki import model as WM + +import logging +log = logging.getLogger(__name__) + +TARGET_APPS = [] + +try: + from forgewiki.wiki_main import ForgeWikiApp + TARGET_APPS.append(ForgeWikiApp) +except ImportError: + pass + + +class GitHubWikiImporter(ToolImporter): + target_app = TARGET_APPS + source = 'GitHub' + tool_label = 'Wiki' + tool_description = 'Import your wiki from GitHub' + + def import_tool(self, project, user, project_name=None, mount_point=None, mount_label=None, user_name=None, **kw): + """ Import a GitHub wiki into a new Wiki Allura tool. + + """ + project_name = "%s/%s" % (user_name, project_name) + extractor = GitHubProjectExtractor(project_name) + if not extractor.has_wiki(): + return + + app = project.install_app( + "Wiki", + mount_point=mount_point or 'wiki', + mount_label=mount_label or 'Wiki') + + with h.push_config(c, app=app): + for page_name, page_text in self.get_wiki_pages(extractor.get_wiki_url()).iteritems(): + page = WM.Page.upsert(page_name) + page.text = page_text + page.viewable_by = ['all'] + + g.post_event('project_updated') + return app + + def get_wiki_pages(self, wiki_url): + result = dict() + wiki_pages = self.get_wiki_pages_form_git(wiki_url) + for page_name, page_text in wiki_pages.iteritems(): + page_text = h.render_any_markup(page_name, page_text) + result[page_name.split('.')[0]] = page_text + return result + + def get_wiki_pages_form_git(self, wiki_url): + result = dict() + wiki_path = mkdtemp() + wiki = git.Repo.clone_from(wiki_url, wiki_path) + for page in wiki.heads.master.commit.tree.blobs: + result[page.name] = page.data_stream.read() + rmtree(wiki_path) + return result http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9f088874/ForgeImporters/forgeimporters/tests/github/test_extractor.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py index fbbdc3d..0193002 100644 --- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py +++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py @@ -29,6 +29,7 @@ class TestGitHubProjectExtractor(TestCase): PROJECT_INFO = { 'description': 'project description', 'homepage': 'http://example.com', + 'has_wiki': True, } CLOSED_ISSUES_LIST = [ {u'number': 1}, @@ -83,7 +84,6 @@ class TestGitHubProjectExtractor(TestCase): def setUp(self): self.extractor = github.GitHubProjectExtractor('test_project') self.extractor.urlopen = self.mocked_urlopen - def test_get_next_page_url(self): self.assertIsNone(self.extractor.get_next_page_url(None)) self.assertIsNone(self.extractor.get_next_page_url('')) @@ -97,7 +97,7 @@ class TestGitHubProjectExtractor(TestCase): link = '; rel="prev"' self.assertIsNone(self.extractor.get_next_page_url(link)) - + def test_get_summary(self): self.assertEqual(self.extractor.get_summary(), 'project description') @@ -120,3 +120,9 @@ class TestGitHubProjectExtractor(TestCase): mock_issue = {'events_url': '/issues/1/events'} events = list(self.extractor.iter_events(mock_issue)) self.assertEqual(events, self.ISSUE_EVENTS + self.ISSUE_EVENTS_PAGE2[:1]) + + def test_has_wiki(self): + assert self.extractor.has_wiki() + + def test_get_wiki_url(self): + self.assertEqual(self.extractor.get_wiki_url(), 'https://github.com/test_project.wiki') http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9f088874/ForgeImporters/setup.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/setup.py b/ForgeImporters/setup.py index 12811f2..4d5f866 100644 --- a/ForgeImporters/setup.py +++ b/ForgeImporters/setup.py @@ -42,6 +42,7 @@ setup(name='ForgeImporters', github-tracker = forgeimporters.github.tracker:GitHubTrackerImporter google-code-tracker = forgeimporters.google.tracker:GoogleCodeTrackerImporter google-code-repo = forgeimporters.google.code:GoogleRepoImporter + github-wiki = forgeimporters.github.wiki:GitHubWikiImporter github-repo = forgeimporters.github.code:GitHubRepoImporter trac-tickets = forgeimporters.trac.tickets:TracTicketImporter forge-tracker = forgeimporters.forge.tracker:ForgeTrackerImporter