Return-Path: X-Original-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5877710609 for ; Fri, 4 Oct 2013 16:30:58 +0000 (UTC) Received: (qmail 29496 invoked by uid 500); 4 Oct 2013 16:30:44 -0000 Delivered-To: apmail-incubator-allura-commits-archive@incubator.apache.org Received: (qmail 29169 invoked by uid 500); 4 Oct 2013 16:30:42 -0000 Mailing-List: contact allura-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: allura-dev@incubator.apache.org Delivered-To: mailing list allura-commits@incubator.apache.org Received: (qmail 28550 invoked by uid 99); 4 Oct 2013 16:30:31 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Oct 2013 16:30:31 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 857FB90FAC3; Fri, 4 Oct 2013 16:30:30 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: johnsca@apache.org To: allura-commits@incubator.apache.org Date: Fri, 04 Oct 2013 16:30:55 -0000 Message-Id: In-Reply-To: <22c701cf1faa461199ee44abb47bd1f2@git.apache.org> References: <22c701cf1faa461199ee44abb47bd1f2@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [27/50] git commit: [#6534] ticket:441 Refactor code and apply text conversion [#6534] ticket:441 Refactor code and apply text conversion Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/2b1f3c06 Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/2b1f3c06 Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/2b1f3c06 Branch: refs/heads/cj/5561 Commit: 2b1f3c061bf82fdbb4f3e60fa97be9d3c3321008 Parents: af36eaf Author: Igor Bondarenko Authored: Fri Sep 20 12:01:35 2013 +0300 Committer: Dave Brondsema Committed: Fri Oct 4 14:21:22 2013 +0000 ---------------------------------------------------------------------- .../forgeimporters/github/tests/test_wiki.py | 29 ++++---- ForgeImporters/forgeimporters/github/wiki.py | 73 +++++++++++--------- 2 files changed, 55 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2b1f3c06/ForgeImporters/forgeimporters/github/tests/test_wiki.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/tests/test_wiki.py b/ForgeImporters/forgeimporters/github/tests/test_wiki.py index 352ce24..01dec62 100644 --- a/ForgeImporters/forgeimporters/github/tests/test_wiki.py +++ b/ForgeImporters/forgeimporters/github/tests/test_wiki.py @@ -40,10 +40,12 @@ class TestGitHubRepoImporter(TestCase): project.get_tool_data.side_effect = lambda *args: gh_proj_name return project + + @patch('forgeimporters.github.wiki.ThreadLocalORMSession') @patch('forgeimporters.github.wiki.g') @patch('forgeimporters.github.wiki.GitHubProjectExtractor') - def test_import_tool_happy_path(self, ghpe, g): - with patch('forgeimporters.github.wiki.GitHubWikiImporter.get_wiki_pages'), patch('forgeimporters.github.wiki.c'): + def test_import_tool_happy_path(self, ghpe, g, tlorms): + with patch('forgeimporters.github.wiki.GitHubWikiImporter.import_pages'), patch('forgeimporters.github.wiki.c'): ghpe.return_value.has_wiki.return_value = True p = self._make_project(gh_proj_name='myproject') GitHubWikiImporter().import_tool(p, Mock(name='c.user'), project_name='project_name', user_name='testuser') @@ -76,13 +78,14 @@ class TestGitHubWikiImporter(TestCase): self.commit2 = Mock() self.commit2.tree.blobs = [self.blob1, self.blob2, self.blob3] + self.commit2.tree.__contains__ = lambda _, item: item in [self.blob1.name, self.blob2.name, self.blob3.name] self.commit2.committed_date = 1256291446 @patch('forgeimporters.github.wiki.WM.Page.upsert') @patch('forgeimporters.github.wiki.h.render_any_markup') - def test_get_blobs_without_history(self, render, upsert): + def test_without_history(self, render, upsert): upsert.text = Mock() - GitHubWikiImporter().get_blobs_without_history(self.commit2) + GitHubWikiImporter()._without_history(self.commit2) assert_equal(upsert.call_args_list, [call('Home'), call('Home2'), call('Home3')]) assert_equal(render.call_args_list, [ @@ -95,24 +98,24 @@ class TestGitHubWikiImporter(TestCase): def test_clone_from(self, path, repo): with patch('forgeimporters.github.wiki.rmtree'): path.return_value = 'temp_path' - GitHubWikiImporter().get_wiki_pages('wiki_url') + GitHubWikiImporter().import_pages('wiki_url') repo.clone_from.assert_called_with('wiki_url', to_path='temp_path', bare=True) @patch('forgeimporters.github.wiki.git.Repo._clone') - @patch('forgeimporters.github.wiki.GitHubWikiImporter.get_blobs_with_history') - @patch('forgeimporters.github.wiki.GitHubWikiImporter.get_blobs_without_history') - def test_get_commits_with_history(self, without_history, with_history, clone): + @patch('forgeimporters.github.wiki.GitHubWikiImporter._with_history') + @patch('forgeimporters.github.wiki.GitHubWikiImporter._without_history') + def test_with_history(self, without_history, with_history, clone): repo = clone.return_value repo.iter_commits.return_value = [self.commit1, self.commit2] - GitHubWikiImporter().get_wiki_pages('wiki_url', history=True) + GitHubWikiImporter().import_pages('wiki_url', history=True) assert_equal(with_history.call_count, 2) assert_equal(without_history.call_count, 0) - @patch('forgeimporters.github.wiki.GitHubWikiImporter.get_blobs_with_history') - @patch('forgeimporters.github.wiki.GitHubWikiImporter.get_blobs_without_history') + @patch('forgeimporters.github.wiki.GitHubWikiImporter._with_history') + @patch('forgeimporters.github.wiki.GitHubWikiImporter._without_history') def test_get_commits_without_history(self, without_history, with_history): with patch('forgeimporters.github.wiki.git.Repo._clone'): - GitHubWikiImporter().get_wiki_pages('wiki_url') + GitHubWikiImporter().import_pages('wiki_url') assert_equal(with_history.call_count, 0) assert_equal(without_history.call_count, 1) @@ -121,7 +124,7 @@ class TestGitHubWikiImporter(TestCase): def test_get_blobs_with_history(self, render, upsert): self.commit2.stats.files = {"Home.md": self.blob1} self.commit2.tree = {"Home.md": self.blob1} - GitHubWikiImporter().get_blobs_with_history(self.commit2) + GitHubWikiImporter()._with_history(self.commit2) assert_equal(upsert.call_args_list, [call('Home')]) assert_equal(render.call_args_list, [call('Home.md', u'# test message')]) http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2b1f3c06/ForgeImporters/forgeimporters/github/wiki.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/wiki.py b/ForgeImporters/forgeimporters/github/wiki.py index b299a85..11fb50b 100644 --- a/ForgeImporters/forgeimporters/github/wiki.py +++ b/ForgeImporters/forgeimporters/github/wiki.py @@ -154,7 +154,7 @@ class GitHubWikiImporter(ToolImporter): try: M.session.artifact_orm_session._get().skip_mod_date = True with h.push_config(c, app=app): - self.get_wiki_pages(extractor.get_page_url('wiki_url'), history=with_history) + self.import_pages(extractor.get_page_url('wiki_url'), history=with_history) ThreadLocalORMSession.flush_all() g.post_event('project_updated') return app @@ -164,44 +164,49 @@ class GitHubWikiImporter(ToolImporter): finally: M.session.artifact_orm_session._get().skip_mod_date = False - def get_blobs_without_history(self, commit): + def _without_history(self, commit): for page in commit.tree.blobs: - name, ext = page.name.split('.', 1) - if ext not in self.supported_formats: - log.info('Not a wiki page %s. Skipping.' % page.name) - continue - wiki_page = WM.Page.upsert(name) - wiki_page.text = h.render_any_markup(page.name, h.really_unicode(page.data_stream.read())) - wiki_page.timestamp = wiki_page.mod_date = datetime.utcfromtimestamp(commit.committed_date) - wiki_page.viewable_by = ['all'] - wiki_page.commit() - - def get_blobs_with_history(self, commit): - for page_name in commit.stats.files.keys(): - name, ext = page_name.split('.', 1) - if ext not in self.supported_formats: - log.info('Not a wiki page %s. Skipping.' % page_name) - continue - wiki_page = WM.Page.upsert(name) - if page_name in commit.tree: - wiki_page.text = h.render_any_markup( - page_name, - h.really_unicode(commit.tree[page_name].data_stream.read())) - wiki_page.timestamp = wiki_page.mod_date = datetime.utcfromtimestamp(commit.committed_date) - wiki_page.viewable_by = ['all'] - else: - wiki_page.deleted = True - suffix = " {dt.hour}:{dt.minute}:{dt.second} {dt.day}-{dt.month}-{dt.year}".format(dt=datetime.utcnow()) - wiki_page.title += suffix - wiki_page.commit() + self._make_page(page.data_stream.read(), page.name, commit) - def get_wiki_pages(self, wiki_url, history=None): + def _with_history(self, commit): + for filename in commit.stats.files.keys(): + if filename in commit.tree: + text = commit.tree[filename].data_stream.read() + else: + # file is deleted + text = '' + self._make_page(text, filename, commit) + + def _make_page(self, text, filename, commit): + name_and_ext = filename.split('.', 1) + if len(name_and_ext) == 1: + name, ext = name_and_ext[0], None + else: + name, ext = name_and_ext + if ext and ext not in self.supported_formats: + log.info('Not a wiki page %s. Skipping.' % filename) + return + mod_date = datetime.utcfromtimestamp(commit.committed_date) + wiki_page = WM.Page.upsert(name) + if filename in commit.tree: + wiki_page.text = self.convert_markup(h.really_unicode(text), filename) + wiki_page.timestamp = wiki_page.mod_date = mod_date + wiki_page.viewable_by = ['all'] + else: + wiki_page.deleted = True + suffix = " {dt.hour}:{dt.minute}:{dt.second} {dt.day}-{dt.month}-{dt.year}".format(dt=mod_date) + wiki_page.title += suffix + wiki_page.commit() + return wiki_page + + def import_pages(self, wiki_url, history=None): wiki_path = mkdtemp() wiki = git.Repo.clone_from(wiki_url, to_path=wiki_path, bare=True) if not history: - return self.get_blobs_without_history(wiki.heads.master.commit) - for commit in reversed(list(wiki.iter_commits())): - self.get_blobs_with_history(commit) + self._without_history(wiki.heads.master.commit) + else: + for commit in reversed(list(wiki.iter_commits())): + self._with_history(commit) rmtree(wiki_path) def convert_markup(self, text, filename):