incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject [4/4] git commit: [#6845] Preserve inter-project links when importing from GC
Date Tue, 12 Nov 2013 18:14:08 GMT
[#6845] Preserve inter-project links when importing from GC

Signed-off-by: Cory Johns <cjohns@slashdotmedia.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/b459bd37
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/b459bd37
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/b459bd37

Branch: refs/heads/cj/6845
Commit: b459bd37057a733eaa61c2cc48076b4d0b3f0650
Parents: 010d6a4
Author: Cory Johns <cjohns@slashdotmedia.com>
Authored: Tue Nov 12 18:13:26 2013 +0000
Committer: Cory Johns <cjohns@slashdotmedia.com>
Committed: Tue Nov 12 18:13:26 2013 +0000

----------------------------------------------------------------------
 .../forgeimporters/google/__init__.py           | 36 +++++++++++++++-----
 .../tests/data/google/test-issue.html           |  2 +-
 .../tests/google/functional/test_tracker.py     |  4 +--
 .../tests/google/test_extractor.py              |  6 ++--
 4 files changed, 34 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b459bd37/ForgeImporters/forgeimporters/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/__init__.py b/ForgeImporters/forgeimporters/google/__init__.py
index 261b9f9..7a75eaf 100644
--- a/ForgeImporters/forgeimporters/google/__init__.py
+++ b/ForgeImporters/forgeimporters/google/__init__.py
@@ -52,21 +52,41 @@ def _as_text(node, chunks=None):
             _as_text(n, chunks)
     return ''.join(chunks)
 
-def _as_markdown(tag):
+def _as_markdown(tag, project_name):
     fragments = []
     for fragment in tag:
         if getattr(fragment, 'name', None) == 'a':
             href = urlparse(fragment['href'])
             qs = parse_qs(href.query)
-            if not href.netloc and 'id' in qs:
+            gc_link = not href.netloc or href.netloc == 'code.google.com'
+            target_project = href.path.split('/')[2]
+            internal_link = target_project == project_name
+            if gc_link and internal_link and 'id' in qs:
+                # rewrite issue 123 project-internal issue links
                 fragment = '[%s](#%s)' % (fragment.text, qs['id'][0])
-            elif not href.netloc and 'r' in qs:
+            elif gc_link and internal_link and 'r' in qs:
+                # rewrite r123 project-internal revision links
                 fragment = '[r%s]' % qs['r'][0]
+            elif gc_link:
+                # preserve GC-internal links (probably issue PROJECT:123 inter-project issue
links)
+                fragment = '[%s](%s)' % (
+                        h.plain2markdown(fragment.text, preserve_multiple_spaces=True, has_html_entities=True),
+                        urljoin('https://code.google.com/p/%s/issues/' % project_name, fragment['href']),
+                    )
             else:
+                # un-link all others
                 fragment = h.plain2markdown(fragment.text, preserve_multiple_spaces=True,
has_html_entities=True)
         elif getattr(fragment, 'name', None) == 'i':
+            # preserve styling of "(No comment was entered for this change.)" messages
             fragment = '*%s*' % h.plain2markdown(fragment.text, preserve_multiple_spaces=True,
has_html_entities=True)
+        elif getattr(fragment, 'name', None) == 'b':
+            # preserve styling of issue template
+            fragment = '**%s**' % h.plain2markdown(fragment.text, preserve_multiple_spaces=True,
has_html_entities=True)
+        elif getattr(fragment, 'name', None) == 'br':
+            # preserve forced line-breaks
+            fragment = '\n'
         else:
+            # convert all others to plain MD
             fragment = h.plain2markdown(str(fragment), preserve_multiple_spaces=True, has_html_entities=True)
         fragments.append(fragment)
     return ''.join(fragments).strip()
@@ -203,7 +223,7 @@ class GoogleCodeProjectExtractor(ProjectExtractor):
         return bs.text
 
     def get_issue_description(self):
-        return _as_markdown(self.page.find(id='hc0').pre)
+        return _as_markdown(self.page.find(id='hc0').pre, self.project_name)
 
     def get_issue_created_date(self):
         return self.page.find(id='hc0').find('span', 'date').get('title')
@@ -211,7 +231,7 @@ class GoogleCodeProjectExtractor(ProjectExtractor):
     def get_issue_mod_date(self):
         comments = self.page.findAll('div', 'issuecomment')
         if comments:
-            last_update = Comment(comments[-1])
+            last_update = Comment(comments[-1], self.project_name)
             return last_update.created_date
         else:
             return self.get_issue_created_date()
@@ -250,7 +270,7 @@ class GoogleCodeProjectExtractor(ProjectExtractor):
 
     def iter_comments(self):
         for comment in self.page.findAll('div', 'issuecomment'):
-            yield Comment(comment)
+            yield Comment(comment, self.project_name)
 
 class UserLink(object):
     def __init__(self, tag):
@@ -283,10 +303,10 @@ def _get_attachments(tag):
         return []
 
 class Comment(object):
-    def __init__(self, tag):
+    def __init__(self, tag, project_name):
         self.author = UserLink(tag.find('span', 'author').find(True, 'userlink'))
         self.created_date = tag.find('span', 'date').get('title')
-        self.body = _as_markdown(tag.find('pre'))
+        self.body = _as_markdown(tag.find('pre'), project_name)
         self._get_updates(tag)
         self.attachments = _get_attachments(tag)
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b459bd37/ForgeImporters/forgeimporters/tests/data/google/test-issue.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/data/google/test-issue.html b/ForgeImporters/forgeimporters/tests/data/google/test-issue.html
index 374c0e8..c6bce0a 100644
--- a/ForgeImporters/forgeimporters/tests/data/google/test-issue.html
+++ b/ForgeImporters/forgeimporters/tests/data/google/test-issue.html
@@ -452,7 +452,7 @@ Last comment
 <a name="c4" href="/p/allura-google-importer/issues/detail?id=6#c4">#4</a>
 <a class="userlink" href="/u/101557263855536553789/">john...@gmail.com</a></span>
 <pre>
-Oh, I forgot one
+Oh, I forgot one (with an inter-project reference to <a href="/p/other-project/issues/detail?id=1">issue
other-project:1</a>)
 </pre>
 <div class="updates">
 <div class="round4"></div>

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b459bd37/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py b/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
index 2e52be6..c22f7d0 100644
--- a/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
+++ b/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
@@ -41,7 +41,7 @@ class TestGCTrackerImporter(TestCase):
     def _make_extractor(self, html):
         with mock.patch.object(base.h, 'urlopen') as urlopen:
             urlopen.return_value = ''
-            extractor = google.GoogleCodeProjectExtractor('my-project', 'project_info')
+            extractor = google.GoogleCodeProjectExtractor('allura-google-importer', 'project_info')
         extractor.page = BeautifulSoup(html)
         extractor.url = "http://test/issue/?id=1"
         return extractor
@@ -241,7 +241,7 @@ class TestGCTrackerImporter(TestCase):
                     'text': (
                             '*Originally posted by:* [john...@gmail.com](http://code.google.com/u/101557263855536553789/)\n'
                             '\n'
-                            'Oh, I forgot one\n'
+                            'Oh, I forgot one \\(with an inter\\-project reference to [issue
other\\-project:1](https://code.google.com/p/other-project/issues/detail?id=1)\\)\n'
                             '\n'
                             '**Labels:** OpSys-OSX'
                         ),

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b459bd37/ForgeImporters/forgeimporters/tests/google/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_extractor.py b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
index 032e122..78bde81 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
@@ -124,7 +124,7 @@ class TestGoogleCodeProjectExtractor(TestCase):
     def _make_extractor(self, html):
         from BeautifulSoup import BeautifulSoup
         with mock.patch.object(base.ProjectExtractor, 'urlopen'):
-            extractor = google.GoogleCodeProjectExtractor('my-project')
+            extractor = google.GoogleCodeProjectExtractor('allura-google-importer')
         extractor.page = BeautifulSoup(html)
         extractor.get_page = lambda pagename: extractor.page
         extractor.url="http://test/source/browse"
@@ -278,7 +278,7 @@ class TestGoogleCodeProjectExtractor(TestCase):
                     'author.name': 'john...@gmail.com',
                     'author.url': 'http://code.google.com/u/101557263855536553789/',
                     'created_date': 'Thu Aug  8 15:36:57 2013',
-                    'body': 'Oh, I forgot one',
+                    'body': 'Oh, I forgot one \\(with an inter-project reference to [issue
other-project:1](https://code.google.com/p/other-project/issues/detail?id=1)\\)',
                     'updates': {'Labels:': 'OpSys-OSX'},
                     'attachments': [],
                 },
@@ -390,7 +390,7 @@ class TestComment(TestCase):
     def test_init(self):
         from BeautifulSoup import BeautifulSoup
         html = BeautifulSoup(self.html)
-        comment = google.Comment(html.find('div', 'issuecomment'))
+        comment = google.Comment(html.find('div', 'issuecomment'), 'pychess')
         self.assertEqual(comment.updates, {
             u'Summary:': u'Make PyChess keyboard accessible',
             u'Status:': u'Accepted',


Mime
View raw message