incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject [07/15] git commit: [#6456] Added Google Code project importer
Date Mon, 22 Jul 2013 22:54:14 GMT
[#6456] Added Google Code project importer

Signed-off-by: Cory Johns <cjohns@slashdotmedia.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/b82a3767
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/b82a3767
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/b82a3767

Branch: refs/heads/cj/6456
Commit: b82a37670d20fb55fe0848db454bc180d901128b
Parents: d971b6c
Author: Cory Johns <cjohns@slashdotmedia.com>
Authored: Fri Jul 19 19:21:05 2013 +0000
Committer: Cory Johns <cjohns@slashdotmedia.com>
Committed: Mon Jul 22 18:36:15 2013 +0000

----------------------------------------------------------------------
 .../forgeimporters/google/__init__.py           |  17 +--
 ForgeImporters/forgeimporters/google/project.py |  83 +++++++++++++++
 ForgeImporters/forgeimporters/google/tasks.py   |  33 ++++++
 .../google/templates/project.html               | 104 +++++++++++++++++++
 .../forgeimporters/tests/google/__init__.py     |   0
 .../tests/google/test_extractor.py              |  93 +++++++++++++++++
 .../forgeimporters/tests/google/test_tasks.py   |  33 ++++++
 ForgeImporters/setup.py                         |   1 +
 8 files changed, 358 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/__init__.py b/ForgeImporters/forgeimporters/google/__init__.py
index 0feb64a..e01dcfb 100644
--- a/ForgeImporters/forgeimporters/google/__init__.py
+++ b/ForgeImporters/forgeimporters/google/__init__.py
@@ -19,6 +19,10 @@ import urllib
 import urllib2
 from urlparse import urlparse
 from collections import defaultdict
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
 
 from BeautifulSoup import BeautifulSoup
 
@@ -50,19 +54,20 @@ class GoogleCodeProjectExtractor(object):
         self.page = BeautifulSoup(page)
 
     def get_short_description(self):
-        self.project.short_description = str(self.page.find(itemprop='description')).strip()
+        self.project.short_description = self.page.find(itemprop='description').string.strip()
 
     def get_icon(self):
-        icon_url = self.page.find(itemprop='image').src
+        icon_url = self.page.find(itemprop='image').attrMap['src']
         icon_name = urllib.unquote(urlparse(icon_url).path).split('/')[-1]
-        fp = urllib2.urlopen(icon_url)
+        fp_ish = urllib2.urlopen(icon_url)
+        fp = StringIO(fp_ish.read())
         M.ProjectFile.save_image(
-            icon_name, fp, fp.info()['content-type'], square=True,
+            icon_name, fp, fp_ish.info()['content-type'], square=True,
             thumbnail_size=(48,48),
-            thumbnail_meta=dict(project_id=self.project._id, category='icon'))
+            thumbnail_meta={'project_id': self.project._id, 'category': 'icon'})
 
     def get_license(self):
-        license = str(self.page.find(text='Code license:').findNext('td')).strip()
+        license = self.page.find(text='Code license').findNext().find('a').string.strip()
         trove = M.TroveCategory.query.get(fullname=self.LICENSE_MAP[license])
         self.project.trove_license.append(trove._id)
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/google/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/project.py b/ForgeImporters/forgeimporters/google/project.py
new file mode 100644
index 0000000..fdd3c97
--- /dev/null
+++ b/ForgeImporters/forgeimporters/google/project.py
@@ -0,0 +1,83 @@
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+
+from tg import expose, validate, flash, redirect
+from tg.decorators import with_trailing_slash
+from pylons import tmpl_context as c
+from formencode import validators as fev, schema
+
+from allura import model as M
+from allura.lib.decorators import require_post
+from allura.lib.widgets.forms import NeighborhoodProjectShortNameValidator
+from allura.lib import helpers as h
+
+from .. import base
+from . import tasks
+
+
+
+class GoogleCodeProjectForm(schema.Schema):
+    neighborhood = fev.PlainText(not_empty=True)
+    gc_project_name = fev.Regex(r'^[a-z0-9][a-z0-9-]{,61}$', not_empty=True)
+    project_unixname = NeighborhoodProjectShortNameValidator()
+    tools = base.ToolsValidator('Google Code')
+
+
+class GoogleCodeProjectImporter(base.ProjectImporter):
+    source = 'Google Code'
+
+    @with_trailing_slash
+    @expose('jinja:forgeimporters.google:templates/project.html')
+    def index(self, **kw):
+        neighborhood = M.Neighborhood.query.get(url_prefix='/p/')
+        return {'importer': self, 'neighborhood': neighborhood}
+
+    @require_post()
+    @expose()
+    @validate(GoogleCodeProjectForm(), error_handler=index)
+    def process(self, gc_project_name=None, project_unixname=None, tools=None, **kw):
+        neighborhood = M.Neighborhood.query.get(url_prefix='/p/')
+        project_name = h.really_unicode(gc_project_name).encode('utf-8')
+        project_unixname = h.really_unicode(project_unixname).encode('utf-8').lower()
+
+        try:
+            c.project = neighborhood.register_project(project_unixname,
+                    project_name=project_name)
+        except exceptions.ProjectOverlimitError:
+            flash("You have exceeded the maximum number of projects you are allowed to create",
'error')
+            redirect('.')
+        except exceptions.ProjectRatelimitError:
+            flash("Project creation rate limit exceeded.  Please try again later.", 'error')
+            redirect('.')
+        except Exception as e:
+            log.error('error registering project: %s', project_unixname, exc_info=True)
+            flash('Internal Error. Please try again later.', 'error')
+            redirect('.')
+
+        c.project.set_tool_data('google-code', project_name=gc_project_name)
+        tasks.import_project_info.post()
+        for tool in tools:
+            tool.import_tool(c.project)
+
+        flash('Welcome to the SourceForge Project System! '
+              'Your project data will be imported and should show up here shortly.')
+        redirect(c.project.script_name + 'admin/overview')
+
+    @expose('json:')
+    @validate(GoogleCodeProjectForm())
+    def check_names(self, gc_project_name=None, project_unixname=None, tools=None, **kw):
+        return c.form_errors

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/google/tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tasks.py b/ForgeImporters/forgeimporters/google/tasks.py
new file mode 100644
index 0000000..79851c3
--- /dev/null
+++ b/ForgeImporters/forgeimporters/google/tasks.py
@@ -0,0 +1,33 @@
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+
+from pylons import tmpl_context as c
+
+from ming.orm import ThreadLocalORMSession
+
+from allura.lib.decorators import task
+
+from . import GoogleCodeProjectExtractor
+
+
+@task
+def import_project_info():
+    extractor = GoogleCodeProjectExtractor(c.project, 'project_info')
+    extractor.get_short_description()
+    extractor.get_icon()
+    extractor.get_license()
+    ThreadLocalORMSession.flush_all()

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/google/templates/project.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/templates/project.html b/ForgeImporters/forgeimporters/google/templates/project.html
new file mode 100644
index 0000000..ae9362a
--- /dev/null
+++ b/ForgeImporters/forgeimporters/google/templates/project.html
@@ -0,0 +1,104 @@
+{#-
+       Licensed to the Apache Software Foundation (ASF) under one
+       or more contributor license agreements.  See the NOTICE file
+       distributed with this work for additional information
+       regarding copyright ownership.  The ASF licenses this file
+       to you under the Apache License, Version 2.0 (the
+       "License"); you may not use this file except in compliance
+       with the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+       Unless required by applicable law or agreed to in writing,
+       software distributed under the License is distributed on an
+       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+       KIND, either express or implied.  See the License for the
+       specific language governing permissions and limitations
+       under the License.
+-#}
+{% extends 'forgeimporters:templates/project_base.html' %}
+
+{% block extra_css %}
+    {{ super() }}
+    <style type="text/css">
+        #project-import-form #project-fields input {
+            width: 88%;
+        }
+
+        .hidden { display: none; }
+    </style>
+{% endblock %}
+
+{% block extra_js %}
+    {{ super() }}
+    <script type="text/javascript">
+        var timers = {};
+        function delay(callback, ms) {
+            clearTimeout(timers[callback]);
+            timers[callback] = setTimeout(callback, ms);
+        }
+
+        var manual = false;
+        function suggest_name() {
+            if (!manual) {
+                $('#project_unixname').val($('#gc_project_name').val()).trigger('change');
+            }
+        }
+
+        function check_names() {
+            var data = {
+                'neighborhood': $('#neighborhood').val(),
+                'gc_project_name': $('#gc_project_name').val(),
+                'project_unixname': $('#project_unixname').val()
+            };
+            $.getJSON('check_names', data, function(result) {
+                $('#gc_project_name_error').addClass('hidden');
+                $('#project_unixname_error').addClass('hidden');
+                for(var field in result) {
+                    $('#'+field+'_error').text(result[field]).removeClass('hidden');
+                }
+            });
+        }
+
+        function update_url() {
+            $('#url-fragment').text($('#project_unixname').val());
+        }
+
+        $(function() {
+            $('#gc_project_name').focus().bind('change keyup', suggest_name);
+
+            $('#project_unixname').bind('change keyup', function(event) {
+                if (event.type == 'keyup') {
+                    manual = true;
+                }
+                update_url();
+                delay(check_names, 500);
+            });
+        });
+    </script>
+{% endblock %}
+
+{% block project_fields %}
+    <div class="grid-6">
+        <label>Google Project Name</label>
+    </div>
+    <div class="grid-10">
+        <input id="gc_project_name" name="gc_project_name" value="{{c.form_values['gc_project_name']}}"/>
+        <div id="gc_project_name_error" class="error{% if not c.form_errors['gc_project_name']
%} hidden{% endif %}">
+            {{c.form_errors['gc_project_name']}}
+        </div>
+    </div>
+
+    <div class="grid-6" style="clear:left">
+        <label>URL Name</label>
+    </div>
+    <div class="grid-10">
+        <input id="project_unixname" name="project_unixname" value="{{c.form_values['project_unixname']}}"/>
+        <div id="project_unixname_error" class="error{% if not c.form_errors['project_unixname']
%} hidden{% endif %}">
+            {{c.form_errors['project_unixname']}}
+        </div>
+        <div id="project-url">
+            http://{{request.environ['HTTP_HOST']}}{{neighborhood.url()}}<span id="url-fragment">{{c.form_values['project_unixname']}}</span>
+        </div>
+    </div>
+{% endblock %}

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/tests/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/__init__.py b/ForgeImporters/forgeimporters/tests/google/__init__.py
new file mode 100644
index 0000000..e69de29

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/tests/google/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_extractor.py b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
new file mode 100644
index 0000000..f03c82c
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
@@ -0,0 +1,93 @@
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+
+from unittest import TestCase
+
+import mock
+
+from ... import google
+
+
+class TestGoogleCodeProjectExtractor(TestCase):
+    def setUp(self):
+        self._p_urlopen = mock.patch.object(google.urllib2, 'urlopen')
+        self._p_soup = mock.patch.object(google, 'BeautifulSoup')
+        self.urlopen = self._p_urlopen.start()
+        self.soup = self._p_soup.start()
+        self.project = mock.Mock(name='project')
+        self.project.get_tool_data.return_value = 'my-project'
+
+    def tearDown(self):
+        self._p_urlopen.stop()
+        self._p_soup.stop()
+
+    def test_init(self):
+        extractor = google.GoogleCodeProjectExtractor(self.project, 'project_info')
+
+        self.project.get_tool_data.assert_called_once_with('google-code', 'project_name')
+        self.urlopen.assert_called_once_with('http://code.google.com/p/my-project/')
+        self.assertEqual(extractor.project, self.project)
+        self.soup.assert_called_once_with(self.urlopen.return_value)
+        self.assertEqual(extractor.page, self.soup.return_value)
+
+    def test_get_short_description(self):
+        extractor = google.GoogleCodeProjectExtractor(self.project, 'project_info')
+        extractor.page.find.return_value.string = 'My Super Project'
+
+        extractor.get_short_description()
+
+        extractor.page.find.assert_called_once_with(itemprop='description')
+        self.assertEqual(self.project.short_description, 'My Super Project')
+
+    @mock.patch.object(google, 'StringIO')
+    @mock.patch.object(google, 'M')
+    def test_get_icon(self, M, StringIO):
+        self.urlopen.return_value.info.return_value = {'content-type': 'image/png'}
+        extractor = google.GoogleCodeProjectExtractor(self.project, 'project_info')
+        extractor.page.find.return_value.attrMap = {'src': 'http://example.com/foo/bar/my-logo.png'}
+        self.urlopen.reset_mock()
+
+        extractor.get_icon()
+
+        extractor.page.find.assert_called_once_with(itemprop='image')
+        self.urlopen.assert_called_once_with('http://example.com/foo/bar/my-logo.png')
+        self.urlopen.return_value.info.assert_called_once_with()
+        StringIO.assert_called_once_with(self.urlopen.return_value.read.return_value)
+        M.ProjectFile.save_image.assert_called_once_with(
+            'my-logo.png', StringIO.return_value, 'image/png', square=True,
+            thumbnail_size=(48,48), thumbnail_meta={
+                'project_id': self.project._id, 'category': 'icon'})
+
+    @mock.patch.object(google, 'M')
+    def test_get_license(self, M):
+        self.project.trove_license = []
+        extractor = google.GoogleCodeProjectExtractor(self.project, 'project_info')
+        extractor.page.find.return_value.findNext.return_value.find.return_value.string =
'  New BSD License  '
+        trove = M.TroveCategory.query.get.return_value
+
+        extractor.get_license()
+
+        extractor.page.find.assert_called_once_with(text='Code license')
+        extractor.page.find.return_value.findNext.assert_called_once_with()
+        extractor.page.find.return_value.findNext.return_value.find.assert_called_once_with('a')
+        self.assertEqual(self.project.trove_license, [trove._id])
+        M.TroveCategory.query.get.assert_called_once_with(fullname='BSD License')
+
+        M.TroveCategory.query.get.reset_mock()
+        extractor.page.find.return_value.findNext.return_value.find.return_value.string =
'non-existant license'
+        extractor.get_license()
+        M.TroveCategory.query.get.assert_called_once_with(fullname='Other/Proprietary License')

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/tests/google/test_tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_tasks.py b/ForgeImporters/forgeimporters/tests/google/test_tasks.py
new file mode 100644
index 0000000..a21d083
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/google/test_tasks.py
@@ -0,0 +1,33 @@
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+
+import mock
+
+from ...google import tasks
+
+
+@mock.patch.object(tasks, 'GoogleCodeProjectExtractor')
+@mock.patch.object(tasks, 'ThreadLocalORMSession')
+@mock.patch.object(tasks, 'c')
+def test_import_project_info(c, session, gpe):
+    c.project = mock.Mock(name='project')
+    tasks.import_project_info()
+    gpe.assert_called_once_with(c.project, 'project_info')
+    gpe.return_value.get_short_description.assert_called_once_with()
+    gpe.return_value.get_icon.assert_called_once_with()
+    gpe.return_value.get_license.assert_called_once_with()
+    session.flush_all.assert_called_once_with()

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/setup.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/setup.py b/ForgeImporters/setup.py
index 5e45638..a928504 100644
--- a/ForgeImporters/setup.py
+++ b/ForgeImporters/setup.py
@@ -34,6 +34,7 @@ setup(name='ForgeImporters',
       entry_points="""
       # -*- Entry points: -*-
       [allura.project_importers]
+      google-code = forgeimporters.google.project:GoogleCodeProjectImporter
 
       [allura.importers]
       """,)


Mime
View raw message