incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brond...@apache.org
Subject [17/41] git commit: [#6535] ticket:417 basic github tickets import
Date Thu, 19 Sep 2013 16:21:49 GMT
[#6535] ticket:417 basic github tickets import

Conflicts:
	ForgeImporters/forgeimporters/base.py


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/5a82f13a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/5a82f13a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/5a82f13a

Branch: refs/heads/db/5822
Commit: 5a82f13a32c497cf6d858a3c8155c5e274d34fc6
Parents: 8a32af8
Author: Anton Kasyanov <mind1master@gmail.com>
Authored: Mon Aug 26 18:56:42 2013 +0300
Committer: Dave Brondsema <dbrondsema@slashdotmedia.com>
Committed: Thu Sep 19 14:46:49 2013 +0000

----------------------------------------------------------------------
 ForgeImporters/forgeimporters/base.py           |   1 -
 .../forgeimporters/github/__init__.py           |  25 ++++-
 ForgeImporters/forgeimporters/github/tracker.py | 112 ++++++++++++++++++-
 3 files changed, 135 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5a82f13a/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 2f296a0..ea01968 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -210,7 +210,6 @@ class ProjectImporter(BaseController):
         tools = {}
         for ep in h.iter_entry_points('allura.importers'):
             epv = ep.load()
-            print epv, epv.source, self.source
             if epv.source == self.source:
                 tools[ep.name] = epv()
         return tools

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5a82f13a/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 5cd6dc1..35c5d03 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -25,12 +25,22 @@ log = logging.getLogger(__name__)
 
 class GitHubProjectExtractor(base.ProjectExtractor):
     PAGE_MAP = {
-            'project_info': 'https://api.github.com/repos/{project_name}',
+            'project_info': 'https://api.github.com/repos/{project}',
+            'issues': 'https://api.github.com/repos/{project}/issues',
         }
+    POSSIBLE_STATES = ('opened', 'closed')
 
     def parse_page(self, page):
         return json.loads(page.read().decode('utf8'))
 
+    def __init__(self, allura_project, gh_project_name, page):
+        self.project = allura_project
+        self.gh_project_name = gh_project_name
+        self.url = self.PAGE_MAP[page].format(
+            project=urllib.quote(gh_project_name),
+        )
+        self.page = json.loads(urllib2.urlopen(self.url).read().decode('utf8'))
+
     def get_summary(self):
         return self.get_page('project_info').get('description')
 
@@ -39,3 +49,16 @@ class GitHubProjectExtractor(base.ProjectExtractor):
 
     def get_repo_url(self):
         return self.get_page('project_info').get('clone_url')
+
+    def iter_issues(self):
+        # github api doesn't allow getting closed and opened tickets in one query
+        issues = []
+        self.url += '?state={state}'
+        for state in self.POSSIBLE_STATES:
+            issue_list_url = self.url.format(
+                state=state,
+            )
+            issues += json.loads(urllib2.urlopen(issue_list_url).read().decode('utf8'))
+        issues.sort(key=lambda x: x['number'])
+        for issue in issues:
+            yield (issue['number'], issue)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5a82f13a/ForgeImporters/forgeimporters/github/tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tracker.py b/ForgeImporters/forgeimporters/github/tracker.py
index 798da5c..4aa73a4 100644
--- a/ForgeImporters/forgeimporters/github/tracker.py
+++ b/ForgeImporters/forgeimporters/github/tracker.py
@@ -1,8 +1,118 @@
+from datetime import datetime
+
+from allura import model as M
+from allura.lib import helpers as h
+from ming.orm import session, ThreadLocalORMSession
+from pylons import tmpl_context as c
+from pylons import app_globals as g
+
+from . import GitHubProjectExtractor
 from ..base import ToolImporter
 from forgetracker.tracker_main import ForgeTrackerApp
+from forgetracker import model as TM
+
+
 
 class GitHubTrackerImporter(ToolImporter):
     source = 'GitHub'
     target_app = ForgeTrackerApp
     controller = None
-    tool_label = 'Issues'
\ No newline at end of file
+    tool_label = 'Issues'
+    max_ticket_num = 0
+
+    def import_tool(self, project, user, project_name, mount_point=None,
+            mount_label=None, **kw):
+        app = project.install_app('tickets', mount_point, mount_label,
+                EnableVoting=True,
+                open_status_names='New Accepted',
+                closed_status_names='Done',
+            )
+        ThreadLocalORMSession.flush_all()
+        extractor = GitHubProjectExtractor(
+            project,
+            '{}/{}'.format(kw['user_name'],project_name),
+            'issues',
+        )
+        try:
+            M.session.artifact_orm_session._get().skip_mod_date = True
+            with h.push_config(c, user=M.User.anonymous(), app=app):
+                for ticket_num, issue in extractor.iter_issues():
+                    self.max_ticket_num = max(ticket_num, self.max_ticket_num)
+                    ticket = TM.Ticket(
+                        app_config_id=app.config._id,
+                        custom_fields=dict(),
+                        ticket_num=ticket_num)
+                    self.process_fields(ticket, issue)
+                    #self.process_labels(ticket, issue)
+                    #self.process_comments(ticket, issue)
+                    session(ticket).flush(ticket)
+                    session(ticket).expunge(ticket)
+                #app.globals.custom_fields = self.postprocess_custom_fields()
+                app.globals.last_ticket_num = self.max_ticket_num
+                ThreadLocalORMSession.flush_all()
+            g.post_event('project_updated')
+            app.globals.invalidate_bin_counts()
+            return app
+        finally:
+            M.session.artifact_orm_session._get().skip_mod_date = False
+
+    def process_fields(self, ticket, issue):
+        ticket.summary = issue['title']
+        ticket.status = issue['state']
+        ticket.created_date = datetime.strptime(issue['created_at'], '%Y-%m-%dT%H:%M:%SZ')
+        ticket.mod_date = datetime.strptime(issue['updated_at'], '%Y-%m-%dT%H:%M:%SZ')
+        if issue['assignee']:
+            owner_line = '*Originally owned by:* {}\n'.format(issue['assignee']['login'])
+        else:
+            owner_line = ''
+        ticket.description = (
+                u'*Originally created by:* {creator}\n'
+                u'{owner}'
+                u'\n'
+                u'{body}').format(
+                    creator=issue['user']['login'],
+                    owner=owner_line,
+                    body=issue['body'],
+                )
+
+    def process_labels(self, ticket, issue):
+        labels = set()
+        custom_fields = defaultdict(set)
+        for label in issue.get_issue_labels():
+            if u'-' in label:
+                name, value = label.split(u'-', 1)
+                cf = self.custom_field(name)
+                cf['options'].add(value)
+                custom_fields[cf['name']].add(value)
+                if cf['name'] == '_milestone' and ticket.status in c.app.globals.open_status_names:
+                    self.open_milestones.add(value)
+            else:
+                labels.add(label)
+        ticket.labels = list(labels)
+        ticket.custom_fields = {n: u', '.join(sorted(v)) for n,v in custom_fields.iteritems()}
+
+    def process_comments(self, ticket, issue):
+        for comment in issue.iter_comments():
+            p = ticket.discussion_thread.add_post(
+                    text = comment.annotated_text,
+                    ignore_security = True,
+                    timestamp = datetime.strptime(comment.created_date, '%c'),
+                )
+            p.add_multiple_attachments(comment.attachments)
+
+    def postprocess_custom_fields(self):
+        custom_fields = []
+        for name, field in self.custom_fields.iteritems():
+            if field['name'] == '_milestone':
+                field['milestones'] = [{
+                        'name': milestone,
+                        'due_date': None,
+                        'complete': milestone not in self.open_milestones,
+                    } for milestone in sorted(field['options'])]
+                field['options'] = ''
+            elif field['type'] == 'select':
+                field['options'] = ' '.join(field['options'])
+            else:
+                field['options'] = ''
+            custom_fields.append(field)
+        return custom_fields
\ No newline at end of file


Mime
View raw message