Return-Path: X-Original-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id ED46910CEF for ; Tue, 24 Sep 2013 17:07:58 +0000 (UTC) Received: (qmail 54184 invoked by uid 500); 24 Sep 2013 17:07:46 -0000 Delivered-To: apmail-incubator-allura-commits-archive@incubator.apache.org Received: (qmail 53962 invoked by uid 500); 24 Sep 2013 17:07:44 -0000 Mailing-List: contact allura-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: allura-dev@incubator.apache.org Delivered-To: mailing list allura-commits@incubator.apache.org Received: (qmail 53557 invoked by uid 99); 24 Sep 2013 17:07:40 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 24 Sep 2013 17:07:40 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 9C393908A5E; Tue, 24 Sep 2013 17:07:40 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: johnsca@apache.org To: allura-commits@incubator.apache.org Date: Tue, 24 Sep 2013 17:07:50 -0000 Message-Id: <80ab2db14f184107a244b303c8c05f0c@git.apache.org> In-Reply-To: <033c86e9ed7e48a6ad1fef6bd663c4e1@git.apache.org> References: <033c86e9ed7e48a6ad1fef6bd663c4e1@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [11/50] git commit: [#6535] ticket:417 github tickets import labels and attaches [#6535] ticket:417 github tickets import labels and attaches Conflicts: ForgeImporters/forgeimporters/github/__init__.py Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/6062bc73 Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/6062bc73 Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/6062bc73 Branch: refs/heads/cj/6422 Commit: 6062bc73b3502f4e7ba2dbd9ebef506ec987638f Parents: 5a82f13 Author: Anton Kasyanov Authored: Mon Aug 26 21:24:38 2013 +0300 Committer: Dave Brondsema Committed: Thu Sep 19 14:46:49 2013 +0000 ---------------------------------------------------------------------- .../forgeimporters/github/__init__.py | 3 +- ForgeImporters/forgeimporters/github/tracker.py | 66 +++++++++++++------- 2 files changed, 47 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6062bc73/ForgeImporters/forgeimporters/github/__init__.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py index 35c5d03..8e1a802 100644 --- a/ForgeImporters/forgeimporters/github/__init__.py +++ b/ForgeImporters/forgeimporters/github/__init__.py @@ -17,6 +17,8 @@ import logging import json +import urllib +import urllib2 from forgeimporters import base @@ -39,7 +41,6 @@ class GitHubProjectExtractor(base.ProjectExtractor): self.url = self.PAGE_MAP[page].format( project=urllib.quote(gh_project_name), ) - self.page = json.loads(urllib2.urlopen(self.url).read().decode('utf8')) def get_summary(self): return self.get_page('project_info').get('description') http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6062bc73/ForgeImporters/forgeimporters/github/tracker.py ---------------------------------------------------------------------- diff --git a/ForgeImporters/forgeimporters/github/tracker.py b/ForgeImporters/forgeimporters/github/tracker.py index 4aa73a4..4cd433b 100644 --- a/ForgeImporters/forgeimporters/github/tracker.py +++ b/ForgeImporters/forgeimporters/github/tracker.py @@ -1,5 +1,11 @@ +import re from datetime import datetime +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + from allura import model as M from allura.lib import helpers as h from ming.orm import session, ThreadLocalORMSession @@ -43,11 +49,10 @@ class GitHubTrackerImporter(ToolImporter): custom_fields=dict(), ticket_num=ticket_num) self.process_fields(ticket, issue) - #self.process_labels(ticket, issue) #self.process_comments(ticket, issue) session(ticket).flush(ticket) session(ticket).expunge(ticket) - #app.globals.custom_fields = self.postprocess_custom_fields() + #app.globals.custom_fields = self.get_milestones() app.globals.last_ticket_num = self.max_ticket_num ThreadLocalORMSession.flush_all() g.post_event('project_updated') @@ -65,6 +70,9 @@ class GitHubTrackerImporter(ToolImporter): owner_line = '*Originally owned by:* {}\n'.format(issue['assignee']['login']) else: owner_line = '' + # body processing happens here + body, attachments = self._get_attachments(issue['body']) + ticket.add_multiple_attachments(attachments) ticket.description = ( u'*Originally created by:* {creator}\n' u'{owner}' @@ -72,24 +80,9 @@ class GitHubTrackerImporter(ToolImporter): u'{body}').format( creator=issue['user']['login'], owner=owner_line, - body=issue['body'], + body=body, ) - - def process_labels(self, ticket, issue): - labels = set() - custom_fields = defaultdict(set) - for label in issue.get_issue_labels(): - if u'-' in label: - name, value = label.split(u'-', 1) - cf = self.custom_field(name) - cf['options'].add(value) - custom_fields[cf['name']].add(value) - if cf['name'] == '_milestone' and ticket.status in c.app.globals.open_status_names: - self.open_milestones.add(value) - else: - labels.add(label) - ticket.labels = list(labels) - ticket.custom_fields = {n: u', '.join(sorted(v)) for n,v in custom_fields.iteritems()} + ticket.labels = [label['name'] for label in issue['labels']] def process_comments(self, ticket, issue): for comment in issue.iter_comments(): @@ -100,8 +93,9 @@ class GitHubTrackerImporter(ToolImporter): ) p.add_multiple_attachments(comment.attachments) - def postprocess_custom_fields(self): + def get_milestones(self): custom_fields = [] + milestones = [] for name, field in self.custom_fields.iteritems(): if field['name'] == '_milestone': field['milestones'] = [{ @@ -115,4 +109,34 @@ class GitHubTrackerImporter(ToolImporter): else: field['options'] = '' custom_fields.append(field) - return custom_fields \ No newline at end of file + return custom_fields + + def _get_attachments(self, body): + # at github, attachments are images only and are included into comment's body + # usual syntax is + # ![cdbpzjc5ex4](https://f.cloud.github.com/assets/979771/1027411/a393ab5e-0e70-11e3-8a38-b93a3df904cf.jpg)\r\n + REGEXP = r'!\[[\w0-9]+?\]\(((?:https?:\/\/)?[\da-z\.-]+\.[a-z\.]{2,6}'\ + '(?:[\/\w\.-]+)*.(jpg|jpeg|png|gif))\)\r\n' + attachments = [] + found_matches = re.finditer(REGEXP, body, re.IGNORECASE) + for i, match in enumerate(found_matches): + # removing attach text from comment + body = body.replace(match.group(0), '') + # stripping url and extension + attachments.append(Attachment( + match.group(1), # url + 'attach{}.{}'.format(i + 1, match.group(2)) # extension + )) + return (body, attachments) + +class Attachment(object): + def __init__(self, url, filename): + self.url = url + self.filename = filename + self.type = None + + @property + def file(self): + fp_ish = GitHubProjectExtractor.urlopen(self.url) + fp = StringIO(fp_ish.read()) + return fp