Return-Path: X-Original-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5401A100B5 for ; Tue, 6 Aug 2013 19:06:51 +0000 (UTC) Received: (qmail 37590 invoked by uid 500); 6 Aug 2013 19:06:51 -0000 Delivered-To: apmail-incubator-allura-commits-archive@incubator.apache.org Received: (qmail 37565 invoked by uid 500); 6 Aug 2013 19:06:50 -0000 Mailing-List: contact allura-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: allura-dev@incubator.apache.org Delivered-To: mailing list allura-commits@incubator.apache.org Received: (qmail 37539 invoked by uid 99); 6 Aug 2013 19:06:48 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 06 Aug 2013 19:06:48 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id B99A91B5B1; Tue, 6 Aug 2013 19:06:48 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tvansteenburgh@apache.org To: allura-commits@incubator.apache.org Date: Tue, 06 Aug 2013 19:06:49 -0000 Message-Id: <17ba7c29d1d0415f97e734de4bf85f00@git.apache.org> In-Reply-To: <53fa85bf885b446b892e729acc74d902@git.apache.org> References: <53fa85bf885b446b892e729acc74d902@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/3] git commit: [#6480] TracExport bug fixes [#6480] TracExport bug fixes - Pass in options explicitly instead of attempting to read from a non-existent global object. - Improve logging. - Fix infinite loop bug. Signed-off-by: Tim Van Steenburgh Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/ec8deca5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/ec8deca5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/ec8deca5 Branch: refs/heads/tv/6480 Commit: ec8deca5f787193ac0ffc40a91cb085aeac62f39 Parents: eb38fed Author: Tim Van Steenburgh Authored: Tue Aug 6 18:50:46 2013 +0000 Committer: Tim Van Steenburgh Committed: Tue Aug 6 18:50:46 2013 +0000 ---------------------------------------------------------------------- Allura/allura/scripts/trac_export.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/ec8deca5/Allura/allura/scripts/trac_export.py ---------------------------------------------------------------------- diff --git a/Allura/allura/scripts/trac_export.py b/Allura/allura/scripts/trac_export.py index aeb14ea..bce548e 100644 --- a/Allura/allura/scripts/trac_export.py +++ b/Allura/allura/scripts/trac_export.py @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. - +import logging import sys import csv import urlparse @@ -35,6 +35,8 @@ from BeautifulSoup import BeautifulSoup, NavigableString import dateutil.parser import pytz +log = logging.getLogger(__name__) + def parse_options(): optparser = OptionParser(usage=''' %prog @@ -66,7 +68,7 @@ class TracExport(object): 'owner': 'assigned_to', } - def __init__(self, base_url, start_id=1): + def __init__(self, base_url, start_id=1, verbose=False, do_attachments=True): """start_id - start with at least that ticket number (actual returned ticket may have higher id if we don't have access to exact one). @@ -78,6 +80,9 @@ class TracExport(object): self.ticket_map = {} self.start_id = start_id self.page = (start_id - 1) / self.PAGE_SIZE + 1 + self.verbose = verbose + self.do_attachments = do_attachments + self.exhausted = False self.ticket_queue = self.next_ticket_ids() def remap_fields(self, dict): @@ -98,9 +103,9 @@ class TracExport(object): glue = '&' if '?' in suburl else '?' return url + glue + 'format=' + type - @staticmethod - def log_url(url): - if options.verbose: + def log_url(self, url): + log.info(url) + if self.verbose: print >>sys.stderr, url @classmethod @@ -198,7 +203,7 @@ class TracExport(object): ''' t = self.parse_ticket_body(id) t['comments'] = self.parse_ticket_comments(id) - if options.do_attachments: + if self.do_attachments: atts = self.parse_ticket_attachments(id) if atts: t['attachments'] = atts @@ -230,6 +235,9 @@ class TracExport(object): res.append((id, extra)) self.page += 1 + if len(res) < self.PAGE_SIZE: + self.exhausted = True + return res def __iter__(self): @@ -238,7 +246,7 @@ class TracExport(object): def next(self): while True: # queue empty, try to fetch more - if len(self.ticket_queue) == 0: + if len(self.ticket_queue) == 0 and not self.exhausted: self.ticket_queue = self.next_ticket_ids() # there aren't any more, we're really done if len(self.ticket_queue) == 0: @@ -258,7 +266,8 @@ class DateJSONEncoder(json.JSONEncoder): def main(): options, args = parse_options() - ex = TracExport(args[0], start_id=options.start_id) + ex = TracExport(args[0], start_id=options.start_id, + verbose=options.verbose, do_attachments=options.do_attachments) # Implement iterator sequence limiting using islice() doc = [t for t in islice(ex, options.limit)]