Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 5A6E8200B66 for ; Thu, 4 Aug 2016 03:00:11 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 59026160AAD; Thu, 4 Aug 2016 01:00:11 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 4E6E5160A86 for ; Thu, 4 Aug 2016 03:00:10 +0200 (CEST) Received: (qmail 16369 invoked by uid 500); 4 Aug 2016 01:00:09 -0000 Mailing-List: contact commits-help@yetus.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@yetus.apache.org Delivered-To: mailing list commits@yetus.apache.org Received: (qmail 16354 invoked by uid 99); 4 Aug 2016 01:00:09 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 04 Aug 2016 01:00:09 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D0B47E3839; Thu, 4 Aug 2016 01:00:08 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sekikn@apache.org To: commits@yetus.apache.org Message-Id: <36e8a960e937471bb0e1c2e9c931e9e8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: yetus git commit: YETUS-430 Refactor releasedocmaker.py in separate modules Date: Thu, 4 Aug 2016 01:00:08 +0000 (UTC) archived-at: Thu, 04 Aug 2016 01:00:11 -0000 Repository: yetus Updated Branches: refs/heads/master b110b2580 -> 6f3aa7ae3 YETUS-430 Refactor releasedocmaker.py in separate modules - Removed redundant fallback imports - Renamed functions to snake_case - Fixed the pylint issue of iterating over dictionary keys - Removed _ from parameter names - Renamed mstr to to_unicode Signed-off-by: Kengo Seki Signed-off-by: Andrew Wang Project: http://git-wip-us.apache.org/repos/asf/yetus/repo Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/6f3aa7ae Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/6f3aa7ae Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/6f3aa7ae Branch: refs/heads/master Commit: 6f3aa7ae3ea63d55233419b8979564ec41e17d60 Parents: b110b25 Author: Ajay Yadava Authored: Sun Jul 24 22:26:17 2016 +0530 Committer: Kengo Seki Committed: Thu Aug 4 09:59:09 2016 +0900 ---------------------------------------------------------------------- release-doc-maker/releasedocmaker.py | 145 ++++-------------------------- release-doc-maker/utils.py | 127 ++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 127 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/yetus/blob/6f3aa7ae/release-doc-maker/releasedocmaker.py ---------------------------------------------------------------------- diff --git a/release-doc-maker/releasedocmaker.py b/release-doc-maker/releasedocmaker.py index 5f32491..c2c6ac8 100755 --- a/release-doc-maker/releasedocmaker.py +++ b/release-doc-maker/releasedocmaker.py @@ -28,14 +28,10 @@ import sys import urllib import urllib2 import httplib -try: - import json -except ImportError: - import simplejson as json -try: - set -except NameError: - from sets import Set as set # pylint: disable=redefined-builtin +import json +from utils import to_unicode, text_sanitize, processrelnote, Outputs + + try: import dateutil.parser except ImportError: @@ -44,8 +40,7 @@ except ImportError: sys.exit(1) RELEASE_VERSION = {} -NAME_PATTERN = re.compile(r' \([0-9]+\)') -RELNOTE_PATTERN = re.compile('^\<\!\-\- ([a-z]+) \-\-\>') + JIRA_BASE_URL = "https://issues.apache.org/jira" SORTTYPE = 'resolutiondate' SORTORDER = 'older' @@ -78,63 +73,6 @@ ASF_LICENSE = ''' ''' -def clean(_str): - return markdownsanitize(re.sub(NAME_PATTERN, "", _str)) - - -def format_components(_str): - _str = re.sub(NAME_PATTERN, '', _str).replace("'", "") - if _str != "": - ret = _str - else: - # some markdown parsers don't like empty tables - ret = "." - return clean(ret) - - -# convert to utf-8 -def markdownsanitize(_str): - _str = _str.encode('utf-8') - _str = _str.replace("\r", "") - _str = _str.rstrip() - return _str - - -# same thing as markdownsanitize, -# except markdown metachars are also -# escaped as well as more -# things we don't want doxia, etc, to -# screw up -def textsanitize(_str): - _str = markdownsanitize(_str) - _str = _str.replace("_", r"\_") - _str = _str.replace("|", r"\|") - _str = _str.replace("<", r"\<") - _str = _str.replace(">", r"\>") - _str = _str.replace("*", r"\*") - _str = _str.rstrip() - return _str - - -# if release notes have a special marker, -# we'll treat them as already in markdown format -def processrelnote(_str): - fmt = RELNOTE_PATTERN.match(_str) - if fmt is None: - return textsanitize(_str) - else: - return { - 'markdown': markdownsanitize(_str), - }.get( - fmt.group(1), textsanitize(_str)) - - -def mstr(obj): - if obj is None: - return "" - return unicode(obj) - - def buildindex(title, asf_license): """Write an index file for later conversion using mvn site""" versions = glob("[0-9]*.[0-9]*") @@ -233,16 +171,16 @@ class Jira(object): self.important = None def get_id(self): - return mstr(self.key) + return to_unicode(self.key) def get_description(self): - return mstr(self.fields['description']) + return to_unicode(self.fields['description']) def get_release_note(self): if self.notes is None: field = self.parent.field_id_map['Release Note'] if field in self.fields: - self.notes = mstr(self.fields[field]) + self.notes = to_unicode(self.fields[field]) elif self.get_incompatible_change() or self.get_important(): self.notes = self.get_description() else: @@ -254,14 +192,14 @@ class Jira(object): pri = self.fields['priority'] if pri is not None: ret = pri['name'] - return mstr(ret) + return to_unicode(ret) def get_assignee(self): ret = "" mid = self.fields['assignee'] if mid is not None: ret = mid['displayName'] - return mstr(ret) + return to_unicode(ret) def get_components(self): if len(self.fields['components']) > 0: @@ -278,21 +216,21 @@ class Jira(object): mid = self.fields['issuetype'] if mid is not None: ret = mid['name'] - return mstr(ret) + return to_unicode(ret) def get_reporter(self): ret = "" mid = self.fields['reporter'] if mid is not None: ret = mid['displayName'] - return mstr(ret) + return to_unicode(ret) def get_project(self): ret = "" mid = self.fields['project'] if mid is not None: ret = mid['key'] - return mstr(ret) + return to_unicode(ret) def __cmp__(self, other): result = 0 @@ -452,53 +390,6 @@ class JiraIter(object): return j -class Outputs(object): - """Several different files to output to at the same time""" - - def __init__(self, base_file_name, file_name_pattern, keys, params=None): - if params is None: - params = {} - self.params = params - self.base = open(base_file_name % params, 'w') - self.others = {} - for key in keys: - both = dict(params) - both['key'] = key - self.others[key] = open(file_name_pattern % both, 'w') - - def write_all(self, pattern): - both = dict(self.params) - both['key'] = '' - self.base.write(pattern % both) - for key in self.others.keys(): - both = dict(self.params) - both['key'] = key - self.others[key].write(pattern % both) - - def write_key_raw(self, key, _str): - self.base.write(_str) - if key in self.others: - self.others[key].write(_str) - - def close(self): - self.base.close() - for value in self.others.values(): - value.close() - - def write_list(self, mylist): - for jira in sorted(mylist): - line = '| [%s](' + JIRA_BASE_URL + '/browse/%s) ' +\ - '| %s | %s | %s | %s | %s |\n' - line = line % (textsanitize(jira.get_id()), - textsanitize(jira.get_id()), - textsanitize(jira.get_summary()), - textsanitize(jira.get_priority()), - format_components(jira.get_components()), - textsanitize(jira.get_reporter()), - textsanitize(jira.get_assignee())) - self.write_key_raw(jira.get_project(), line) - - class Linter(object): """Encapsulates lint-related functionality. Maintains running lint statistics about JIRAs.""" @@ -621,11 +512,11 @@ class Linter(object): if self._filters["incompatible"] and jira.get_incompatible_change(): self._warning_count += 1 self._lint_message += "\nWARNING: incompatible change %s lacks release notes." % \ - (textsanitize(jira.get_id())) + (text_sanitize(jira.get_id())) if self._filters["important"] and jira.get_important(): self._warning_count += 1 self._lint_message += "\nWARNING: important issue %s lacks release notes." % \ - (textsanitize(jira.get_id())) + (text_sanitize(jira.get_id())) if self._check_version_string(jira): self._warning_count += 1 @@ -901,10 +792,10 @@ def main(): else: otherlist.append(jira) - line = '* [%s](' % (textsanitize(jira.get_id())) + JIRA_BASE_URL + \ + line = '* [%s](' % (text_sanitize(jira.get_id())) + JIRA_BASE_URL + \ '/browse/%s) | *%s* | **%s**\n' \ - % (textsanitize(jira.get_id()), - textsanitize(jira.get_priority()), textsanitize(jira.get_summary())) + % (text_sanitize(jira.get_id()), + text_sanitize(jira.get_priority()), text_sanitize(jira.get_summary())) if len(jira.get_release_note()) > 0 or \ jira.get_incompatible_change() or jira.get_important(): http://git-wip-us.apache.org/repos/asf/yetus/blob/6f3aa7ae/release-doc-maker/utils.py ---------------------------------------------------------------------- diff --git a/release-doc-maker/utils.py b/release-doc-maker/utils.py new file mode 100644 index 0000000..0ef9290 --- /dev/null +++ b/release-doc-maker/utils.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +NAME_PATTERN = re.compile(r' \([0-9]+\)') +BASE_URL = "https://issues.apache.org/jira" + + +def clean(input_string): + return markdown_sanitize(re.sub(NAME_PATTERN, "", input_string)) + + +def format_components(input_string): + input_string = re.sub(NAME_PATTERN, '', input_string).replace("'", "") + if input_string != "": + ret = input_string + else: + # some markdown parsers don't like empty tables + ret = "." + return clean(ret) + + +# convert to utf-8 +def markdown_sanitize(input_string): + input_string = input_string.encode('utf-8') + input_string = input_string.replace("\r", "") + input_string = input_string.rstrip() + return input_string + + +# same thing as markdownsanitize, +# except markdown metachars are also +# escaped as well as more +# things we don't want doxia, etc, to +# screw up +def text_sanitize(input_string): + input_string = markdown_sanitize(input_string) + input_string = input_string.replace("_", r"\_") + input_string = input_string.replace("|", r"\|") + input_string = input_string.replace("<", r"\<") + input_string = input_string.replace(">", r"\>") + input_string = input_string.replace("*", r"\*") + input_string = input_string.rstrip() + return input_string + + +# if release notes have a special marker, +# we'll treat them as already in markdown format +def processrelnote(input_string): + relnote_pattern = re.compile('^\<\!\-\- ([a-z]+) \-\-\>') + fmt = relnote_pattern.match(input_string) + if fmt is None: + return text_sanitize(input_string) + else: + return { + 'markdown': markdown_sanitize(input_string), + }.get( + fmt.group(1), text_sanitize(input_string)) + + +def to_unicode(obj): + if obj is None: + return "" + return unicode(obj) + + +class Outputs(object): + """Several different files to output to at the same time""" + + def __init__(self, base_file_name, file_name_pattern, keys, params=None): + if params is None: + params = {} + self.params = params + self.base = open(base_file_name % params, 'w') + self.others = {} + for key in keys: + both = dict(params) + both['key'] = key + self.others[key] = open(file_name_pattern % both, 'w') + + def write_all(self, pattern): + both = dict(self.params) + both['key'] = '' + self.base.write(pattern % both) + for key in self.others: + both = dict(self.params) + both['key'] = key + self.others[key].write(pattern % both) + + def write_key_raw(self, key, input_string): + self.base.write(input_string) + if key in self.others: + self.others[key].write(input_string) + + def close(self): + self.base.close() + for value in self.others.values(): + value.close() + + def write_list(self, mylist): + for jira in sorted(mylist): + line = '| [%s](' + BASE_URL + '/browse/%s) ' +\ + '| %s | %s | %s | %s | %s |\n' + line = line % (text_sanitize(jira.get_id()), + text_sanitize(jira.get_id()), + text_sanitize(jira.get_summary()), + text_sanitize(jira.get_priority()), + format_components(jira.get_components()), + text_sanitize(jira.get_reporter()), + text_sanitize(jira.get_assignee())) + self.write_key_raw(jira.get_project(), line)