From commits-return-1297-archive-asf-public=cust-asf.ponee.io@yetus.apache.org Mon Apr 8 17:54:20 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 4A1C6180627 for ; Mon, 8 Apr 2019 19:54:20 +0200 (CEST) Received: (qmail 44526 invoked by uid 500); 8 Apr 2019 17:34:42 -0000 Mailing-List: contact commits-help@yetus.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@yetus.apache.org Delivered-To: mailing list commits@yetus.apache.org Received: (qmail 44415 invoked by uid 99); 8 Apr 2019 17:34:42 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 08 Apr 2019 17:34:42 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 4BF7E80A47; Mon, 8 Apr 2019 17:54:19 +0000 (UTC) Date: Mon, 08 Apr 2019 17:54:19 +0000 To: "commits@yetus.apache.org" Subject: [yetus] branch master updated: YETUS-842. remove pylintrc MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <155474605920.23114.9320471391699025595@gitbox.apache.org> From: aw@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: yetus X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: ed583bb8c5b4ab3f15a7c3f6b283651c5664955a X-Git-Newrev: c78ec24c74d6443ee8de768b9e5c855d7001c812 X-Git-Rev: c78ec24c74d6443ee8de768b9e5c855d7001c812 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. aw pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/yetus.git The following commit(s) were added to refs/heads/master by this push: new c78ec24 YETUS-842. remove pylintrc c78ec24 is described below commit c78ec24c74d6443ee8de768b9e5c855d7001c812 Author: Allen Wittenauer AuthorDate: Thu Apr 4 10:06:31 2019 -0700 YETUS-842. remove pylintrc Signed-off-by: Allen Wittenauer --- .pylintrc | 305 --------------------- pom.xml | 1 - precommit/src/main/python/jenkins-admin.py | 167 +++++------ releasedocmaker/src/main/python/releasedocmaker.py | 2 + .../src/main/python/releasedocmaker/__init__.py | 217 ++++++++------- .../src/main/python/releasedocmaker/utils.py | 81 +++--- shelldocs/src/main/python/shelldocs.py | 2 + shelldocs/src/main/python/shelldocs/__init__.py | 12 +- 8 files changed, 257 insertions(+), 530 deletions(-) diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index b4cc79a..0000000 --- a/.pylintrc +++ /dev/null @@ -1,305 +0,0 @@ -[MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Profiled execution. -#profile=no - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Pickle collected data for later comparisons. -persistent=yes - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - - -[MESSAGES CONTROL] - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). -# CHANGED: -# C0103: Invalid name "" -# C0111: Missing docstring -# C0302: Too many lines in module (N) -# I0010: Unable to consider inline option '' -# I0011: Locally disabling WNNNN -# -# R0801: Similar lines in N files -# R0901: Too many ancestors (8/7) -# R0902: Too many instance attributes (N/7) -# R0903: Too few public methods (N/2) -# R0904: Too many public methods (N/20) -# R0911: Too many return statements (N/6) -# R0912: Too many branches (N/12) -# R0913: Too many arguments (N/5) -# R0914: Too many local variables (N/15) -# R0915: Too many statements (N/50) -# R0921: Abstract class not referenced -# R0922: Abstract class is only referenced 1 times -# W0122: Use of the exec statement -# W0141: Used builtin function '' -# W0142: Used * or ** magic -# W0402: Uses of a deprecated module 'string' -# W0404: 41: Reimport 'XX' (imported line NN) -# W0511: TODO -# W0603: Using the global statement -# W0703: Catch "Exception" -# W1201: Specify string format arguments as logging function parameters -# -# These should get enabled, but the codebase has too many violations currently. -# bad-continuation -# anomalous-backslash-in-string -# bad-context-manager -# bad-indentation -# bad-str-strip-call -# bad-whitespace -# cell-var-from-loop -# deprecated-lambda -# eval-used -# function-redefined -# import-error -# locally-enabled -# missing-final-newline -# no-init -# no-name-in-module -# no-self-use -# not-callable -# old-style-class -# protected-access -# superfluous-parens -# super-on-old-class -# too-many-function-args -# trailing-whitespace -# unnecessary-semicolon -# unpacking-non-sequence -# unused-import -# useless-else-on-loop -disable=C0103,C0111,C0302,I0010,I0011,R0801,R0901,R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0921,R0922,W0122,W0141,W0142,W0402,W0404,W0511,W0603,W0703,W1201,bad-continuation,anomalous-backslash-in-string,bad-context-manager,bad-indentation,bad-str-strip-call,bad-whitespace,cell-var-from-loop,deprecated-lambda,eval-used,function-redefined,import-error,locally-enabled,missing-final-newline,no-init,no-name-in-module,no-self-use,not-callable,old-style-class,protected-access,superfluou [...] - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html -output-format=text - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Tells whether to display a full report or only the messages -# CHANGED: -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Add a comment according to your evaluation note. This is used by the global -# evaluation report (RP0004). -#comment=no - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the beginning of the name of dummy variables -# (i.e. not used). -dummy-variables-rgx=_|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). -ignored-classes=SQLObject,twisted.internet.reactor,hashlib,google.appengine.api.memcache - -# When zope mode is activated, add a predefined set of Zope acquired attributes -# to generated-members. -#zope=no - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E0201 when accessed. Python regular -# expressions are accepted. -generated-members=REQUEST,acl_users,aq_parent,multiprocessing.managers.SyncManager - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -# CHANGED: -indent-string=' ' - - -[BASIC] - -# Required attributes for module, separated by a comma -#required-attributes= - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input - -# Regular expression which should only match correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression which should only match correct module level names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression which should only match correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression which should only match correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct instance attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match correct list comprehension / -# generator expression variable names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Regular expression which should only match functions or classes name which do -# not require a docstring -no-docstring-rgx=__.*__ - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branchs=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - - -[CLASSES] - -# List of interface methods to ignore, separated by a comma. This is used for -# instance to not check methods defines in Zope's Interface base class. -#ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,string,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/pom.xml b/pom.xml index 19f262f..9a6568f 100644 --- a/pom.xml +++ b/pom.xml @@ -216,7 +216,6 @@ false - .pylintrc .rubocop.yml Formula/.rubocop.yml diff --git a/precommit/src/main/python/jenkins-admin.py b/precommit/src/main/python/jenkins-admin.py index f6ffdfe..0786cf4 100755 --- a/precommit/src/main/python/jenkins-admin.py +++ b/precommit/src/main/python/jenkins-admin.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 +#pylint: disable=invalid-name # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -18,6 +19,8 @@ # under the License. # +""" Process patch file attachments from JIRA using a query """ + from optparse import OptionParser from tempfile import NamedTemporaryFile from xml.etree import ElementTree @@ -28,69 +31,68 @@ import re import sys import urllib2 -def httpGet(resource, ignoreError=False, username=None, password=None): +def http_get(resource, ignore_error=False, username=None, password=None): + """ get the contents of a URL """ request = urllib2.Request(resource) if username and password: - base64string = base64.b64encode('%s:%s' % (username, password)) - request.add_header("Authorization", "Basic %s" % base64string) + base64string = base64.b64encode('%s:%s' % (username, password)) + request.add_header("Authorization", "Basic %s" % base64string) try: response = urllib2.urlopen(request) except urllib2.HTTPError, http_err: code = http_err.code print '%s returns HTTP error %d: %s' \ % (resource, code, http_err.reason) - if ignoreError: + if ignore_error: return '' else: print 'Aborting.' sys.exit(1) except urllib2.URLError, url_err: print 'Error contacting %s: %s' % (resource, url_err.reason) - if ignoreError: + if ignore_error: return '' else: raise url_err except httplib.BadStatusLine, err: - if ignoreError: + if ignore_error: return '' else: raise err return response.read() - -# returns a map of (project, issue) => attachment id - -def parseJiraData(fileName): - tree = ElementTree.parse(fileName) +def parse_jira_data(filename): + """ returns a map of (project, issue) => attachment id """ + tree = ElementTree.parse(filename) root = tree.getroot() - jiraPattern = re.compile('([A-Z]+)\-([0-9]+)') + jirapattern = re.compile(r'([A-Z]+)\-([0-9]+)') result = {} for item in root.findall('./channel/item'): jirakey = item.find('key') if jirakey is None: continue jiraissue = jirakey.text - matcher = jiraPattern.match(jiraissue) + matcher = jirapattern.match(jiraissue) if not matcher: continue jiraissue = (matcher.group(1), matcher.group(2)) - attachmentIds = [] + attachmentids = [] for jiraattachment in item.findall('./attachments/attachment'): - attachmentId = jiraattachment.get('id') + attachmentid = jiraattachment.get('id') try: - attachmentIds.append(int(attachmentId)) + attachmentids.append(int(attachmentid)) except ValueError: pass - if attachmentIds: - attachmentIds.sort() - result[jiraissue] = attachmentIds[-1] + if attachmentids: + attachmentids.sort() + result[jiraissue] = attachmentids[-1] return result - -if __name__ == '__main__': - parser = OptionParser(prog = 'jenkins-admin') +def main(): #pylint: disable=too-many-branches, too-many-statements, too-many-locals + """ main program """ + parser = OptionParser(prog='jenkins-admin') if os.getenv('JENKINS_URL'): - parser.set_defaults(jenkinsUrl=os.getenv('JENKINS_URL')) + parser.set_defaults(jenkinsurl=os.getenv('JENKINS_URL')) if os.getenv('JOB_NAME'): parser.set_defaults(jenkinsJobName=os.getenv('JOB_NAME')) else: @@ -110,12 +112,12 @@ if __name__ == '__main__': parser.add_option('--jenkins-token', type='string', dest='jenkinsToken', help='Jenkins Token', metavar='TOKEN') - parser.add_option('--jenkins-url', type='string', dest='jenkinsUrl' + parser.add_option('--jenkins-url', type='string', dest='jenkinsurl' , help='Jenkins base URL', metavar='URL') parser.add_option( '--jenkins-url-override', type='string', - dest='jenkinsUrlOverrides', + dest='jenkinsurloverrides', action='append', help='Project specific Jenkins base URL', metavar='PROJECT=URL', @@ -138,108 +140,111 @@ if __name__ == '__main__': default=False, help="display version information for jenkins-admin and exit.") - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() #pylint: disable=unused-variable # Handle the version string right away and exit if options.release_version: with open( - os.path.join( - os.path.dirname(__file__), "../../VERSION"), 'r') as ver_file: + os.path.join( + os.path.dirname(__file__), "../../VERSION"), 'r') as ver_file: print ver_file.read() sys.exit(0) - tokenFrag = '' + token_frag = '' if options.jenkinsToken: - tokenFrag = 'token=%s' % options.jenkinsToken + token_frag = 'token=%s' % options.jenkinsToken else: - tokenFrag = 'token={project}-token' + token_frag = 'token={project}-token' if not options.jiraFilter: parser.error('ERROR: --jira-filter is a required argument.') - if not options.jenkinsUrl: + if not options.jenkinsurl: parser.error('ERROR: --jenkins-url or the JENKINS_URL environment variable is required.' - ) + ) if options.history < 0: parser.error('ERROR: --max-history must be 0 or a positive integer.' - ) - jenkinsUrlOverrides = {} - if options.jenkinsUrlOverrides: - for override in options.jenkinsUrlOverrides: + ) + jenkinsurloverrides = {} + if options.jenkinsurloverrides: + for override in options.jenkinsurloverrides: if '=' not in override: parser.error('Invalid Jenkins Url Override: ' + override) (project, url) = override.split('=', 1) - jenkinsUrlOverrides[project.upper()] = url - tempFile = NamedTemporaryFile(delete=False) + jenkinsurloverrides[project.upper()] = url + tempfile = NamedTemporaryFile(delete=False) try: - jobLogHistory = None + jobloghistory = None if not options.jenkinsInit: - jobLogHistory = httpGet(options.jenkinsUrl - + '/job/%s/lastSuccessfulBuild/artifact/patch_tested.txt' + jobloghistory = http_get(options.jenkinsurl + + '/job/%s/lastSuccessfulBuild/artifact/patch_tested.txt' % options.jenkinsJobName, True) # if we don't have a successful build available try the last build - if not jobLogHistory: - jobLogHistory = httpGet(options.jenkinsUrl - + '/job/%s/lastCompletedBuild/artifact/patch_tested.txt' - % options.jenkinsJobName) - jobLogHistory = jobLogHistory.strip().split('\n') - if 'TESTED ISSUES' not in jobLogHistory[0]: + if not jobloghistory: + jobloghistory = http_get(options.jenkinsurl + + '/job/%s/lastCompletedBuild/artifact/patch_tested.txt' + % options.jenkinsJobName) + jobloghistory = jobloghistory.strip().split('\n') + if 'TESTED ISSUES' not in jobloghistory[0]: print 'Downloaded patch_tested.txt control file may be corrupted. Failing.' sys.exit(1) # we are either going to write a new one or rewrite the old one - jobLog = open('patch_tested.txt', 'w+') + joblog = open('patch_tested.txt', 'w+') - if jobLogHistory: - if len(jobLogHistory) > options.history: - jobLogHistory = [jobLogHistory[0]] \ - + jobLogHistory[len(jobLogHistory) - - options.history:] - for jobHistoryRecord in jobLogHistory: - jobLog.write(jobHistoryRecord + '\n') + if jobloghistory: + if len(jobloghistory) > options.history: + jobloghistory = [jobloghistory[0]] \ + + jobloghistory[len(jobloghistory) \ + - options.history:] + for jobhistoryrecord in jobloghistory: + joblog.write(jobhistoryrecord + '\n') else: - jobLog.write('TESTED ISSUES\n') - jobLog.flush() - rssData = httpGet(options.jiraFilter,False,options.jiraUser,options.jiraPassword) - tempFile.write(rssData) - tempFile.flush() - for (key, attachment) in parseJiraData(tempFile.name).items(): + joblog.write('TESTED ISSUES\n') + joblog.flush() + rssdata = http_get(options.jiraFilter, False, options.jiraUser, options.jiraPassword) + tempfile.write(rssdata) + tempfile.flush() + for (key, attachment) in parse_jira_data(tempfile.name).items(): (project, issue) = key - if jenkinsUrlOverrides.has_key(project): - url = jenkinsUrlOverrides[project] + if jenkinsurloverrides.has_key(project): + url = jenkinsurloverrides[project] else: - url = options.jenkinsUrl + url = options.jenkinsurl - jenkinsUrlTemplate = url + '/job/' \ + jenkinsurltemplate = url + '/job/' \ + options.jenkinsJobTemplate \ - + '/buildWithParameters?' + tokenFrag \ + + '/buildWithParameters?' + token_frag \ + '&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}' - urlArgs = { + url_args = { 'project': project, 'issue': issue, 'attachment': attachment, } - jenkinsUrl = jenkinsUrlTemplate.format(**urlArgs) + jenkinsurl = jenkinsurltemplate.format(**url_args) # submit job - jobName = '%s-%s,%s' % (project, issue, attachment) - if not jobLogHistory or jobName not in jobLogHistory: - print jobName + ' has not been processed, submitting' - jobLog.write(jobName + '\n') - jobLog.flush() + jobname = '%s-%s,%s' % (project, issue, attachment) + if not jobloghistory or jobname not in jobloghistory: + print jobname + ' has not been processed, submitting' + joblog.write(jobname + '\n') + joblog.flush() if options.live: - httpGet(jenkinsUrl, True) + http_get(jenkinsurl, True) else: - print 'GET ' + jenkinsUrl + print 'GET ' + jenkinsurl else: - print jobName + ' has been processed, ignoring' - jobLog.close() + print jobname + ' has been processed, ignoring' + joblog.close() finally: if options.live: - os.remove(tempFile.name) + os.remove(tempfile.name) else: - print 'JIRA Data is located: ' + tempFile.name + print 'JIRA Data is located: ' + tempfile.name + +if __name__ == '__main__': + main() diff --git a/releasedocmaker/src/main/python/releasedocmaker.py b/releasedocmaker/src/main/python/releasedocmaker.py index 63c2e00..f4527c7 100755 --- a/releasedocmaker/src/main/python/releasedocmaker.py +++ b/releasedocmaker/src/main/python/releasedocmaker.py @@ -16,6 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" wrapper to launch releasedocmaker from the CLI """ + import sys sys.dont_write_bytecode = True # pylint: disable=wrong-import-position,import-self diff --git a/releasedocmaker/src/main/python/releasedocmaker/__init__.py b/releasedocmaker/src/main/python/releasedocmaker/__init__.py index 900f497..47f104c 100755 --- a/releasedocmaker/src/main/python/releasedocmaker/__init__.py +++ b/releasedocmaker/src/main/python/releasedocmaker/__init__.py @@ -1,4 +1,4 @@ -# +#!/usr/bin/env python2 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,6 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" Generate releasenotes based upon JIRA """ + +from __future__ import print_function import sys from glob import glob from optparse import OptionParser @@ -36,8 +39,8 @@ from utils import get_jira, to_unicode, sanitize_text, processrelnote, Outputs try: import dateutil.parser except ImportError: - print "This script requires python-dateutil module to be installed. " \ - "You can install it using:\n\t pip install python-dateutil" + print("This script requires python-dateutil module to be installed. " \ + "You can install it using:\n\t pip install python-dateutil") sys.exit(1) RELEASE_VERSION = {} @@ -100,7 +103,7 @@ def buildreadme(title, asf_license): version)) -class GetVersions(object): +class GetVersions(object): # pylint: disable=too-few-public-methods """ List of version strings """ def __init__(self, versions, projects): @@ -108,7 +111,7 @@ class GetVersions(object): projects = projects self.newversions = [] versions.sort(key=LooseVersion) - print "Looking for %s through %s" % (versions[0], versions[-1]) + print("Looking for %s through %s" % (versions[0], versions[-1])) newversions = set() for project in projects: url = JIRA_BASE_URL + \ @@ -129,10 +132,11 @@ class GetVersions(object): end_index = len(newlist) - 1 - newlist[::-1].index(versions[-1]) for newversion in newlist[start_index + 1:end_index]: if newversion in newversions: - print "Adding %s to the list" % newversion + print("Adding %s to the list" % newversion) self.newversions.append(newversion) def getlist(self): + """ Get the list of versions """ return self.newversions @@ -172,12 +176,15 @@ class Jira(object): self.important = None def get_id(self): + """ get the Issue ID """ return to_unicode(self.key) def get_description(self): + """ get the description """ return to_unicode(self.fields['description']) def get_release_note(self): + """ get the release note field """ if self.notes is None: field = self.parent.field_id_map['Release Note'] if field in self.fields: @@ -189,6 +196,7 @@ class Jira(object): return self.notes def get_priority(self): + """ Get the priority """ ret = "" pri = self.fields['priority'] if pri is not None: @@ -196,6 +204,7 @@ class Jira(object): return to_unicode(ret) def get_assignee(self): + """ Get the assignee """ ret = "" mid = self.fields['assignee'] if mid is not None: @@ -203,15 +212,18 @@ class Jira(object): return to_unicode(ret) def get_components(self): + """ Get the component(s) """ if self.fields['components']: return ", ".join([comp['name'] for comp in self.fields['components'] ]) return "" def get_summary(self): + """ Get the summary """ return self.fields['summary'] def get_type(self): + """ Get the Issue type """ ret = "" mid = self.fields['issuetype'] if mid is not None: @@ -219,6 +231,7 @@ class Jira(object): return to_unicode(ret) def get_reporter(self): + """ Get the issue reporter """ ret = "" mid = self.fields['reporter'] if mid is not None: @@ -226,6 +239,7 @@ class Jira(object): return to_unicode(ret) def get_project(self): + """ get the project """ ret = "" mid = self.fields['project'] if mid is not None: @@ -244,18 +258,19 @@ class Jira(object): result = cmp(int(selfsplit[1]), int(othersplit[1])) # dec is supported for backward compatibility if SORTORDER in ['dec', 'desc']: - result *= -1 + result *= -1 elif SORTTYPE == 'resolutiondate': dts = dateutil.parser.parse(self.fields['resolutiondate']) dto = dateutil.parser.parse(other.fields['resolutiondate']) result = cmp(dts, dto) if SORTORDER == 'newer': - result *= -1 + result *= -1 return result def get_incompatible_change(self): + """ get incompatible flag """ if self.incompat is None: field = self.parent.field_id_map['Hadoop Flags'] self.reviewed = False @@ -278,6 +293,7 @@ class Jira(object): return self.incompat def get_important(self): + """ get importat flag """ if self.important is None: field = self.parent.field_id_map['Flags'] self.important = False @@ -339,11 +355,11 @@ class JiraIter(object): print(err) fail_count += 1 if fail_count <= NUM_RETRIES: - print "Connection failed %d times. Retrying." % (fail_count) + print("Connection failed %d times. Retrying." % (fail_count)) sleep(1) return JiraIter.load_jira(params, fail_count) else: - print "Connection failed %d times. Aborting." % (fail_count) + print("Connection failed %d times. Aborting." % (fail_count)) sys.exit(1) @staticmethod @@ -356,7 +372,7 @@ class JiraIter(object): while pos < end: data = JiraIter.query_jira(ver, projects, pos) if 'error_messages' in data: - print "JIRA returns error message: %s" % data['error_messages'] + print("JIRA returns error message: %s" % data['error_messages']) sys.exit(1) pos = data['startAt'] + data['maxResults'] end = data['total'] @@ -382,6 +398,7 @@ class JiraIter(object): return self def next(self): + """ get next """ data = self.iter.next() j = Jira(data, self) return j @@ -438,25 +455,25 @@ class Linter(object): enabled = [] disabled = [] - for o in options.lint: - for token in o.split(","): + for opt in options.lint: + for token in opt.split(","): if token not in valid: - print "Unknown lint filter '%s', valid options are: %s" % \ - (token, ", ".join(v for v in sorted(valid))) + print("Unknown lint filter '%s', valid options are: %s" % \ + (token, ", ".join(v for v in sorted(valid)))) sys.exit(1) if token.startswith("-"): disabled.append(token[1:]) else: enabled.append(token) - for e in enabled: - if e == "all": - for f in self._valid_filters: - self._filters[f] = True + for eopt in enabled: + if eopt == "all": + for filt in self._valid_filters: + self._filters[filt] = True else: - self._filters[e] = True - for d in disabled: - self._filters[d] = False + self._filters[eopt] = True + for disopt in disabled: + self._filters[disopt] = False def had_errors(self): """Returns True if a lint error was encountered, else False.""" @@ -532,7 +549,7 @@ class Linter(object): % (" and ".join(error_message), jira.get_id()) -def parse_args(): +def parse_args(): # pylint: disable=too-many-branches """Parse command-line arguments with optparse.""" usage = "usage: %prog [OPTIONS] " + \ "--project PROJECT [--project PROJECT] " + \ @@ -663,9 +680,9 @@ def parse_args(): # Handle the version string right away and exit if options.release_version: with open( - os.path.join( - os.path.dirname(__file__), "../VERSION"), 'r') as ver_file: - print ver_file.read() + os.path.join( + os.path.dirname(__file__), "../VERSION"), 'r') as ver_file: + print(ver_file.read()) sys.exit(0) # Validate options @@ -686,13 +703,20 @@ def parse_args(): options.output_directory = options.output_directory[0] if options.range or len(options.versions) > 1: - if not options.versiondirs and not options.versionfiles: - parser.error("Multiple versions require either --fileversions or --dirversions") + if not options.versiondirs and not options.versionfiles: + parser.error("Multiple versions require either --fileversions or --dirversions") return options -def main(): +def main(): # pylint: disable=too-many-statements, too-many-branches, too-many-locals + """ hey, it's main """ + global JIRA_BASE_URL #pylint: disable=global-statement + global BACKWARD_INCOMPATIBLE_LABEL #pylint: disable=global-statement + global SORTTYPE #pylint: disable=global-statement + global SORTORDER #pylint: disable=global-statement + global NUM_RETRIES #pylint: disable=global-statement + options = parse_args() if options.output_directory is not None: @@ -705,17 +729,15 @@ def main(): options.output_directory): pass else: - print "Unable to create output directory %s: %u, %s" % \ - (options.output_directory, exc.errno, exc.message) + print("Unable to create output directory %s: %u, %s" % \ + (options.output_directory, exc.errno, exc.message)) sys.exit(1) os.chdir(options.output_directory) if options.base_url is not None: - global JIRA_BASE_URL JIRA_BASE_URL = options.base_url if options.incompatible_label is not None: - global BACKWARD_INCOMPATIBLE_LABEL BACKWARD_INCOMPATIBLE_LABEL = options.incompatible_label @@ -728,9 +750,7 @@ def main(): versions = [Version(v) for v in options.versions] versions.sort() - global SORTTYPE SORTTYPE = options.sorttype - global SORTORDER SORTORDER = options.sortorder if options.title is None: @@ -739,7 +759,6 @@ def main(): title = options.title if options.retries is not None: - global NUM_RETRIES NUM_RETRIES = options.retries[0] haderrors = False @@ -749,7 +768,7 @@ def main(): linter = Linter(vstr, options) jlist = sorted(JiraIter(vstr, projects)) if not jlist and not options.empty: - print "There is no issue which has the specified version: %s" % version + print("There is no issue which has the specified version: %s" % version) continue if vstr in RELEASE_VERSION: @@ -763,49 +782,49 @@ def main(): os.mkdir(vstr) if options.versionfiles and options.versiondirs: - reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md", - "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - choutputs = Outputs("%(ver)s/CHANGELOG.%(ver)s.md", - "%(ver)s/CHANGELOG.%(key)s.%(ver)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) + reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md", + "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + choutputs = Outputs("%(ver)s/CHANGELOG.%(ver)s.md", + "%(ver)s/CHANGELOG.%(key)s.%(ver)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) elif options.versiondirs: - reloutputs = Outputs("%(ver)s/RELEASENOTES.md", - "%(ver)s/RELEASENOTES.%(key)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - choutputs = Outputs("%(ver)s/CHANGELOG.md", - "%(ver)s/CHANGELOG.%(key)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) + reloutputs = Outputs("%(ver)s/RELEASENOTES.md", + "%(ver)s/RELEASENOTES.%(key)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + choutputs = Outputs("%(ver)s/CHANGELOG.md", + "%(ver)s/CHANGELOG.%(key)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) elif options.versionfiles: - reloutputs = Outputs("RELEASENOTES.%(ver)s.md", - "RELEASENOTES.%(key)s.%(ver)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - choutputs = Outputs("CHANGELOG.%(ver)s.md", - "CHANGELOG.%(key)s.%(ver)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) + reloutputs = Outputs("RELEASENOTES.%(ver)s.md", + "RELEASENOTES.%(key)s.%(ver)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + choutputs = Outputs("CHANGELOG.%(ver)s.md", + "CHANGELOG.%(key)s.%(ver)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) else: - reloutputs = Outputs("RELEASENOTES.md", - "RELEASENOTES.%(key)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) - choutputs = Outputs("CHANGELOG.md", - "CHANGELOG.%(key)s.md", [], - {"ver": version, - "date": reldate, - "title": title}) + reloutputs = Outputs("RELEASENOTES.md", + "RELEASENOTES.%(key)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) + choutputs = Outputs("CHANGELOG.md", + "CHANGELOG.%(key)s.md", [], + {"ver": version, + "date": reldate, + "title": title}) if options.license is True: reloutputs.write_all(ASF_LICENSE) @@ -870,7 +889,7 @@ def main(): linter.lint(jira) if linter.enabled: - print linter.message() + print(linter.message()) if linter.had_errors(): haderrors = True shutil.rmtree(vstr) @@ -880,60 +899,60 @@ def main(): reloutputs.close() if options.skip_credits: - CHANGEHDR1 = "| JIRA | Summary | Priority | " + \ + change_header21 = "| JIRA | Summary | Priority | " + \ "Component |\n" - CHANGEHDR2 = "|:---- |:---- | :--- |:---- |\n" + change_header22 = "|:---- |:---- | :--- |:---- |\n" else: - CHANGEHDR1 = "| JIRA | Summary | Priority | " + \ + change_header21 = "| JIRA | Summary | Priority | " + \ "Component | Reporter | Contributor |\n" - CHANGEHDR2 = "|:---- |:---- | :--- |:---- |:---- |:---- |\n" + change_header22 = "|:---- |:---- | :--- |:---- |:---- |:---- |\n" if incompatlist: choutputs.write_all("### INCOMPATIBLE CHANGES:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) + choutputs.write_all(change_header21) + choutputs.write_all(change_header22) choutputs.write_list(incompatlist, options.skip_credits, JIRA_BASE_URL) if importantlist: choutputs.write_all("\n\n### IMPORTANT ISSUES:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) + choutputs.write_all(change_header21) + choutputs.write_all(change_header22) choutputs.write_list(importantlist, options.skip_credits, JIRA_BASE_URL) if newfeaturelist: choutputs.write_all("\n\n### NEW FEATURES:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) + choutputs.write_all(change_header21) + choutputs.write_all(change_header22) choutputs.write_list(newfeaturelist, options.skip_credits, JIRA_BASE_URL) if improvementlist: choutputs.write_all("\n\n### IMPROVEMENTS:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) + choutputs.write_all(change_header21) + choutputs.write_all(change_header22) choutputs.write_list(improvementlist, options.skip_credits, JIRA_BASE_URL) if buglist: choutputs.write_all("\n\n### BUG FIXES:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) + choutputs.write_all(change_header21) + choutputs.write_all(change_header22) choutputs.write_list(buglist, options.skip_credits, JIRA_BASE_URL) if testlist: choutputs.write_all("\n\n### TESTS:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) + choutputs.write_all(change_header21) + choutputs.write_all(change_header22) choutputs.write_list(testlist, options.skip_credits, JIRA_BASE_URL) if subtasklist: choutputs.write_all("\n\n### SUB-TASKS:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) + choutputs.write_all(change_header21) + choutputs.write_all(change_header22) choutputs.write_list(subtasklist, options.skip_credits, JIRA_BASE_URL) if tasklist or otherlist: choutputs.write_all("\n\n### OTHER:\n\n") - choutputs.write_all(CHANGEHDR1) - choutputs.write_all(CHANGEHDR2) + choutputs.write_all(change_header21) + choutputs.write_all(change_header22) choutputs.write_list(otherlist, options.skip_credits, JIRA_BASE_URL) choutputs.write_list(tasklist, options.skip_credits, JIRA_BASE_URL) @@ -949,4 +968,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/releasedocmaker/src/main/python/releasedocmaker/utils.py b/releasedocmaker/src/main/python/releasedocmaker/utils.py index db957d3..ae57fd1 100644 --- a/releasedocmaker/src/main/python/releasedocmaker/utils.py +++ b/releasedocmaker/src/main/python/releasedocmaker/utils.py @@ -16,6 +16,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" Utility methods used by releasedocmaker """ + +from __future__ import print_function import base64 import os import re @@ -27,10 +30,6 @@ sys.dont_write_bytecode = True NAME_PATTERN = re.compile(r' \([0-9]+\)') -def clean(input_string): - return sanitize_markdown(re.sub(NAME_PATTERN, "", input_string)) - - def get_jira(jira_url): """ Provide standard method for fetching content from apache jira and handling of potential errors. Returns urllib2 response or @@ -48,21 +47,21 @@ def get_jira(jira_url): response = urllib2.urlopen(req) except urllib2.HTTPError as http_err: code = http_err.code - print "JIRA returns HTTP error %d: %s. Aborting." % \ - (code, http_err.msg) + print("JIRA returns HTTP error %d: %s. Aborting." % \ + (code, http_err.msg)) error_response = http_err.read() try: error_response = json.loads(error_response) - print "- Please ensure that specified authentication, projects,"\ - " fixVersions etc. are correct." + print("- Please ensure that specified authentication, projects,"\ + " fixVersions etc. are correct.") for message in error_response['errorMessages']: - print "-", message + print("-", message) except ValueError: - print "FATAL: Could not parse json response from server." + print("FATAL: Could not parse json response from server.") sys.exit(1) except urllib2.URLError as url_err: - print "Error contacting JIRA: %s\n" % jira_url - print "Reason: %s" % url_err.reason + print("Error contacting JIRA: %s\n" % jira_url) + print("Reason: %s" % url_err.reason) raise url_err except httplib.BadStatusLine as err: raise err @@ -70,40 +69,41 @@ def get_jira(jira_url): def format_components(input_string): + """ format the string """ input_string = re.sub(NAME_PATTERN, '', input_string).replace("'", "") if input_string != "": ret = input_string else: # some markdown parsers don't like empty tables ret = "." - return clean(ret) + return sanitize_markdown(re.sub(NAME_PATTERN, "", ret)) - -# Return the string encoded as UTF-8. -# -# This is necessary for handling markdown in Python. def encode_utf8(input_string): + """ Return the string encoded as UTF-8. + This is necessary for handling markdown in Python. """ return input_string.encode('utf-8') -# Sanitize Markdown input so it can be handled by Python. -# -# The expectation is that the input is already valid Markdown, -# so no additional escaping is required. def sanitize_markdown(input_string): + """ Sanitize Markdown input so it can be handled by Python. + + The expectation is that the input is already valid Markdown, + so no additional escaping is required. """ input_string = encode_utf8(input_string) input_string = input_string.replace("\r", "") input_string = input_string.rstrip() return input_string -# Sanitize arbitrary text so it can be embedded in MultiMarkdown output. -# -# Note that MultiMarkdown is not Markdown, and cannot be parsed as such. -# For instance, when using pandoc, invoke it as `pandoc -f markdown_mmd`. -# -# Calls sanitize_markdown at the end as a final pass. + def sanitize_text(input_string): + """ Sanitize arbitrary text so it can be embedded in MultiMarkdown output. + + Note that MultiMarkdown is not Markdown, and cannot be parsed as such. + For instance, when using pandoc, invoke it as `pandoc -f markdown_mmd`. + + Calls sanitize_markdown at the end as a final pass. + """ escapes = dict() # See: https://daringfireball.net/projects/markdown/syntax#backslash # We only escape a subset of special characters. We ignore characters @@ -113,26 +113,25 @@ def sanitize_text(input_string): slash_escapes += "\\" all_chars = set() # Construct a set of escapes - for c in slash_escapes: - all_chars.add(c) - for c in all_chars: - escapes[c] = "\\" + c + for char in slash_escapes: + all_chars.add(char) + for char in all_chars: + escapes[char] = "\\" + char # Build the output string character by character to prevent double escaping output_string = "" - for c in input_string: - o = c - if c in escapes: - o = escapes[c] - output_string += o + for char in input_string: + out = char + if char in escapes: + out = escapes[char] + output_string += out return sanitize_markdown(output_string.rstrip()) -# if release notes have a special marker, -# we'll treat them as already in markdown format def processrelnote(input_string): - relnote_pattern = re.compile('^\<\!\-\- ([a-z]+) \-\-\>') + """ if release notes have a special marker, we'll treat them as already in markdown format """ + relnote_pattern = re.compile(r'^\<\!\-\- ([a-z]+) \-\-\>') fmt = relnote_pattern.match(input_string) if fmt is None: return sanitize_text(input_string) @@ -142,6 +141,7 @@ def processrelnote(input_string): def to_unicode(obj): + """ convert string to unicode """ if obj is None: return "" return unicode(obj) @@ -162,6 +162,7 @@ class Outputs(object): self.others[key] = open(file_name_pattern % both, 'w') def write_all(self, pattern): + """ write everything given a pattern """ both = dict(self.params) both['key'] = '' self.base.write(pattern % both) @@ -171,11 +172,13 @@ class Outputs(object): self.others[key].write(pattern % both) def write_key_raw(self, key, input_string): + """ write everything withotu changes """ self.base.write(input_string) if key in self.others: self.others[key].write(input_string) def close(self): + """ close all the outputs """ self.base.close() for value in self.others.values(): value.close() diff --git a/shelldocs/src/main/python/shelldocs.py b/shelldocs/src/main/python/shelldocs.py index 57f912f..f1c28fa 100755 --- a/shelldocs/src/main/python/shelldocs.py +++ b/shelldocs/src/main/python/shelldocs.py @@ -16,6 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" wrapper to run shelldocs from the CLI """ + import sys sys.dont_write_bytecode = True # pylint: disable=wrong-import-position,import-self diff --git a/shelldocs/src/main/python/shelldocs/__init__.py b/shelldocs/src/main/python/shelldocs/__init__.py index 108f3b3..e6c16c2 100755 --- a/shelldocs/src/main/python/shelldocs/__init__.py +++ b/shelldocs/src/main/python/shelldocs/__init__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" process bash scripts and generate documentation from them """ + # Do this immediately to prevent compiled forms import sys import os @@ -67,7 +69,7 @@ def toc(tlist): return tocout -class ShellFunction(object): +class ShellFunction(object): # pylint: disable=too-many-public-methods, too-many-instance-attributes """a shell function""" def __init__(self, filename): @@ -304,7 +306,7 @@ def marked_as_ignored(file_path): return False -def main(): +def main(): # pylint: disable=too-many-statements, too-many-branches '''main entry point''' parser = OptionParser( usage="usage: %prog [--skipprnorep] " + "[--output OUTFILE|--lint] " + @@ -347,8 +349,8 @@ def main(): if options.release_version: with open( - os.path.join( - os.path.dirname(__file__), "../VERSION"), 'r') as ver_file: + os.path.join( + os.path.dirname(__file__), "../VERSION"), 'r') as ver_file: print ver_file.read() sys.exit(0)