incubator-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject svn commit: r1855462 - /incubator/public/trunk/clutch2.py
Date Thu, 14 Mar 2019 00:25:57 GMT
Author: wave
Date: Thu Mar 14 00:25:57 2019
New Revision: 1855462

URL: http://svn.apache.org/viewvc?rev=1855462&view=rev
Log:
Mentors now have apacheid and roster is deduplicated. Release processing is enhanced with
svn verbose listing and metadata dictionary. Stronger warning for missing KEYS. Hashes for
detached signatures are ignored. Added wiki processing. Error message on fixme sections are
now tuples. Reviewed FIXME comments. Removed ignore state.

Modified:
    incubator/public/trunk/clutch2.py

Modified: incubator/public/trunk/clutch2.py
URL: http://svn.apache.org/viewvc/incubator/public/trunk/clutch2.py?rev=1855462&r1=1855461&r2=1855462&view=diff
==============================================================================
--- incubator/public/trunk/clutch2.py (original)
+++ incubator/public/trunk/clutch2.py Thu Mar 14 00:25:57 2019
@@ -82,7 +82,7 @@ SITE_CONTENT/clutcht.ent
 SITE_CONTENT/clutchr1.ent
 SITE_CONTENT/clutchr2.ent
 SITE_CONTENT/clutcho2.ent
-SITE_CONTENT/clutchm.ent
+SITE_CONTENT/clutchm.txt
 SITE_CONTENT/clutchmy.ent
 SITE_CONTENT/report_due_1.txt
 SITE_CONTENT/report_due_2.txt
@@ -115,14 +115,26 @@ Pickle file:
 # FIXME: Detect if they have SVN repo yet. (won't fix) (as of 2019 only one podling is in
svn)
 #        - http://svn.apache.org/repos/asf/incubator/* ensure more than ".."
 # FIXME: Similarly with website. Ensure that there is some content length.
+#        Solution:
+#        (1) At program start update STATUS file with the value 'STARTED'
+#        (2) At completion update STATUS file with the value 'COMPLETE'
+#        (3) In the script that runs this program check the STATUS and output files to assure
+#            consistency. If that fails then abort the process.
 # FIXME: Get better hints from Status pages, e.g. sometimes they don't link
 #        to their "tracker" etc. they just use text.
+#        (We are now looking at the text and finding URLs.)
 # FIXME: News parser gets extra committer if source has commented xml template.
+#        (News is not a reliable source for new committers as most podlings aren't using
it)
 # FIXME: Use fragments via other files for the sets of html notes.
+#        (will be updating notes for clutch in newer asciidoc based incubator site)
 # FIXME: See some other suggestions on the general@ list.
+#        (not going to review 10 years of general@ to find these)
 # FIXME: See some other suggestions in clutch.html#notes-2
+#        (links to email thread are stale(404/523) - need to update historical note)
 # FIXME: Better deal with input/output/unicode.
+#        (much of the output calls for utf-8)
 # FIXME: See some other suggestions in issue INCUBATOR-78.
+#        (Most of these are acted upon by the creation of clutch.py and clutch2report.py)
 
 import sys
 if sys.version_info < (3, 2):
@@ -148,6 +160,7 @@ import json
 # infra moved to github
 # ASF_AUTH_TEMPLATE = 'https://raw.githubusercontent.com/apache/infrastructure-puppet/deployment/modules/subversion_server/files/authorization/asf-authorization-template'
 
+# Currently prefer these, but consider switch to lists.apache.org
 MAIL_LIST_URL = "http://mail-archives.apache.org/mod_mbox/"
 
 # All project git repositories in Apache are organized by Apache Infra and can be found on
Gitbox.
@@ -159,15 +172,14 @@ PROJECT_LDAP = "https://whimsy.apache.or
 # All podlings site scan results
 PODLING_SITE = "https://whimsy.apache.org/public/pods-scan.json"
 
-# Constant for site content location ---
-
+# Constant for site content location and clutch reports ---
 SITE_CONTENT_DIR = 'content/'
 CLUTCH_CONTENT_DIR = SITE_CONTENT_DIR + 'clutch/'
 
 parser = argparse.ArgumentParser(
     description='Gather details about projects currently in incubation.')
-parser.add_argument('--ignoreState',    action='store_true',
-                    default='True', help='Ignore state (default false)')
+#parser.add_argument('--ignoreState',    action='store_true',
+#                    default='True', help='Ignore state (default true)')
 parser.add_argument('-v', '--verbose',  action='store_true',
                     default='False', help='verbose mode (default false)')
 parser.add_argument('-q', '--quiet',    action='store_true',
@@ -189,6 +201,7 @@ if optionVerbose:
 # So need to ignore the cached data and perform all resource availability
 # tests.
 optionUseClutchState = False
+# stickiness is not needed during development and is bad in current build from scratch state.
 #args.ignoreState != True
 
 # Should we log external requests?
@@ -221,7 +234,6 @@ def osPopen(list):
     logexternal(" ".join(list))
     return subprocess.Popen(list, stdout=subprocess.PIPE, universal_newlines=True)
 
-
 def getText(nodelist):
     """http://www.python.org/doc/2.5.2/lib/minidom-example.txt"""
     rc = ""
@@ -230,7 +242,6 @@ def getText(nodelist):
             rc = rc + node.data
     return rc
 
-
 def normaliseSVNurl(url):
     rc = url.replace('https://', 'http://')
     if not rc[-1] == '/':
@@ -257,9 +268,11 @@ projects = {}  # internal data, derived
 otherIssues = []
 persist = {}  # persistent data to be utilised by other tools
 mentorsProjects = {}  # internal data
+mentorsName = {}  # internal data
 
 gatherDate = datetime.datetime.utcnow()
 gatherDateString = datetime.datetime.utcnow().ctime()
+gatherYear = gatherDate.year
 # 2 months
 delta = datetime.timedelta(days=61)
 statusTallyDate1 = gatherDate - delta
@@ -330,7 +343,7 @@ for row in dom.getElementsByTagName("pod
 
         if id in projects:
             print("ERROR: {0}: row exists".format(id))
-            errorMsg = "00|Duplicate entry in podlings.xml"
+            errorMsg = ["00","Duplicate entry in podlings.xml"]
             projects[id]['fixMeList'].append(errorMsg)
         else:
             projects[id] = {}
@@ -363,9 +376,11 @@ for row in dom.getElementsByTagName("pod
             except (Exception) as e:
                 print("Exception Gitbox repositories " + resource + " : " + str(e))
             projects[id]['urlTracker'] = ""
+            projects[id]['urlWiki'] = ""
             projects[id]['urlWww'] = ""
             projects[id]['urlDist'] = ""
             projects[id]['urlKeys'] = ""
+            projects[id]['releases'] = []
             projects[id]['distributions'] = []
             projects[id]['distribHash'] = []
             projects[id]['hasEntryIssues'] = False
@@ -389,7 +404,7 @@ for row in dom.getElementsByTagName("pod
                 projects[id]['numberPMCMembers'] = len(projectLDAP[resource]["owners"])
             except (Exception) as e:
                 print("Exception project LDAP " + resource + " : " + str(e))
-                projects[id]['fixMeList'].append("01|No project LDAP")
+                projects[id]['fixMeList'].append(["01","No project LDAP"])
                 projects[id]['numberCommitters'] = 0
                 projects[id]['numberPMCMembers'] = 0
             if projects[id]['numberCommitters'] == projects[id]['numberPMCMembers']:
@@ -406,7 +421,7 @@ for row in dom.getElementsByTagName("pod
             projects[id]['sponsor'] = row.getAttribute("sponsor")
             projects[id]['startDate'] = startDate
             projects[id]['statusFileName'] = row.getAttribute("resource")
-            # FIXME: Get availid from row.getAttribute
+
             mentors = [mentor.firstChild.data.strip()
                        for mentor in row.getElementsByTagName("mentor")]
             projects[id]['mentors'] = mentors
@@ -417,17 +432,23 @@ for row in dom.getElementsByTagName("pod
                 print('ERROR:', errorMsg)
                 errorMsg += " Please maintain your records in the content/podlings.xml file.
See <a href=\"#h-hasStatusEntry\">help</a>."
                 otherIssues.append(errorMsg)
-                errorMsg = "02|Project metadata missing in content/podlings.xml"
+                errorMsg = ["02","Podling metadata missing in content/podlings.xml file.
This is ground truth."]
                 projects[id]['fixMeList'].append(errorMsg)
-
-            # determine projects for each mentor
+            # accumulate projects for each mentor
+            mentors = [(mentor.firstChild.data.strip(),mentor.getAttribute("username"))
+                       for mentor in row.getElementsByTagName("mentor")]
+            projects[id]['mentorsApacheId'] = mentors
             for mentor in mentors:
                 try:
-                    mentorsProjects[mentor]
+                    mentorsProjects[mentor[1]]
                 except KeyError:
-                    mentorsProjects[mentor] = []
-                mentorsProjects[mentor].append(name)
-
+                    mentorsProjects[mentor[1]] = []
+                mentorsProjects[mentor[1]].append(name)
+                try:
+                    mentorsName[mentor[1]] = mentor[0]
+                except KeyError:
+                    dontcare = True
+            
             isGraduating = row.getElementsByTagName("graduating").length > 0
             if isGraduating:
                 graduatingOrRetiring.append(id)
@@ -436,7 +457,7 @@ for row in dom.getElementsByTagName("pod
                     print('ERROR:', errorMsg)
                     errorMsg += " See <a href=\"#h-Graduate\">help</a>."
                     otherIssues.append(errorMsg)
-                    errorMsg = "03|Graduated, but some graduation steps are incomplete"
+                    errorMsg = ["03","Graduated, but some graduation steps are incomplete"]
                     projects[id]['fixMeList'].append(errorMsg)
 
             isRetiring = row.getElementsByTagName("retiring").length > 0
@@ -447,7 +468,7 @@ for row in dom.getElementsByTagName("pod
                     print('ERROR:', errorMsg)
                     errorMsg += " See <a href=\"#h-Retire\">help</a>."
                     otherIssues.append(errorMsg)
-                    errorMsg = "04|Retired, but some retirement steps are incomplete"
+                    errorMsg = ["04","Retired, but some retirement steps are incomplete"]
                     projects[id]['fixMeList'].append(errorMsg)
 
             # Is it reporting monthly?
@@ -458,7 +479,7 @@ for row in dom.getElementsByTagName("pod
                     errorMsg = "ERROR: {0}: expecting a singleton report group".format(id)
                     print(errorMsg)
                     otherIssues.append(errorMsg)
-                    errorMsg = "05|Reporting group is either missing or a multiple"
+                    errorMsg = ["05","Reporting group is either missing or a multiple"]
                     projects[id]['fixMeList'].append(errorMsg)
             else:
                 if reporting[0].getAttribute("monthly") == 'true':
@@ -471,7 +492,7 @@ for row in dom.getElementsByTagName("pod
                     errorMsg =" ERROR: {0}: missing group attribute".format(id)
                     print(errorMsg)
                     otherIssues.append(errorMsg)
-                    errorMsg = "06|Reporting group is missing"
+                    errorMsg = ["06","Reporting group is missing"]
                     projects[id]['fixMeList'].append(errorMsg)
                 else:
                     projects[id]['reportingGroup'] = 'group-' + group
@@ -573,6 +594,11 @@ for k in sorted(projectNames, key=str.lo
                     if optionVerbose:
                         print("{0}: urlWww={1}".format(k,value))
                     continue
+                if cell.getAttribute('id') == "wiki" and hasUrl:
+                    projects[k]['urlWiki'] = value
+                    if optionVerbose:
+                        print("{0}: urlWiki={1}".format(k,value))
+                    continue
         # Scan the project News section and count new commiters.
         projects[k]['news'] = []        
         for section in dom.getElementsByTagName("section"):
@@ -681,13 +707,36 @@ print("Gather incubator PGP keys data an
 keysList = {}
 releases = {}
 releasesBadName = {}
+releasesListing = {}
 distributions = {}
 distribHash = {}
 distareas = {}  # podlings with dist areas
 
-with osPopen(['svn', 'ls', '-R', 'https://dist.apache.org/repos/dist/release/incubator'])
as s:
+with osPopen(['svn', 'ls', '-Rv', 'https://dist.apache.org/repos/dist/release/incubator'])
as s:
     for line in s.stdout:
-        line = line.rstrip()
+        line = line.strip()
+        if line[-1:] == '/':
+            # skip directories
+            continue
+        listing = line.split(' ')
+        revision = "r{0}".format(listing[0])
+        user = listing[1]
+        if listing[-6] == '':
+            dtm1 = datetime.datetime.strptime(" ".join(listing[-4:-2])+" "+str(gatherYear),"%b
%d %Y")
+            if dtm1 > gatherDate:
+                dtm1 = datetime.datetime.strptime(" ".join(listing[-4:-2])+" "+str(gatherYear-1),"%b
%d %Y")
+            fsize = listing[-5]
+        else:
+            dtm1 = datetime.datetime.strptime(" ".join(listing[-5:-1]),"%b %d %Y")
+            fsize = listing[-6]
+        dtm = dtm1.strftime("%m/%d/%Y")
+        line = listing[-1]
+        releasesListing[line] = {
+            'user': user,
+            'revision': revision,
+            'dtm': dtm,
+            'size': fsize
+            }
         fields = line.split('/')
         podling = fields[0]
         distareas[podling] = True
@@ -695,7 +744,7 @@ with osPopen(['svn', 'ls', '-R', 'https:
         if file:
             if re.search('KEYS(\.txt)?$', file):
                 keysList[podling] = "{0}/{1}".format("https://www.apache.org/dist/incubator",
line)
-            if re.search('(rpm|msi|bz2|gz|zip)\.(asc|sig)$', file, flags=re.IGNORECASE):
+            if re.search('\.(asc|sig)$', file, flags=re.IGNORECASE):
                 path = "/".join(fields[1:])
                 if optionVerbose:
                     print("DEBUG: {0} - {1}".format(podling,path))
@@ -709,16 +758,21 @@ with osPopen(['svn', 'ls', '-R', 'https:
                     releases[podling] = True
                 else:
                     releasesBadName[podling] = True
-                    if podling in projects:
+                    if podling in projects and podling != "netbeans":
+                        # netbeans has over 600 release parts and had legacy naming requirements
                         errorMsg = "INFO: {0}: Release is missing incubator/incubating in
file name {1}".format(podling,file)
                         print(errorMsg)
-                        errorMsg = "42|Distribution is missing incubator/incubating in file
name {0}".format(file)
+                        errorMsg = ["42","Distribution is missing incubator/incubating in
file name {0}".format(file)]
                         projects[podling]['fixMeList'].append(errorMsg)                 
  
-            if re.search('(rpm|msi|bz2|gz|zip)\.(sha512|sha1|sha256|sha|md5)$', file, flags=re.IGNORECASE):
+            if re.search('\.(sha512|sha1|sha256|sha|md5)$', file, flags=re.IGNORECASE):
                 path = "/".join(fields[1:])
                 if optionVerbose:
                     print("DEBUG: {0} - {1}".format(podling,path))
                 try:
+                    # some projects have done a hash on the detached signature. Identify
and ignore.
+                    part0 = ".".join(path.split('.')[-2:-1])
+                    if part0 == "asc":
+                        continue
                     # see if this is not the first hash for this release
                     # sha512 is preferred and it will be the last provided.
                     part1 = ".".join(distribHash[podling][-1].split('.')[:-1])
@@ -738,11 +792,11 @@ for k in releases:
             errorMsg = "{0}: Has graduated, but still has remains on Incubator distribution
mirrors".format(
                 k)
             print('ERROR:', errorMsg)
-            errorMsg += ". See <a href=\"#h-Graduate\">help</a>."
             otherIssues.append(errorMsg)
             continue
         if k in retiredProjects:
             print("INFO: {0}: retired project has remains on Incubator mirrors".format(k))
+            otherIssues.append(errorMsg)
 
 for k in releasesBadName:
     errorMsg = '{0}: Has a distribution filename missing the word "incubating/incubator"'.format(k)
@@ -772,7 +826,7 @@ for k in sorted(projectNames, key=str.lo
         errorMsg += ". See <a href=\"#h-hasStatusEntry\">help</a>."
         otherIssues.append(errorMsg)
         projects[k]['hasStatusEntry'] = False
-        errorMsg = "07|Status file is missing"
+        errorMsg = ["07","Status file is missing"]
         projects[k]['fixMeList'].append(errorMsg)
         continue
 
@@ -789,7 +843,7 @@ for k in sorted(projectNames, key=str.lo
         except ValueError:
             errorMsg = "ERROR: {0}: ValueError with date".format(k)
             print(errorMsg)
-            errorMsg = "08|Start date is not a valid date"
+            errorMsg = ["08","Start date is not a valid date"]
             projects[k]['fixMeList'].append(errorMsg)
         else:
             projects[k]['entryDate'] = entryDate
@@ -910,9 +964,30 @@ for k in sorted(projectNames, key=str.lo
         except IOError:
             projects[k]['urlTracker'] = ""
     if not projects[k]['urlTracker']:
-        errorMsg = "INFO: {0}: Does not yet have an Issue Tracker".format(k)
+        errorMsg = "INFO: {0}: Does not yet have a valid Issue Tracker".format(k)
         print(errorMsg)
-        errorMsg = "10|Project is missing its link to an Issue Tracker"
+        errorMsg = ["10","Has no valid issue tracker link"]
+        projects[k]['fixMeList'].append(errorMsg)
+
+    # Detect if they have Wiki yet.
+    # First, try the url from their status page
+    # then, possibly, try a standard url.
+    urlWikiDefault = "" # no default
+    for url in [projects[k]['urlWiki'], urlWikiDefault]:
+        if url == "":
+            continue
+        if optionVerbose:
+            print("DEBUG: Trying Wiki URL: " + url)
+        try:
+            getUrl(url)
+            projects[k]['urlWiki'] = url
+            break
+        except IOError:
+            projects[k]['urlWiki'] = ""
+    if not projects[k]['urlWiki']:
+        errorMsg = "INFO: {0}: Does not yet have a Wiki".format(k)
+        print(errorMsg)
+        errorMsg = ["18","Has no valid wiki link"]
         projects[k]['fixMeList'].append(errorMsg)
 
     # Detect if they have a website yet.
@@ -938,7 +1013,7 @@ for k in sorted(projectNames, key=str.lo
     if not projects[k]['urlWww']:
         errorMsg = "INFO: {0}: Does not yet have a website".format(k)
         print(errorMsg)
-        errorMsg = "11|Website is missing"
+        errorMsg = ["11","Website is missing"]
         projects[k]['fixMeList'].append(errorMsg)
         
     if projects[k]['urlWww']:
@@ -949,42 +1024,42 @@ for k in sorted(projectNames, key=str.lo
                 if not podlingSiteScan[j]['foundation']:
                     errorMsg = "INFO: {0}: Does not yet have foundation link".format(k)
                     print(errorMsg)
-                    errorMsg = "31|ASF Foundation link is missing"
+                    errorMsg = ["31","ASF Foundation link is missing"]
                     projects[k]['fixMeList'].append(errorMsg)
                 if not podlingSiteScan[j]['events']:
                     errorMsg = "INFO: {0}: Does not yet have events link".format(k)
                     print(errorMsg)
-                    errorMsg = "32|ASF Events link is missing"
+                    errorMsg = ["32","ASF Events link is missing"]
                     projects[k]['fixMeList'].append(errorMsg)
                 if not podlingSiteScan[j]['license']:
                     errorMsg = "INFO: {0}: Does not yet have license link".format(k)
                     print(errorMsg)
-                    errorMsg = "33|Apache License link is missing"
+                    errorMsg = ["33","Apache License link is missing"]
                     projects[k]['fixMeList'].append(errorMsg)
                 if not podlingSiteScan[j]['thanks']:
                     errorMsg = "INFO: {0}: Does not yet have thanks link".format(k)
                     print(errorMsg)
-                    errorMsg = "34|Apache Thanks link is missing"
+                    errorMsg = ["34","Apache Thanks link is missing"]
                     projects[k]['fixMeList'].append(errorMsg)
                 if not podlingSiteScan[j]['security']:
                     errorMsg = "INFO: {0}: Does not yet have security link".format(k)
                     print(errorMsg)
-                    errorMsg = "35|Apache Security link is missing"
+                    errorMsg = ["35","Apache Security link is missing"]
                     projects[k]['fixMeList'].append(errorMsg)
                 if not podlingSiteScan[j]['sponsorship']:
                     errorMsg = "INFO: {0}: Does not yet have sponsorship link".format(k)
                     print(errorMsg)
-                    errorMsg = "36|Apache Sponsorship link is missing"
+                    errorMsg = ["36","Apache Sponsorship link is missing"]
                     projects[k]['fixMeList'].append(errorMsg)
                 if not podlingSiteScan[j]['trademarks']:
                     errorMsg = "INFO: {0}: Does not yet have trademarks".format(k)
                     print(errorMsg)
-                    errorMsg = "37|Trademarks are missing"
+                    errorMsg = ["37","Trademarks are missing"]
                     projects[k]['fixMeList'].append(errorMsg)
                 if not podlingSiteScan[j]['copyright']:
                     errorMsg = "INFO: {0}: Does not yet have copyright".format(k)
                     print(errorMsg)
-                    errorMsg = "38|Copyright is missing"
+                    errorMsg = ["38","Copyright is missing"]
                     projects[k]['fixMeList'].append(errorMsg)
                 flag = False
                 try:
@@ -995,7 +1070,7 @@ for k in sorted(projectNames, key=str.lo
                 if flag:
                     errorMsg = "INFO: {0}: Does not yet have disclaimer".format(k)
                     print(errorMsg)
-                    errorMsg = "39|Disclaimer is missing"
+                    errorMsg = ["39","Disclaimer is missing"]
                     projects[k]['fixMeList'].append(errorMsg)
             else:
                 print('missing from site scan')
@@ -1015,7 +1090,7 @@ for k in sorted(projectNames, key=str.lo
     if not projects[k]['urlDist']:
         errorMsg = "INFO: {0}: Does not yet have a distribution area".format(k)
         print(errorMsg)
-        errorMsg = "12|Release distribution area is not setup"
+        errorMsg = ["12","Release distribution area is not setup"]
         projects[k]['fixMeList'].append(errorMsg)
 
     elif optionVerbose:
@@ -1033,42 +1108,74 @@ for k in sorted(projectNames, key=str.lo
                 except KeyError:
                     errorMsg = "INFO: {0}: Has a PGP KEYS file, but no signed distribution".format(k)
                     print(errorMsg)
-                    errorMsg = "41|PGP KEYS file available without a distribution"
+                    errorMsg = ["41","PGP KEYS file available without a distribution"]
                     projects[k]['fixMeList'].append(errorMsg)                    
                 try:
                     projects[k]['distribHash'] = distribHash[nameDistArea]
                 except KeyError:
-                    errorMsg = "INFO: {0}: Has a signed distribution without a hash".format(k)
-                    print(errorMsg)
-                    errorMsg = "43|Signed distribution without a hash"
-                    projects[k]['fixMeList'].append(errorMsg)
+                    dontcare = True
                 try:
                     l1 = len(projects[k]['distributions'])
                     l2 = len(projects[k]['distribHash'])
                     ld = l1-l2
-                    if l1 > 9:
+                    if l1 > 8:
                         errorMsg = "INFO: {0}: Has {1} distributions. Non-current distributions
must be removed".format(k,l1)
                         print(errorMsg)
-                        errorMsg = "45|{0} is a large number of releases. Non-current release
must be removed".format(l1)
+                        errorMsg = ["45","{0} is a large number of releases. Non-current
release must be removed".format(l1)]
                         projects[k]['fixMeList'].append(errorMsg)
                     if ld > 0:
                         errorMsg = "INFO: {0}: Has {1} signed distribution without a hash".format(k,ld)
                         print(errorMsg)
-                        errorMsg = "43|Signed distribution without a hash = {0}".format(ld)
+                        errorMsg = ["43","Signed distribution without a hash = {0}".format(ld)]
                         projects[k]['fixMeList'].append(errorMsg)
+                        pprint.pprint(projects[k]['distributions'])
+                        pprint.pprint(projects[k]['distribHash'])
                     elif ld < 0:
                         ld = l2-l1
                         errorMsg = "INFO: {0}: Has {1} unsigned distributions".format(k,ld)
                         print(errorMsg)
-                        errorMsg = "44|Unsigned distributions = {0}".format(ld)
+                        errorMsg = ["44","Unsigned distributions = {0}".format(ld)]
                         projects[k]['fixMeList'].append(errorMsg)
+                        pprint.pprint(projects[k]['distributions'])
+                        pprint.pprint(projects[k]['distribHash'])
+                    else:
+                        # create a dictionary of podling releases.
+                        url1 = projects[k]['urlKeys']
+                        url1 = "/".join(url1.split("/")[:-1])+"/"
+                        url2 = projects[k]['urlDist']
+                        releasesDist = {}
+                        for i in range(l1):
+                            l = projects[k]['distributions'][i]
+                            kk = projects[k]['distribHash'][i]
+                            r = ".".join(l.split(".")[:-1])
+                            rr = "".join(r.split("/")[-1:])
+                            rMeta = releasesListing[k+"/"+r]
+                            release = {
+                                'download': "{0}{1}".format(url2,r),
+                                'signature': "{0}{1}".format(url1,l),
+                                'hash': "{0}{1}".format(url1,kk),
+                                'user': rMeta['user'],
+                                'revision': rMeta['revision'],
+                                'dtm': rMeta['dtm'],
+                                'size': rMeta['size']
+                                }
+                            releasesDist[rr] = release
+                        projects[k]['releases'] = releasesDist
                 except:
-                    print("failed to process distribution to hashes comparison")
+                    dontcare = True
             else:
                 errorMsg = "INFO: {0}: Does not yet have a PGP KEYS file".format(k)
                 print(errorMsg)
-                errorMsg = "13|Release signing PGP KEYS file is missing"
+                errorMsg = ["13","Release signing PGP KEYS file is missing"]
                 projects[k]['fixMeList'].append(errorMsg)
+                try:
+                    if len(distributions[nameDistArea]) > 0 or len(distribHash[nameDistArea])
> 0:
+                        errorMsg = "INFO: {0}: Apache Releases cannot be validated without
a KEYS file. Add required KEYS!".format(k)
+                        print(errorMsg)
+                        errorMsg = ["46","Apache Releases cannot be validated without a KEYS
file. Add required KEYS!"]
+                        projects[k]['fixMeList'].append(errorMsg)
+                except:
+                    dontcare = True
 
     if optionVerbose:
         print("DEBUG: KEYS=" + projects[k]['urlKeys'])
@@ -1127,7 +1234,7 @@ for k in sorted(projectNames, key=str.lo
                     else:
                         errorMsg = "INFO: {0}: Does not yet have hinted incubator mail list
'{1}-{2}'".format(k, projectName, listType)
                         print(errorMsg)
-                        errorMsg = "14|Expected mailing list (incubator-{0}-{1}) is missing".format(projectName,
listType)
+                        errorMsg = ["14","Expected mailing list (incubator-{0}-{1}) is missing".format(projectName,
listType)]
                         projects[k]['fixMeList'].append(errorMsg)
                         projects[k][mailListKey] = ""
             # End of processing incubator group mail list.
@@ -1152,12 +1259,12 @@ for k in sorted(projectNames, key=str.lo
     if not projects[k]['hasMailListDev']:
         errorMsg = "INFO: {0}: Does not yet have 'dev' mail list".format(k)
         print(errorMsg)
-        errorMsg = "15|Project does not have 'dev' mailing list"
+        errorMsg = ["15","Project does not have 'dev' mailing list"]
         projects[k]['fixMeList'].append(errorMsg)
     if not projects[k]['hasMailListCommits']:
         errorMsg = "INFO: {0}: Does not yet have 'commits' mail list".format(k)
         print(errorMsg)
-        errorMsg = "16|Project does not have 'commits' mailing list"
+        errorMsg = ["16","Project does not have 'commits' mailing list"]
         projects[k]['fixMeList'].append(errorMsg)
 
 # end of processing each podling to detect resource availability
@@ -1243,6 +1350,7 @@ for k in sorted(projectNames, key=str.lo
     persist[k]['sponsor'] = projects[k]['sponsor']
     persist[k]['description'] = projects[k]['description']
     persist[k]['mentors'] = projects[k]['mentors']
+    persist[k]['mentorsApacheId'] = projects[k]['mentorsApacheId']
 
     fileXml.write('          <td>{0}</td>\n'.format(projects[k]['startDate']))
     persist[k]['startDate'] = projects[k]['startDate']
@@ -1364,12 +1472,12 @@ for k in sorted(projectNames, key=str.lo
     if not projects[k]['urlGit'] and not projects[k]['urlSvn']:
         errorMsg = "INFO: {0}: Does not yet have a source code repository".format(k)
         print(errorMsg)
-        errorMsg = "17|No source code repository"
+        errorMsg = ["17","No source code repository"]
         projects[k]['fixMeList'].append(errorMsg)
     if len(projects[k]['urlGit']) > 0 and len(projects[k]['gitbox']) == 0:
         errorMsg = "INFO: {0}: Does not yet have a source code repository".format(k)
         print(errorMsg)
-        errorMsg = "17|No source code repository. Declared but nonexistent"
+        errorMsg = ["17","No source code repository. Declared but nonexistent"]
         projects[k]['fixMeList'].append(errorMsg)
 
     if projects[k]['urlTracker']:
@@ -1378,6 +1486,7 @@ for k in sorted(projectNames, key=str.lo
     else:
         fileXml.write('          <td class="care">False</td>\n')
     persist[k]['urlTracker'] = projects[k]['urlTracker']
+    persist[k]['urlWiki'] = projects[k]['urlWiki']
 
     hasUrl = re.search(urlHttpRE, projects[k]['hasMailListDev'])
     if hasUrl:
@@ -1415,8 +1524,7 @@ for k in sorted(projectNames, key=str.lo
     else:
         fileXml.write('          <td class="care">False</td>\n')
     persist[k]['urlKeys'] = projects[k]['urlKeys']
-    persist[k]['distributions'] = projects[k]['distributions']
-    persist[k]['distribHash'] = projects[k]['distribHash']
+    persist[k]['releases'] = projects[k]['releases']
     persist[k]['fixMeList'] = projects[k]['fixMeList']
     persist[k]['news'] = projects[k]['news']
 
@@ -1449,14 +1557,18 @@ else:
     fileXmlo2.write("        <li>No known issues.</li>\n")
 fileXmlo2.close()
 
-
-mentors = list(mentorsProjects.keys())
+mentorsList = {}
+for mentor in mentorsName:
+    mentorsList[mentorsName[mentor]] = [mentorsName[mentor],mentor,mentorsProjects[mentor]]
+mentors = list(mentorsList.keys())
 mentors.sort()
-fileXmlm = open(CLUTCH_CONTENT_DIR + 'clutchm.ent', encoding='utf-8', mode='w')
-fileXmlm.write("<!-- generated by clutch; do not edit -->\n")
+
+fileXmlm = open(CLUTCH_CONTENT_DIR + 'clutchm.txt', encoding='utf-8', mode='w')
+fileXmlm.write("=== Mentor Roster\n")
+fileXmlm.write("\n")
 for mentor in mentors:
-    fileXmlm.write("        <li><strong>{0}</strong>: {1}</li>\n".format(
-        mentor, ', '.join(mentorsProjects[mentor])))
+    fileXmlm.write("# {1}: {0}: {2}\n".format(
+        mentor, mentorsList[mentor][1], ', '.join(mentorsList[mentor][2])))
 fileXmlm.close()
 
 fileList.close()



---------------------------------------------------------------------
To unsubscribe, e-mail: cvs-unsubscribe@incubator.apache.org
For additional commands, e-mail: cvs-help@incubator.apache.org


Mime
View raw message