incubator-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject svn commit: r1197740 - /incubator/public/trunk/clutch.py
Date Fri, 04 Nov 2011 20:13:47 GMT
Author: sebb
Date: Fri Nov  4 20:13:47 2011
New Revision: 1197740

URL: http://svn.apache.org/viewvc?rev=1197740&view=rev
Log:
Use subroutines for external access and optionally log them
Simplify key access checks by using "if key in collection" rather than catching KeyError

Modified:
    incubator/public/trunk/clutch.py

Modified: incubator/public/trunk/clutch.py
URL: http://svn.apache.org/viewvc/incubator/public/trunk/clutch.py?rev=1197740&r1=1197739&r2=1197740&view=diff
==============================================================================
--- incubator/public/trunk/clutch.py (original)
+++ incubator/public/trunk/clutch.py Fri Nov  4 20:13:47 2011
@@ -119,6 +119,7 @@ parser = argparse.ArgumentParser(descrip
 parser.add_argument('--ignoreState',   action='store_true', default='False', help='Ignore
state (default false)')
 parser.add_argument('-v', '--verbose', action='store_true', default='False', help='verbose
mode (default false)')
 parser.add_argument('-q', '--quiet',   action='store_true', default='False', help='quiet
mode (default false)')
+parser.add_argument('-x', '--external', action='store_true', default='False', help='log external
requests (e.g. svn, http) (default false)')
 args = parser.parse_args()
 
 # Normal level of info
@@ -134,6 +135,21 @@ if optionVerbose:
 # So need to ignore the cached data and perform all resource availability tests.
 optionUseClutchState = args.ignoreState != True
 
+# Should we log external requests?
+optionExternal = args.external == True
+
+def logexternal(string):
+  if optionExternal:
+    print("External: " + string)
+
+def getUrl(url):
+  logexternal(url)
+  return urllib.request.urlopen(url)
+
+def osExec(list):
+  logexternal(" ".join(list))
+  return subprocess.Popen(list, stdout=subprocess.PIPE).communicate()[0]
+
 projects = {} # internal data, derived from ReportingSchedule
 otherIssues = []
 persist = {} # persistent data to be utilised by other tools
@@ -173,7 +189,7 @@ inputFile.close()
 
 print("Gather data from the ReportingSchedule ...") 
 # Parse the wiki ReportingSchedule to gather project details
-text = urllib.request.urlopen(REPORTING_SCHEDULE).read().decode('utf-8')
+text = getUrl(REPORTING_SCHEDULE).read().decode('utf-8')
 tokens = re.findall('\*.+', text)
 for token in tokens:
   token = token.strip() # strip whitespace
@@ -205,9 +221,7 @@ for token in tokens:
   id = name.lower()
   id = id.replace(' ', '') # strip spaces from project ID
 
-  try:
-    projects[id]
-  except KeyError:
+  if not id in projects:
     projects[id] = {}
     projects[id]['name'] = name
     # Set some defaults
@@ -310,9 +324,9 @@ for row in dom.getElementsByTagName("pod
   description = getText(row.getElementsByTagName("description"))
   mentors = [mentor.firstChild.data.strip() for mentor in row.getElementsByTagName("mentor")]
 
-  try:
-    projectsTable[id]
-  except KeyError:
+  if id in projectsTable:
+    print("ERROR: {0}: row exists".format(id))
+  else:
     projectsTable[id] = {}
     projectsTable[id]['name'] = name
     projectsTable[id]['description'] = description
@@ -328,8 +342,6 @@ for row in dom.getElementsByTagName("pod
       except KeyError:
         mentorsProjects[mentor] = []
       mentorsProjects[mentor].append(projectsTable[id]['name'])
-  else:
-    print("ERROR: {0}: row exists".format(id))
  history[id] = {}
  history[id]['name'] = name
  match = re.search(startDateRE, startDate)
@@ -365,9 +377,7 @@ print("Gather details from project statu
 projectNames = list(projectsTable.keys())
 for k in sorted(projectNames, key=str.lower):
   #print("Name: " + k)
-  try:
-    projects[k]
-  except KeyError:
+  if not k in projects:
     print("ERROR: {0}: Missing from reporting schedule".format(k))
     errorMsg = "{0}: Not listed in {1}, yet listed in {2}".format(k, linkReportingSchedule,
linkIncubationTable)
     errorMsg += ". See <a href=\"#h-Graduate\">help</a>."
@@ -495,7 +505,7 @@ class CommittersParser(HTMLParser):
         if self.rowCount > 1: # The first row is the column headers.
           self.projects[self.projectId].append(name)
 
-committersInput = urllib.request.urlopen(COMMITTERS_BY_PROJECT)
+committersInput = getUrl(COMMITTERS_BY_PROJECT)
 dataCommitters = committersInput.read().decode('utf-8')
 #print("dataCommitters=" + dataCommitters)
 committers = CommittersParser()
@@ -519,7 +529,7 @@ class IncubatorMailListNamesParser(HTMLP
           break
 
 mailLists = IncubatorMailListNamesParser()
-mailLists.feed(urllib.request.urlopen(MAIL_LIST_URL).read().decode('utf-8'))
+mailLists.feed(getUrl(MAIL_LIST_URL).read().decode('utf-8'))
 mailLists.close()
 #pprint.pprint(mailLists.names)
 del mailLists.names[0:5] # the first 5 are page navigation
@@ -563,7 +573,7 @@ if optionVerbose:
 print("Gather incubator PGP keys data ...")
 keysNamesRE = re.compile("/dist/incubator/([^/]+)/(.*)")
 keysList = {}
-lines = urllib.request.urlopen(INCUBATOR_KEYS).readlines()
+lines = getUrl(INCUBATOR_KEYS).readlines()
 for line in lines:
   matchKey = re.search(keysNamesRE, line.decode('utf-8'))
   if matchKey:
@@ -571,7 +581,7 @@ for line in lines:
 
 print("Gather data about releases ...")
 releases = {}
-lines = urllib.request.urlopen(INCUBATOR_RELEASES).readlines()
+lines = getUrl(INCUBATOR_RELEASES).readlines()
 for line in lines:
   match = re.search(releasesRE, line.decode('utf-8'))
   if match:
@@ -580,23 +590,19 @@ for line in lines:
     except KeyError:
       releases[match.group(1)] = True
 for k in releases:
-  try:
-    projects[k]
-  except KeyError:
-    try:
-      graduatedProjects[k]
-    except KeyError:
-      if optionInfo:
-        print("INFO: {0}: dormant/retired project has remains on Incubator mirrors".format(k))
-    else:
+  if not k in projects:
+    if k in graduatedProjects:
       errorMsg = "{0}: Has graduated, but still has remains on Incubator distribution mirrors".format(k)
       print('ERROR:', errorMsg)
       errorMsg += ". See <a href=\"#h-Graduate\">help</a>."
       otherIssues.append(errorMsg)
       continue
+    else:
+      if optionInfo:
+        print("INFO: {0}: dormant/retired project has remains on Incubator mirrors".format(k))
 
 releasesBadName = {}
-lines = urllib.request.urlopen(INCUBATOR_RELEASES_BAD_NAME).readlines()
+lines = getUrl(INCUBATOR_RELEASES_BAD_NAME).readlines()
 for line in lines:
   match = re.search(releasesBadNameRE, line.decode('utf-8'))
   if match:
@@ -616,9 +622,7 @@ print("Processing ...")
 projectNames = list(projects.keys())
 for k in sorted(projectNames, key=str.lower):
   print(k)
-  try:
-    projectsTable[k]
-  except KeyError:
+  if not k in projectsTable:
     print("ERROR: {0}: Missing from incubation table".format(k))
     projects[k]['hasStatusEntry'] = False
     continue
@@ -657,7 +661,7 @@ for k in sorted(projectNames, key=str.lo
   inputFile = "site-author/projects/{0}.xml".format(projects[k]['statusFileName'])
   if optionVerbose:
     print("DEBUG: Parsing svn log for {0} ...".format(inputFile))
-  outputString = subprocess.Popen(['svn', 'log', '--xml', inputFile], stdout=subprocess.PIPE).communicate()[0]
+  outputString = osExec(['svn', 'log', '--xml', inputFile])
   dom = xml.dom.minidom.parseString(outputString)
   rowCounter = 0
   count1 = 0
@@ -694,7 +698,7 @@ for k in sorted(projectNames, key=str.lo
 
 print("Collect SVN directory names")
 incubatorSvnDirs = {} # top-level SVN incubator dirs
-outputString = subprocess.Popen(['svn', 'ls', '--xml', 'http://svn.apache.org/repos/asf/incubator/'],
stdout=subprocess.PIPE).communicate()[0]
+outputString = osExec(['svn', 'ls', '--xml', 'http://svn.apache.org/repos/asf/incubator/'])
 dom = xml.dom.minidom.parseString(outputString)
 """
 Sample output
@@ -745,14 +749,12 @@ for k in sorted(projectNames, key=str.lo
   for svnGroup in svnGroups:
     if optionVerbose:
       print("DEBUG: Trying committers group '{0}'".format(svnGroup))
-    try:
-      committers.projects[svnGroup]
-    except KeyError:
-      continue
-    else:
+    if svnGroup in committers.projects:
       projects[k]['numberCommitters'] = len(committers.projects[svnGroup])
       projects[k]['committersSvn'] = svnGroup
       break
+    else:
+      continue
   if projects[k]['committersSvn'] == None and optionInfo:
     print("INFO: {0}: Does not yet have committers accounts".format(k))
 
@@ -781,7 +783,7 @@ for k in sorted(projectNames, key=str.lo
         incubatorSvnDirs[url] = name # mark used
         break
       try:
-        urllib.request.urlopen(url)
+        getUrl(url)
       except IOError:
         projects[k]['urlSvn'] = ''
       else:
@@ -805,7 +807,7 @@ for k in sorted(projectNames, key=str.lo
       if optionVerbose:
         print("DEBUG: Trying Tracker URL: " + url)
       try:
-        urllib.request.urlopen(url)
+        getUrl(url)
       except IOError:
         projects[k]['urlTracker'] = ""
       else:
@@ -827,7 +829,7 @@ for k in sorted(projectNames, key=str.lo
       if url == "":
         continue
       try:
-        urllib.request.urlopen(url)
+        getUrl(url)
       except IOError:
         projects[k]['urlWww'] = ""
       else:
@@ -846,7 +848,7 @@ for k in sorted(projectNames, key=str.lo
       if optionVerbose:
         print("DEBUG: Trying dist area " + urlDist)
       try:
-        urllib.request.urlopen(urlDist)
+        getUrl(urlDist)
       except IOError:
         pass
       else:
@@ -863,13 +865,11 @@ for k in sorted(projectNames, key=str.lo
     match = re.search("/incubator/([^/]+)/", projects[k]['urlDist'])
     if match:
       nameDistArea = match.group(1)
-      try:
-        keysList[nameDistArea]
-      except KeyError:
+      if nameDistArea in keysList:
+        projects[k]['urlKeys'] = keysList[nameDistArea]
+      else:
         if optionInfo:
           print("INFO: {0}: Does not yet have a PGP KEYS file".format(k))
-      else:
-        projects[k]['urlKeys'] = keysList[nameDistArea]
   if optionVerbose:
     print("DEBUG: KEYS=" + projects[k]['urlKeys'])
 
@@ -909,24 +909,20 @@ for k in sorted(projectNames, key=str.lo
             continue
           if optionVerbose:
             print("DEBUG: Trying listName=" + listName)
-          try:
-            projectMailLists[projectName]
-          except KeyError:
+          if not projectName in projectMailLists:
             if optionVerbose:
               print("DEBUG: {0}: No incubator group mail lists using '{1}'".format(k, projectName))
             break
-          try:
-            projectMailLists[projectName][listType]
-          except:
-            if optionInfo:
-              print("INFO: {0}: Does not yet have hinted incubator mail list '{1}-{2}'".format(k,
projectName, listType))
-            projects[k][mailListKey] = ""
-          else:
+          if listType in projectMailLists[projectName]:
             projects[k][mailListKey] = "http://mail-archives.apache.org/mod_mbox/incubator-{0}/".format(projectMailLists[projectName][listType])
             if optionVerbose:
               print("DEBUG: Successful Incubator mail url: " + projects[k][mailListKey])
             foundMailLists = True
             break
+          else:
+            if optionInfo:
+              print("INFO: {0}: Does not yet have hinted incubator mail list '{1}-{2}'".format(k,
projectName, listType))
+            projects[k][mailListKey] = ""
       # End of processing incubator group mail list.
       else:
         listName = projects[k][mailListHintKey]
@@ -934,7 +930,7 @@ for k in sorted(projectNames, key=str.lo
         if optionVerbose:
           print("DEBUG: Trying mail url: " + url)
         try:
-          urllib.request.urlopen(url)
+          getUrl(url)
         except IOError:
           projects[k][mailListKey] = ""
         else:
@@ -1136,12 +1132,10 @@ for k in sorted(projectNames, key=str.lo
   else:
     fileXml.write('          <td class="care">False</td>\n')
 
-  try:
-    releases[k]
-  except KeyError:
-    fileXml.write('          <td class="care">False</td>\n')
-  else:
+  if k in releases:
     fileXml.write('          <td class="cool1"><a href="{0}">True</a></td>\n'.format(projects[k]['urlDist']))
+  else:
+    fileXml.write('          <td class="care">False</td>\n')
 
   fileXml.write('        </tr>\n')
 



---------------------------------------------------------------------
To unsubscribe, e-mail: cvs-unsubscribe@incubator.apache.org
For additional commands, e-mail: cvs-help@incubator.apache.org


Mime
View raw message