incubator-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject svn commit: r1717128 - /incubator/public/trunk/clutch.py
Date Sun, 29 Nov 2015 22:38:15 GMT
Author: sebb
Date: Sun Nov 29 22:38:15 2015
New Revision: 1717128

URL: http://svn.apache.org/viewvc?rev=1717128&view=rev
Log:
Use svn ls to get list of files under dist/incubator
No longer relies on cronjob running on minotaur

Modified:
    incubator/public/trunk/clutch.py

Modified: incubator/public/trunk/clutch.py
URL: http://svn.apache.org/viewvc/incubator/public/trunk/clutch.py?rev=1717128&r1=1717127&r2=1717128&view=diff
==============================================================================
--- incubator/public/trunk/clutch.py (original)
+++ incubator/public/trunk/clutch.py Sun Nov 29 22:38:15 2015
@@ -62,12 +62,16 @@ find /www/www.apache.org/dist/incubator
   | grep -v incubat \
   > ~/public_html/incubator-releases-bad-filename.txt
 
+The above has now been replaced by parsing the output of
+'svn', 'ls', '-R', 'https://dist.apache.org/repos/dist/release/incubator'
+
 http://people.apache.org/committers-by-project.html
 http://mail-archives.apache.org/mod_mbox/
 http://www.apache.org/dist/incubator/<resource>
 http://svn.apache.org/repos/asf/incubator
 
-SVN commands
+SVN commands 
+'svn', 'ls', '-R', 'https://dist.apache.org/repos/dist/release/incubator'
 'svn', 'ls', '--xml', 'http://svn.apache.org/repos/asf/incubator/'
 'svn', 'log', '--xml', 'SITE_CONTENT/projects/{0}.xml' {status file}
 
@@ -122,6 +126,7 @@ if sys.version_info < (3, 2):
     raise Exception("Python 3.2 or above is required")
 
 import subprocess
+from subprocess import Popen, PIPE
 import datetime
 from html.parser import HTMLParser
 import os.path
@@ -136,9 +141,6 @@ import argparse
 
 COMMITTERS_BY_PROJECT = "http://people.apache.org/committers-by-project.html"
 MAIL_LIST_URL = "http://mail-archives.apache.org/mod_mbox/"
-INCUBATOR_KEYS = 'http://people.apache.org/~crossley/incubator-keys.txt'
-INCUBATOR_RELEASES = 'http://people.apache.org/~crossley/incubator-releases.txt'
-INCUBATOR_RELEASES_BAD_NAME = 'http://people.apache.org/~crossley/incubator-releases-bad-filename.txt'
 
 # Constant for site content location ---
 
@@ -583,27 +585,27 @@ if optionVerbose:
 
 # Gather incubator PGP keys data ---
 
-print("Gather incubator PGP keys data ...")
-keysNamesRE = re.compile("/dist/incubator/([^/]+)/(.*)")
+print("Gather incubator PGP keys data and releases ...")
+
 keysList = {}
-lines = getUrl(INCUBATOR_KEYS).readlines()
-for line in lines:
-  matchKey = re.search(keysNamesRE, line.decode('utf-8'))
-  if matchKey:
-    keysList[matchKey.group(1)] = "{0}/{1}/{2}".format("http://www.apache.org/dist/incubator",
matchKey.group(1), matchKey.group(2))
+releases = {}
+releasesBadName = {}
 
-# Gather data about releases ---
+with Popen(['svn', 'ls', '-R', 'https://dist.apache.org/repos/dist/release/incubator'], stdout=PIPE,
universal_newlines=True) as s:
+    for line in s.stdout:
+        line = line.rstrip()
+        fields = line.split('/')
+        podling = fields[0]
+        file = fields[-1]
+        if file:
+            if re.search('KEYS(\.txt)?$', file):
+                keysList[podling] = "{0}/{1}".format("http://www.apache.org/dist/incubator",
line)
+            if re.search('(bz2|gz|zip)\.(asc|sig)$', file, flags=re.IGNORECASE):
+                if re.search('incubat(ing|or)', file, flags=re.IGNORECASE):
+                    releases[podling] = True
+                else:
+                    releasesBadName[podling] = True
 
-print("Gather data about releases ...")
-releases = {}
-lines = getUrl(INCUBATOR_RELEASES).readlines()
-for line in lines:
-  match = re.search(releasesRE, line.decode('utf-8'))
-  if match:
-    try:
-      releases[match.group(1)]
-    except KeyError:
-      releases[match.group(1)] = True
 for k in releases:
   #FIXME: need to handle projects[k]['resourceNames']
   if not k in projects:
@@ -616,17 +618,8 @@ for k in releases:
     if k in retiredProjects:
         print("INFO: {0}: retired project has remains on Incubator mirrors".format(k))
 
-releasesBadName = {}
-lines = getUrl(INCUBATOR_RELEASES_BAD_NAME).readlines()
-for line in lines:
-  match = re.search(releasesBadNameRE, line.decode('utf-8'))
-  if match:
-    try:
-      releasesBadName[match.group(1)]
-    except KeyError:
-      releasesBadName[match.group(1)] = True
 for k in releasesBadName:
-  errorMsg = '{0}: Has a distribution filename missing the word "incubating"'.format(k)
+  errorMsg = '{0}: Has a distribution filename missing the word "incubating/incubator"'.format(k)
   print('ERROR:', errorMsg)
   errorMsg += ". See <a href=\"#h-hasRelease\">help</a>."
   otherIssues.append(errorMsg)



---------------------------------------------------------------------
To unsubscribe, e-mail: cvs-unsubscribe@incubator.apache.org
For additional commands, e-mail: cvs-help@incubator.apache.org


Mime
View raw message