community-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject svn commit: r1730132 - /comdev/projects.apache.org/scripts/cronjobs/parseprojects.py
Date Fri, 12 Feb 2016 22:30:19 GMT
Author: sebb
Date: Fri Feb 12 22:30:18 2016
New Revision: 1730132

URL: http://svn.apache.org/viewvc?rev=1730132&view=rev
Log:
Delete any projects/*.json files that were not in projects.xml

Modified:
    comdev/projects.apache.org/scripts/cronjobs/parseprojects.py

Modified: comdev/projects.apache.org/scripts/cronjobs/parseprojects.py
URL: http://svn.apache.org/viewvc/comdev/projects.apache.org/scripts/cronjobs/parseprojects.py?rev=1730132&r1=1730131&r2=1730132&view=diff
==============================================================================
--- comdev/projects.apache.org/scripts/cronjobs/parseprojects.py (original)
+++ comdev/projects.apache.org/scripts/cronjobs/parseprojects.py Fri Feb 12 22:30:18 2016
@@ -7,6 +7,7 @@ import xml.etree.ElementTree as ET
 import re, urllib.request
 import json
 import os
+from os.path import join
 import traceback
 
 """
@@ -21,10 +22,15 @@ Writes:
 ../../site/json/projects/%s.json
 parseprojects-failures.xml (if failures occurred)
 
+Deletes any obsolete files from:
+../../site/json/projects/%s.json
+
 """
 
 URL_TIMEOUT = 60.0 # timeout for URL requests (may need tweaking)
 
+PROJECTS_DIR = '../../site/json/projects'
+
 projectsList = "../../data/projects.xml";
 save = True;
 if os.path.exists("parseprojects-failures.xml"):
@@ -115,6 +121,7 @@ def handleChild(el):
                 break
     return tag, retval
 
+files = []
 for s in itemlist :
     url = s.childNodes[0].data
     try:
@@ -205,8 +212,10 @@ for s in itemlist :
             projects[projectJsonFilename] = pjson
             #for e in add:
             #    pjson[e] = add[e]
-            print("Writing projects/%s" % projectJsonFilename+".json")
-            with open ("../../site/json/projects/%s.json" % projectJsonFilename, "w", encoding='utf-8')
as f:
+            name = "%s.json" % projectJsonFilename
+            print("Writing projects/%s" % name)
+            files.append(name)
+            with open (join(PROJECTS_DIR, name), "w", encoding='utf-8') as f:
                 json.dump(pjson, f, sort_keys=True, indent=0, ensure_ascii=False)
                 f.close()
         else:
@@ -228,6 +237,12 @@ if save:
         json.dump(projects, f, sort_keys=True, indent=0, ensure_ascii=False)
         f.close()
 
+# Drop any obsolete files
+for f in os.listdir(PROJECTS_DIR):
+    if re.match(r'.*\.json$', f) and not f in files:
+        print("Deleting obsolete file projects/%s" %f)
+        os.remove(join(PROJECTS_DIR,f))
+
 if len(failures) > 0:
     with open ("parseprojects-failures.xml", "w") as f:
         f.write("<doapFiles>\n")
@@ -243,4 +258,5 @@ else:
         except FileNotFoundError: # should not happen
             pass
 
+
 print("Done!")



Mime
View raw message