community-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject svn commit: r1714894 - /comdev/projects.apache.org/scripts/cronjobs/urlutils.py
Date Tue, 17 Nov 2015 21:45:28 GMT
Author: sebb
Date: Tue Nov 17 21:45:28 2015
New Revision: 1714894

URL: http://svn.apache.org/viewvc?rev=1714894&view=rev
Log:
Optionally use the downloaded file mod time as the last time the file was checked
Intended for URLs that don't support conditional download

Modified:
    comdev/projects.apache.org/scripts/cronjobs/urlutils.py

Modified: comdev/projects.apache.org/scripts/cronjobs/urlutils.py
URL: http://svn.apache.org/viewvc/comdev/projects.apache.org/scripts/cronjobs/urlutils.py?rev=1714894&r1=1714893&r2=1714894&view=diff
==============================================================================
--- comdev/projects.apache.org/scripts/cronjobs/urlutils.py (original)
+++ comdev/projects.apache.org/scripts/cronjobs/urlutils.py Tue Nov 17 21:45:28 2015
@@ -89,7 +89,10 @@ def findRelPath(relpath):
 
 class UrlCache(object):
     """
-        Creates a cache for URLs
+        Creates a cache for URLs.
+        The file modification time is set to the Last-Modified header of the URL (if any)
+        A hidden marker file is used to record the last check time (unless useFileModTime==True)
+
         @param cachedir: the cache directory to use 
             (default data/cache; this is assumed to be at the current directory, its parent
or grandparent)
         @param interval: minimum interval between checks for updates to the URL (default
300 secs)
@@ -135,7 +138,7 @@ class UrlCache(object):
         except FileNotFoundError:
             pass
 
-    def get(self, url, name, encoding=None, errors=None):
+    def get(self, url, name, encoding=None, errors=None, useFileModTime=False):
         """
             Check if the filename exists in the cache.
             If it does not, or if it does and the URL has not been checked recently,
@@ -150,11 +153,17 @@ class UrlCache(object):
             @param encoding: the encoding to use (default None)
             @param errors: If encoding is provided, this specifies the on-error action (e.g.
'ignore')
                         (default None)
+            @param useFileModTime: whether to use the file modification time as the last
check time
+            If not, a hidden marker file is used (default false). Set this to true for URLs
that don't
+            provide a Last-Modified header
             @return: the opened stream, using the encoding if specified. Otherwise opened
in binary mode. 
         """
         target=self.__getname(name)
         fileTime = self.__file_mtime(target)
-        check = self.__getname("."+name)
+        if useFileModTime:
+            check = self.__getname(name)
+        else:
+            check = self.__getname("."+name)
         upToDate = False
         if fileTime >= 0:
             if self.__interval == -1:
@@ -192,8 +201,9 @@ class UrlCache(object):
                 tmpFile = target + ".tmp"
                 with open(tmpFile,'wb') as f:
                     shutil.copyfileobj(response, f)
-                # store the last mod time as the time of the file
-                touchFile(tmpFile, lastModT)
+                if not useFileModTime:
+                    # store the last mod time as the time of the file
+                    touchFile(tmpFile, lastModT)
                 os.rename(tmpFile, target) # seems to preserve file mod time
                 if lastMod:
                     if fileTime > 0:
@@ -207,8 +217,11 @@ class UrlCache(object):
 
     
             if self.__interval > 0: # no point creating a marker file if we won't be using
it
-                with open(check,'a'):
+                if useFileModTime:
                     os.utime(check, None) # touch the marker file
+                else:
+                    with open(check,'a'):
+                        os.utime(check, None) # touch the marker file
 
         if encoding:
             return open(target, 'r', encoding=encoding, errors=errors)



Mime
View raw message