incubator-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1437977 - in /incubator/public/trunk/incuvoter: incuvoter.py mboxparser.py votestatus.py
Date Thu, 24 Jan 2013 12:33:02 GMT
Author: brane
Date: Thu Jan 24 12:33:01 2013
New Revision: 1437977

URL: http://svn.apache.org/viewvc?rev=1437977&view=rev
Log:
* mboxparser.py:
  - Incuvoter expects parsed dates as time.struct_time, not datetime.datetime.
* incuvoter.py:
  - Pick either Atom or mbox parser, depending on parameters.
  - Make feedparser dependency optional.
* votestatys.py:
  - Calculate vote age based on current time, not activity.

Modified:
    incubator/public/trunk/incuvoter/incuvoter.py
    incubator/public/trunk/incuvoter/mboxparser.py
    incubator/public/trunk/incuvoter/votestatus.py

Modified: incubator/public/trunk/incuvoter/incuvoter.py
URL: http://svn.apache.org/viewvc/incubator/public/trunk/incuvoter/incuvoter.py?rev=1437977&r1=1437976&r2=1437977&view=diff
==============================================================================
--- incubator/public/trunk/incuvoter/incuvoter.py (original)
+++ incubator/public/trunk/incuvoter/incuvoter.py Thu Jan 24 12:33:01 2013
@@ -19,29 +19,40 @@
 Purpose: Incuvoter maintaines a list of currently open and recently closed
          votes on the general@incubator mailing list.
 
-It does so by periodically parsing the Atom feed from the list archives
-(see FeedParser.__feed_url, below) and updates information about votes,
+It does so by parsing either the Atom feed (see FeedParser.__feed_url, below)
+ or an mbox file of the list archives and updates information about votes,
 based on tags in from the subject lines, in a SQLite database.
 
+Usage: incuvoter.py [mbox_path]
+
+   If mbox_path is not provided, retreives and parses an Atom feed.
+   Otherwise, parses the mbox file.
+
 Status: Pre-Alpha, under construction.
 '''
 
 from __future__ import absolute_import
 
-import os, re
+import os, re, sys
 import collections
 import datetime
-import feedparser
 import sqlite3
 import time
 
+# The feedparser module is an optional dependency
+try:
+    import feedparser
+except ImportError:
+    feedparser = None
+
 
 class FeedParser(object):
-    # FIXME: Our own feed seems to be woefully incomplete. Use Markmail's for now.
+    # FIXME: Our own feed seems to be woefully incomplete.
+    #        Use Markmail's for now.
     #__feed_url = 'http://mail-archives.apache.org/mod_mbox/incubator-general/?format=atom'
     __feed_url = ('http://markmail.org/atom/list:org.apache.incubator.general+date:'
-                  + datetime.datetime.utcnow().strftime('%Y%m')  # current month
-                  + '+order:date-forward')
+                   + datetime.datetime.utcnow().strftime('%Y%m')  # current month
+                   + '+order:date-forward')
 
     __subject_rx = re.compile(
         # Skip anything before the first tag
@@ -56,8 +67,23 @@ class FeedParser(object):
         r'(?P<subject>.*?)\s*$',
         re.IGNORECASE)
 
-    def __init__(self):
-        self.feed = feedparser.parse(self.__feed_url)
+    def __init__(self, mbox_path = None):
+        if not mbox_path:
+            if not feedparser:
+                sys.stderr.write('ERROR: feedparser module is not available\n')
+                sys.exit(1)
+            self.feed = feedparser.parse(self.__feed_url)
+        else:
+            sys_path_saved = sys.path
+            sys.path.insert(0, os.path.dirname(__file__))
+            try:
+                import mboxparser
+            finally:
+                sys.path = sys_path_saved
+            self.feed  = mboxparser.parse_mbox(mbox_path)
+            if not self.feed:
+                sys.stderr.write('ERROR: mbox not found: %s\n' % mbox_path)
+                sys.exit(1)
 
     ParsedVote = collections.namedtuple(
         'ParsedVote', ('sortkey', 'updated', 'closed', 'subject'))
@@ -66,7 +92,9 @@ class FeedParser(object):
         feed_updated = self.feed.get('updated', None)
         if not feed_updated:
             feed_updated = database.Vote.timefmt(datetime.datetime.utcnow())
-            #return
+        if database.updated == feed_updated:
+            # Nothing changed, do not modify the database.
+            return
 
         votes = []
         for e in self.feed.entries:
@@ -234,6 +262,7 @@ class FeedDatabase(object):
                         (self.noticed, self.updated, self.closed, self.sortkey))
 
     def record_votes(self, updated, votes):
+        self.con.rollback()
         self.con.execute("BEGIN")
         try:
             for v in votes:
@@ -288,12 +317,18 @@ class FeedDatabase(object):
             self.con.commit()
 
 
-if __name__ == '__main__':
+def main():
     votes_path = os.path.join(os.path.dirname(__file__), 'votes.sqlite')
     if not os.path.isfile(votes_path):
         FeedDatabase.create(votes_path)
+    if len(sys.argv) > 1:
+        parser = FeedParser(sys.argv[1])
+    else:
+        parser = FeedParser()
     database = FeedDatabase(votes_path)
-    parser = FeedParser()
     parser.record(database)
     database.prune_old_votes()
     database.close()
+
+if __name__ == '__main__':
+    main()

Modified: incubator/public/trunk/incuvoter/mboxparser.py
URL: http://svn.apache.org/viewvc/incubator/public/trunk/incuvoter/mboxparser.py?rev=1437977&r1=1437976&r2=1437977&view=diff
==============================================================================
--- incubator/public/trunk/incuvoter/mboxparser.py (original)
+++ incubator/public/trunk/incuvoter/mboxparser.py Thu Jan 24 12:33:01 2013
@@ -25,12 +25,13 @@ import os
 import datetime
 import email
 import email.utils
+import time
 
 
 class MBoxFeed(object):
     def __init__(self, updated):
         self.updated_parsed = updated
-        self.updated = updated.isoformat() + 'Z'
+        self.updated = time.strftime('%Y-%m-%dT%H:%M:%SZ', updated)
 
     def get(self, name, default=None):
         if hasattr(self, name):
@@ -49,7 +50,7 @@ def append_message(text, entries):
         return
 
     timestamp = email.utils.mktime_tz(email.utils.parsedate_tz(date))
-    emsg = MBoxFeed(datetime.datetime.utcfromtimestamp(timestamp))
+    emsg = MBoxFeed(time.gmtime(timestamp))
     emsg.title = email.utils.collapse_rfc2231_value(email.utils.decode_rfc2231(subject))
     entries.append(emsg)
 
@@ -59,7 +60,7 @@ def parse_mbox(path):
 
     with open(path, 'rt') as mbox:
         timestamp = os.fstat(mbox.fileno()).st_mtime
-        feed = MBoxFeed(datetime.datetime.utcfromtimestamp(timestamp))
+        feed = MBoxFeed(time.gmtime(timestamp))
         entries = []
         text = None
         for line in mbox:

Modified: incubator/public/trunk/incuvoter/votestatus.py
URL: http://svn.apache.org/viewvc/incubator/public/trunk/incuvoter/votestatus.py?rev=1437977&r1=1437976&r2=1437977&view=diff
==============================================================================
--- incubator/public/trunk/incuvoter/votestatus.py (original)
+++ incubator/public/trunk/incuvoter/votestatus.py Thu Jan 24 12:33:01 2013
@@ -123,6 +123,7 @@ __closed_row = """\
 
 def refresh_page(target, database):
     current = []
+    now = datetime.datetime.utcnow()
     for vote in database.list_open_votes():
         updated = vote.timeobj(vote.updated)
         noticed = vote.timeobj(vote.noticed)
@@ -130,7 +131,7 @@ def refresh_page(target, database):
             or not isinstance(noticed, datetime.datetime)):
             klass = 'warn'
         else:
-            age = updated - noticed
+            age = now - noticed
             if age < datetime.timedelta(hours = 73):
                 klass = 'normal'
             elif age <= datetime.timedelta(days = 7):



---------------------------------------------------------------------
To unsubscribe, e-mail: cvs-unsubscribe@incubator.apache.org
For additional commands, e-mail: cvs-help@incubator.apache.org


Mime
View raw message