incubator-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1439275 - in /incubator/public/trunk: incuvoter/ voter/ voter/incuvoter.py voter/podlings.py voter/status.py voter/utility.py voter/voter.py voter/votestatus.py
Date Mon, 28 Jan 2013 06:43:18 GMT
Author: brane
Date: Mon Jan 28 06:43:18 2013
New Revision: 1439275

URL: http://svn.apache.org/viewvc?rev=1439275&view=rev
Log:
The big restructuring:
  - Renamed incuvoter/ to voter/
  - Renamed voter/incuvoter.py to voter/voter.py
  - Renamed voter/votestatus.py to voter/status.py
  - Split common utilitues from voter/voter.py to voter/utility.py

Functional changes:
  - New script voter/podlings.py finds mail archives of active podlings
  - voter/voter.pt uses that list to scan the mail archives.

Added:
    incubator/public/trunk/voter/
      - copied from r1439256, incubator/public/trunk/incuvoter/
    incubator/public/trunk/voter/podlings.py
    incubator/public/trunk/voter/status.py
      - copied, changed from r1439180, incubator/public/trunk/incuvoter/votestatus.py
    incubator/public/trunk/voter/utility.py
    incubator/public/trunk/voter/voter.py
      - copied, changed from r1439255, incubator/public/trunk/incuvoter/incuvoter.py
Removed:
    incubator/public/trunk/incuvoter/
    incubator/public/trunk/voter/incuvoter.py
    incubator/public/trunk/voter/votestatus.py

Added: incubator/public/trunk/voter/podlings.py
URL: http://svn.apache.org/viewvc/incubator/public/trunk/voter/podlings.py?rev=1439275&view=auto
==============================================================================
--- incubator/public/trunk/voter/podlings.py (added)
+++ incubator/public/trunk/voter/podlings.py Mon Jan 28 06:43:18 2013
@@ -0,0 +1,124 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Purpose: Parses the Incubator podling status files to find the list of
+         development mailing list archive files.
+
+Status: Alpha
+'''
+
+from __future__ import absolute_import
+
+import os, sys
+import collections
+import traceback
+import xml.etree.ElementTree
+
+sys.path.insert(0, os.path.dirname(__file__))
+from utility import SiteStructure, UTC
+
+Podling = collections.namedtuple('Podling', ('name', 'mbox_relpath'))
+
+def __formatted_exception():
+    return ''.join(traceback.format_exception_only(*sys.exc_info()[:2]))
+
+def __parse_xml_file(path):
+    """
+    Parse an XML file and suppress any encountered errors.
+    """
+
+    try:
+        return xml.etree.ElementTree.parse(path)
+    except:
+        sys.stderr.write('ERROR: Could not parse ' + path + '\n')
+        sys.stderr.write(__formatted_exception())
+        return None
+
+def __find_podling_names():
+    """
+    Find names of currently active podlings.
+    """
+
+    doc = __parse_xml_file(SiteStructure.podlings())
+    if not doc:
+        return None
+
+    names = []
+    for info in doc.iterfind('podling'):
+        if info.get('status') != 'current':
+            continue
+        name = info.get('resource')
+        if name is None:
+            continue
+        names.append(name)
+    return names
+
+__incubator = 'incubator.apache.org'
+def __relpath_from_email(email):
+    """
+    Convert a list email address to an archive relative path.
+    """
+
+    email = email.strip()
+
+    # Some podlings are silly and add crud to the mailing list address
+    offset = email.find(__incubator)
+    if offset < 0:
+        sys.stderr.write('WARNING: Not an Incubator address: ' + email + '\n')
+        return None
+    email = email[:offset + len(__incubator)]
+
+    # Split the email into list name and host name
+    list_name, atsign, host = email.partition('@')
+    if not atsign or not host:
+        sys.stderr.write('WARNING: Invalid mail address: ' + email + '\n')
+        return None
+
+    # Old-style podling lists are inside the incubator archives.
+    # The mailing list address is podling-list@incubator.apache.org
+    if host == __incubator:
+        return os.path.join(__incubator, list_name)
+
+    # New-style podling lists have their own archive directory.
+    # The mailing list address is list@podling.incubator.apache.org
+    basedir = host[:len(host) - len(__incubator)] + 'apache.org'
+    return os.path.join(basedir, list_name)
+
+def podling_archives():
+    try:
+        podlings = []
+        for name in __find_podling_names():
+            doc = __parse_xml_file(SiteStructure.project_path(name + '.xml'))
+            if not doc:
+                continue
+            info = doc.find('.//*[@id="mail-dev"]')
+            if info is None:
+                sys.stderr.write('WARNING: ' + name + ' has no dev list?\n')
+                continue
+            email = ''
+            for el in info.iter():
+                email += (el.text or '')
+            podlings.append(Podling(name, __relpath_from_email(email)))
+        return podlings
+    except:
+        sys.stderr.write('ERROR: Could not find podlings\n')
+        sys.stderr.write(__formatted_exception())
+        return []
+
+
+def test():
+    for p in podling_archives:
+        print('%25s %s' % (p.name + ' dev list:', p.mbox_relpath))

Copied: incubator/public/trunk/voter/status.py (from r1439180, incubator/public/trunk/incuvoter/votestatus.py)
URL: http://svn.apache.org/viewvc/incubator/public/trunk/voter/status.py?p2=incubator/public/trunk/voter/status.py&p1=incubator/public/trunk/incuvoter/votestatus.py&r1=1439180&r2=1439275&rev=1439275&view=diff
==============================================================================
--- incubator/public/trunk/incuvoter/votestatus.py (original)
+++ incubator/public/trunk/voter/status.py Mon Jan 28 06:43:18 2013
@@ -16,13 +16,10 @@
 # limitations under the License.
 
 '''
-Purpose: Votestatus generates an HTML with the status of current incubator
-         votes, using data from Incuvoter.
+Purpose: generates an HTML with the status of current incubator
+         votes, using data from the votes database.
 
-It also uses incuvoter as a module, expecting to find it in the same
-directory.
-
-Status: Pre-Alpha, under construction.
+Status: Alpha
 '''
 
 from __future__ import absolute_import
@@ -32,7 +29,8 @@ import cgi
 import datetime
 
 sys.path.insert(0, os.path.dirname(__file__))
-from incuvoter import VoteDatabase, UTC
+from utility import SiteStructure, UTC
+from voter import VoteDatabase
 
 
 __page_template = """\
@@ -128,7 +126,7 @@ __closed_row = """\
     </tr>"""
 
 
-def __datescape(dateobject):
+def escape_date(dateobject):
   return cgi.escape(UTC.timedate(dateobject)).replace('-', '&ndash;')
 
 def refresh_page(target, database):
@@ -160,8 +158,8 @@ def refresh_page(target, database):
                        % dict(klass = klass,
                               duration = duration,
                               subject = cgi.escape(vote.subject),
-                              updated = __datescape(vote.updated),
-                              noticed = __datescape(vote.noticed)))
+                              updated = escape_date(vote.updated),
+                              noticed = escape_date(vote.noticed)))
     if current:
         current = __current_table % '\n'.join(current)
     else:
@@ -175,8 +173,8 @@ def refresh_page(target, database):
                         % dict(klass = klass,
                                status = status,
                                subject = cgi.escape(vote.subject),
-                               noticed = __datescape(vote.noticed),
-                               closed = __datescape(vote.closed)))
+                               noticed = escape_date(vote.noticed),
+                               closed = escape_date(vote.closed)))
     if resolved:
         resolved = __closed_table % '\n'.join(resolved)
     else:
@@ -189,7 +187,10 @@ def refresh_page(target, database):
     os.rename(temp, target)
 
 
-if __name__ == '__main__':
-    status_page = os.path.join(os.path.dirname(__file__), 'votes.html')
-    votes_path = os.path.join(os.path.dirname(__file__), 'votes.sqlite')
+def main():
+    votes_path = SiteStructure.votes_database()
+    status_page = SiteStructure.status_page()
     refresh_page(status_page, VoteDatabase(votes_path))
+
+if __name__ == '__main__':
+    main()

Added: incubator/public/trunk/voter/utility.py
URL: http://svn.apache.org/viewvc/incubator/public/trunk/voter/utility.py?rev=1439275&view=auto
==============================================================================
--- incubator/public/trunk/voter/utility.py (added)
+++ incubator/public/trunk/voter/utility.py Mon Jan 28 06:43:18 2013
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Purpose: Utilities for Voter & friends.
+
+Status: Alpha
+'''
+
+from __future__ import absolute_import
+
+import os.path
+import datetime
+
+
+class __UTC(datetime.tzinfo):
+    """
+    A datetime.tzinfo object representing Universal Coordinated Time
+    and providing a set of parsing and formatting utilities.
+    """
+
+    def utcoffset(self, dt):
+        return 0;
+    def dst(self, dt):
+        return timedelta(0)
+    def tzname(self,dt):
+        return 'UTC'
+
+    def adjust(self, dateobject):
+        if dateobject.tzinfo:
+            dateobject = dateobject.astimezone(self)
+        return dateobject
+
+    def timestring(self, dateobject):
+        if dateobject is not None:
+            return self.adjust(dateobject).strftime('%Y-%m-%d %H:%M:%S')
+        return None
+
+    def timedate(self, dateobject):
+        if dateobject is not None:
+            return self.adjust(dateobject).strftime('%Y-%m-%d')
+        return None
+
+    def timeparse(self, datestring):
+        if datestring is not None:
+            return datetime.datetime.strptime(datestring, '%Y-%m-%d %H:%M:%S')
+        return None
+UTC = __UTC()
+
+
+class SiteStructure(object):
+    """
+    This class describes the structure of the Incubator site and tells
+    other modules where to find information and/or store results.
+    """
+
+    INCUVOTER_DIR = os.path.abspath(os.path.dirname(__file__))
+    CONTENT_DIR = os.path.join(os.path.dirname(INCUVOTER_DIR), 'content')
+    PROJECTS_DIR = os.path.join(CONTENT_DIR, 'projects')
+
+    @classmethod
+    def incuvoter_path(cls, *relpath):
+        return os.path.join(cls.INCUVOTER_DIR, *relpath)
+
+    @classmethod
+    def content_path(cls, *relpath):
+        return os.path.join(cls.CONTENT_DIR, *relpath)
+
+    @classmethod
+    def project_path(cls, *relpath):
+        return os.path.join(cls.PROJECTS_DIR, *relpath)
+
+    @classmethod
+    def votes_database(cls):
+        return cls.incuvoter_path('votes.sqlite')
+
+    @classmethod
+    def status_page(cls):
+        return cls.incuvoter_path('votes.html')
+
+    @classmethod
+    def podlings(cls):
+        return cls.content_path('podlings.xml')

Copied: incubator/public/trunk/voter/voter.py (from r1439255, incubator/public/trunk/incuvoter/incuvoter.py)
URL: http://svn.apache.org/viewvc/incubator/public/trunk/voter/voter.py?p2=incubator/public/trunk/voter/voter.py&p1=incubator/public/trunk/incuvoter/incuvoter.py&r1=1439255&r2=1439275&rev=1439275&view=diff
==============================================================================
--- incubator/public/trunk/incuvoter/incuvoter.py (original)
+++ incubator/public/trunk/voter/voter.py Mon Jan 28 06:43:18 2013
@@ -15,23 +15,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-'''
-Purpose: Incuvoter maintaines a list of currently open and recently closed
+"""
+Purpose: Maintaines a list of currently open and recently closed
          votes on the general@incubator mailing list.
 
-It does so by parsing the mbox files of all Incubator list archives
-and updates information about votes, based on tags in from the subject
-lines, in a SQLite database.
+It does so by parsing the mbox files of general@incubator and all
+active podling list archives and updates information about votes,
+based on tags in from the subject lines, in a SQLite database.
 
-Usage: incuvoter.py <mbox_archive_basedir>
+Usage: voter.py <mbox_archive_basedir>
 
-   where mbox_archive_basedir is the root directory of all Incubator
-   mailing list archives in mbox format; e.g., on minotaur, it's
+   where mbox_archive_basedir is the root directory of all mailing
+   list archives in mbox format; e.g., on minotaur, it's
 
-       ~apmail/public-arch/incubator.apache.org
+       ~apmail/public-arch
 
-Status: Pre-Alpha, under construction.
-'''
+Status: Alpha
+"""
 
 from __future__ import absolute_import
 
@@ -42,40 +42,19 @@ import email
 import email.utils
 import sqlite3
 
-
-class __UTC(datetime.tzinfo):
-    def utcoffset(self, dt):
-        return 0;
-    def dst(self, dt):
-        return timedelta(0)
-    def tzname(self,dt):
-        return 'UTC'
-
-    def adjust(self, dateobject):
-        if dateobject.tzinfo:
-            dateobject = dateobject.astimezone(self)
-        return dateobject
-
-    def timestring(self, dateobject):
-        if dateobject is not None:
-            return self.adjust(dateobject).strftime('%Y-%m-%d %H:%M:%S')
-        return None
-
-    def timedate(self, dateobject):
-        if dateobject is not None:
-            return self.adjust(dateobject).strftime('%Y-%m-%d')
-        return None
-
-    def timeparse(self, datestring):
-        if datestring is not None:
-            return datetime.datetime.strptime(datestring, '%Y-%m-%d %H:%M:%S')
-        return None
-UTC = __UTC()
+sys.path.insert(0, os.path.dirname(__file__))
+from utility import SiteStructure, UTC
+from podlings import Podling, podling_archives
 
 
 class MBoxParser(object):
-    __general_rx = re.compile(r'general@incubator\.apache\.org')
+    """
+    Parser for mail archives in mbox format.
+    Scans the archve for all mails addressed To: or Cc: the Incubator
+    general mailing list.
+    """
 
+    __general_rx = re.compile(r'general@incubator\.apache\.org')
     MBox = collections.namedtuple('MBox', ('mtime', 'entries'))
     Entry = collections.namedtuple('Entry', ('updated', 'title'))
 
@@ -127,6 +106,10 @@ class MBoxParser(object):
 
 
 class VoteUpdater(object):
+    """
+    TODO: Docstring.
+    """
+
     __subject_rx = re.compile(
         # Skip anything before the first tag
         r'^[^[]*'
@@ -144,13 +127,23 @@ class VoteUpdater(object):
     def __init__(self, mbox_archive_basedir):
         self.mbox_basedir = mbox_archive_basedir
         self.mbox_relpaths = []
+
+        archives = ([Podling('Incubator', 'incubator.apache.org/general')]
+                    + podling_archives())
         month = datetime.datetime.utcnow().strftime('%Y%m')
-        for topdir, dirnames, filenames in os.walk(mbox_archive_basedir):
-            if month not in filenames:
+
+        for archive in archives:
+            if not archive.mbox_relpath:
+                continue
+            basedir = os.path.join(mbox_archive_basedir, archive.mbox_relpath)
+            if not os.path.isdir(basedir):
+                sys.stderr.write('WARNING: ' + archive.name
+                                 + ' has no archive at ' + basedir + '\n')
                 continue
-            relpath = os.path.relpath(os.path.join(topdir, month),
-                                      mbox_archive_basedir)
-            self.mbox_relpaths.append(relpath)
+            if not os.path.isfile(os.path.join(basedir, month)):
+                # Skip archives that have no mbox file for the current month.
+                continue
+            self.mbox_relpaths.append(os.path.join(archive.mbox_relpath, month))
 
     ParsedVote = collections.namedtuple(
         'ParsedVote',
@@ -207,6 +200,10 @@ class VoteUpdater(object):
 
 
 class VoteDatabase(object):
+    """
+    TODO: Docstring.
+    """
+
     __schema = """
         DROP TABLE IF EXISTS feedinfo;
         CREATE TABLE feedinfo (
@@ -421,7 +418,7 @@ class VoteDatabase(object):
 
 
 def main():
-    votes_path = os.path.join(os.path.dirname(__file__), 'votes.sqlite')
+    votes_path = SiteStructure.votes_database()
     if not os.path.isfile(votes_path):
         VoteDatabase.create(votes_path)
     updater = VoteUpdater(sys.argv[1])



---------------------------------------------------------------------
To unsubscribe, e-mail: cvs-unsubscribe@incubator.apache.org
For additional commands, e-mail: cvs-help@incubator.apache.org


Mime
View raw message