incubator-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1437841 - in /incubator/public/trunk/incuvoter: ./ incuvoter.py votestatus.py
Date Thu, 24 Jan 2013 02:25:22 GMT
Author: brane
Date: Thu Jan 24 02:25:22 2013
New Revision: 1437841

URL: http://svn.apache.org/viewvc?rev=1437841&view=rev
Log:
Added Incuvoter and Votestatus: scripts for generating an HTML status page with current Incubator
votes.

Added:
    incubator/public/trunk/incuvoter/   (with props)
    incubator/public/trunk/incuvoter/incuvoter.py   (with props)
    incubator/public/trunk/incuvoter/votestatus.py   (with props)

Propchange: incubator/public/trunk/incuvoter/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Thu Jan 24 02:25:22 2013
@@ -0,0 +1,3 @@
+*.pyc
+votes.sqlite
+votes.html

Added: incubator/public/trunk/incuvoter/incuvoter.py
URL: http://svn.apache.org/viewvc/incubator/public/trunk/incuvoter/incuvoter.py?rev=1437841&view=auto
==============================================================================
--- incubator/public/trunk/incuvoter/incuvoter.py (added)
+++ incubator/public/trunk/incuvoter/incuvoter.py Thu Jan 24 02:25:22 2013
@@ -0,0 +1,287 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Purpose: Incuvoter maintaines a list of currently open and recently closed
+         votes on the general@incubator mailing list.
+
+It does so by periodically parsing the Atom feed from the list archives
+(see FeedParser.__feed_url, below) and updates information about votes,
+based on tags in from the subject lines, in a SQLite database.
+
+Status: Pre-Alpha, under construction.
+'''
+
+from __future__ import absolute_import
+
+import os, re
+import collections
+import datetime
+import feedparser
+import sqlite3
+
+
+class FeedParser(object):
+
+    __feed_url = 'http://mail-archives.apache.org/mod_mbox/incubator-general/?format=atom'
+
+    __subject_rx = re.compile(
+        # Skip anything before the first tag
+        r'^[^[]*'
+        # A [RESULT] tag can come before the [VOTE] tag
+        r'(?P<firstres>\[RESULT\]\s*)?'
+        # The [VOTE] tag is mandatory
+        r'(?P<vote>\[VOTE\]\s*)'
+        # Handle [VOTE][RESULT] as well, just in case
+        r'(?P<secondres>\[RESULT\]\s*)?'
+        # The rest of the subject line, and strip off trailing whitespace
+        r'(?P<subject>.*?)\s*$',
+        re.IGNORECASE)
+
+    def __init__(self):
+        self.feed = feedparser.parse(self.__feed_url)
+
+    ParsedVote = collections.namedtuple(
+        'ParsedVote', ('sortkey', 'updated', 'closed', 'subject'))
+
+    def record(self, database):
+        feed_updated = self.feed.get('updated', None)
+        if not feed_updated:
+            return
+
+        votes = []
+        for e in self.feed.entries:
+            title = e.get('title', None)
+            if title is None:
+                continue
+
+            parsed = self.__subject_rx.match(title)
+            if parsed is None:
+                continue
+
+            subject = parsed.group('subject')
+            updated = max(e.get('created', None), e.get('updated', None))
+            if parsed.group('firstres') or parsed.group('secondres'):
+                closed = updated
+            else:
+                closed = None
+            votes.append(self.ParsedVote(subject.upper(),
+                                         updated, closed, subject))
+
+        # Uniquify votes, returning only the latest instance of each.
+        def unique_votes():
+            votes.sort()
+            last = None
+            for vote in votes:
+                if last and (last.subject != vote.subject
+                             or last.closed != vote.closed):
+                    yield last
+                last = vote
+            if last:
+                yield last
+
+        database.record_votes(feed_updated,
+                              (database.Vote(subject = v.subject,
+                                             updated = v.updated,
+                                             closed = v.closed)
+                               for v in unique_votes()))
+
+
+class FeedDatabase(object):
+    __schema = """
+        DROP TABLE IF EXISTS feedinfo;
+        CREATE TABLE feedinfo (
+          rowid INTEGER NOT NULL PRIMARY KEY,
+          updated TEXT NOT NULL,
+          CONSTRAINT singleton CHECK (rowid = 1)
+        );
+        INSERT INTO feedinfo (rowid, updated) VALUES (1, '');
+
+        DROP TABLE IF EXISTS vote;
+        CREATE TABLE vote (
+          sortkey TEXT NOT NULL PRIMARY KEY,
+          subject TEXT NOT NULL,
+          noticed TEXT NOT NULL,
+          updated TEXT NOT NULL,
+          closed TEXT DEFAULT NULL
+        );
+        CREATE INDEX updated_index ON vote(updated DESC);
+        CREATE INDEX closed_index ON vote(closed DESC);
+        """
+
+    @classmethod
+    def __connect(cls, path):
+        con = sqlite3.connect(path, isolation_level = 'IMMEDIATE')
+        con.row_factory = sqlite3.Row
+        cursor = con.cursor()
+        cursor.execute("PRAGMA page_size = 4096")
+        cursor.execute("PRAGMA temp_store = MEMORY")
+        cursor.execute("PRAGMA case_sensitive_like = ON")
+        cursor.execute("PRAGMA encoding = 'UTF-8'")
+        return con
+
+    @classmethod
+    def create(cls, path):
+        con = cls.__connect(path)
+        cursor = con.cursor()
+        cursor.executescript(cls.__schema)
+        con.close()
+
+
+    def __init__(self, path):
+        assert os.path.isfile(path)
+        self.con = self.__connect(path)
+        self.__updated = None
+
+    def close(self):
+        self.con.close()
+
+    @property
+    def updated(self):
+        if self.__updated is None:
+            cursor = self.con.cursor()
+            cursor.execute("SELECT updated FROM feedinfo WHERE rowid = 1")
+            self.__updated = cursor.fetchone()['updated']
+        return self.__updated
+
+    class Vote(object):
+        __slots__ = ('sortkey', 'subject', 'noticed', 'updated', 'closed')
+
+        def __init__(self, **kwargs):
+            for name in self.__slots__:
+                setattr(self, name, kwargs.get(name, None))
+
+        def merge(self, other):
+            if self is other:
+                return
+
+            assert (type(self) == type(other)
+                    and (self.subject is None
+                         or other.subject is None
+                         or self.subject == other.subject))
+
+            if other.updated > self.updated:
+                self.updated = other.updated
+            if not self.closed:
+                self.closed = other.closed
+
+        @classmethod
+        def timeobj(cls, datestring):
+            try:
+                return datetime.datetime.strptime(datestring, '%Y-%m-%dT%H:%M:%SZ')
+            except:
+                return datestring
+
+        @classmethod
+        def timefmt(cls, dateobject):
+            if isinstance(dateobject, datetime.datetime):
+                return dateobject.isoformat().replace('T', ' ')
+            return dateobject
+
+        @classmethod
+        def find(cls, con, subject):
+            sortkey = subject.upper()
+            cursor = con.cursor()
+            cursor.execute("SELECT subject, noticed, updated, closed"
+                           " FROM vote WHERE sortkey = ?", (sortkey,))
+            row = cursor.fetchone()
+            if row is None:
+                return None
+            return cls(sortkey=sortkey, **row)
+
+        def insert(self, con):
+            assert self.sortkey is None and self.subject is not None
+            self.sortkey = self.subject.upper()
+            if self.noticed is None:
+                self.noticed = self.updated
+            con.execute("INSERT INTO vote"
+                        " (sortkey, subject, noticed, updated, closed)"
+                        " VALUES (?, ?, ?, ?, ?)",
+                        (self.sortkey, self.subject,
+                         self.noticed, self.updated, self.closed))
+
+        def update(self, con):
+            assert self.sortkey is not None
+            assert self.subject.upper() == self.sortkey
+            con.execute("UPDATE vote SET noticed = ?, updated = ?, closed = ?"
+                        " WHERE sortkey = ?",
+                        (self.noticed, self.updated, self.closed, self.sortkey))
+
+    def record_votes(self, updated, votes):
+        self.con.execute("BEGIN")
+        try:
+            for v in votes:
+                vote = self.Vote.find(self.con, v.subject)
+                if vote:
+                    vote.merge(v)
+                    vote.update(self.con)
+                else:
+                    v.insert(self.con)
+
+            self.con.execute("UPDATE feedinfo SET updated = ?", (updated,))
+            self.__updated = None
+        except:
+            try:
+                self.con.rollback()
+            except:
+                pass
+            raise
+        else:
+            self.con.commit()
+
+    def __list_votes(self, active):
+        if active:
+            sql = ("SELECT sortkey, subject, noticed, updated, closed"
+                   " FROM vote WHERE closed IS NULL ORDER BY updated DESC")
+        else:
+            sql = ("SELECT sortkey, subject, noticed, updated, closed"
+                   " FROM vote WHERE closed IS NOT NULL ORDER BY closed DESC")
+        cursor = self.con.cursor()
+        cursor.execute(sql)
+        for row in cursor.fetchall():
+            yield self.Vote(**row)
+
+    def list_open_votes(self):
+        return self.__list_votes(True)
+
+    def list_resolved_votes(self):
+        return self.__list_votes(False)
+
+    def prune_old_votes(self):
+        now = datetime.datetime.utcnow()
+        cursor = self.con.cursor()
+        cursor.execute("SELECT sortkey, updated FROM vote"
+                       " WHERE closed IS NOT NULL ORDER BY closed DESC")
+        obsolete = []
+        for row in cursor.fetchall():
+            updated = self.timeobj(row['updated'])
+            if now - updated > datetime.timedelta(days = 30):
+                obsolete.append(row['sortkey'])
+        if obsolete:
+            cursor.execute("DELETE FROM vote WHERE sortkey IN ?", (obsolete,))
+            self.con.commit()
+
+
+if __name__ == '__main__':
+    votes_path = os.path.join(os.path.dirname(__file__), 'votes.sqlite')
+    if not os.path.isfile(votes_path):
+        FeedDatabase.create(votes_path)
+    database = FeedDatabase(votes_path)
+    parser = FeedParser()
+    parser.record(database)
+    database.prune_old_votes()
+    database.close()

Propchange: incubator/public/trunk/incuvoter/incuvoter.py
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/public/trunk/incuvoter/votestatus.py
URL: http://svn.apache.org/viewvc/incubator/public/trunk/incuvoter/votestatus.py?rev=1437841&view=auto
==============================================================================
--- incubator/public/trunk/incuvoter/votestatus.py (added)
+++ incubator/public/trunk/incuvoter/votestatus.py Thu Jan 24 02:25:22 2013
@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Purpose: Votestatus generates an HTML with the status of current incubator
+         votes, using data from Incuvoter.
+
+It also uses incuvoter as a module, expecting to find it in the same
+directory.
+
+Status: Pre-Alpha, under construction.
+'''
+
+from __future__ import absolute_import
+
+import os, sys
+import datetime
+
+sys.path.insert(0, os.path.dirname(__file__))
+from incuvoter import FeedDatabase
+
+
+__page_template = """\
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <title>Apache Incubator Voting Status</title>
+  <style type="text/css">
+  <!--
+  table   { white-space: pre; }
+  .normal {}
+  .nudge  { background: rgb(240, 200, 0); }
+  .nag    { background: rgb(240, 100, 0); font-weight: bold; }
+  .warn   { background: yellow; color: red; font-weight: bold; }
+  -->
+  </style>
+</head>
+<body>
+  <h1>Apache Incubator Voting Status</h1>
+%s
+%s
+</body>
+</html>
+"""
+
+__current_table = """\
+  <h2>Current Votes</h2>
+  <table>
+    <tr>
+     <th>Subject</th>
+     <th>Latest Activity</th>
+     <th>Started</th>
+    </tr>
+%s
+  </table>"""
+
+__current_row = """\
+    <tr class="%(klass)s">
+      <td>%(subject)s</td>
+      <td>%(updated)s</td>
+      <td>%(noticed)s</td>
+    </tr>"""
+
+__closed_table = """\
+  <h2>Recently Closed Votes</h2>
+  <table>
+    <tr>
+     <th>Subject</th>
+     <th>Started</th>
+     <th>Closed</th>
+    </tr>
+%s
+  </table>"""
+
+__closed_row = """\
+    <tr class="%(klass)s">
+      <td>%(subject)s</td>
+      <td>%(noticed)s</td>
+      <td>%(closed)s</td>
+    </tr>"""
+
+
+def __htmlescape(text):
+    #FIXME: TODO:
+    return text
+
+def refresh_page(target, database):
+    current = []
+    for vote in database.list_open_votes():
+        updated = vote.timeobj(vote.updated)
+        noticed = vote.timeobj(vote.noticed)
+        if (not isinstance(updated, datetime.datetime)
+            or not isinstance(noticed, datetime.datetime)):
+            klass = 'warn'
+        else:
+            age = updated - noticed
+            if age < datetime.timedelta(hours = 73):
+                klass = 'normal'
+            elif age <= datetime.timedelta(days = 7):
+                klass = 'nudge'
+            else:
+                klass = 'nag'
+        current.append(__current_row
+                       % dict(klass = klass,
+                              subject = __htmlescape(vote.subject),
+                              updated = __htmlescape(vote.timefmt(updated)),
+                              noticed = __htmlescape(vote.timefmt(noticed))))
+    if current:
+        current = __current_table % '\n'.join(current)
+    else:
+        current = ''
+
+    closed = []
+    for vote in database.list_resolved_votes():
+        noticed = vote.timeobj(vote.noticed)
+        closed = vote.timeobj(vote.closed)
+        if (not isinstance(noticed, datetime.datetime)
+            or not isinstance(closed, datetime.datetime)):
+            klass = 'warn'
+        else:
+            klass = 'normal'
+        closed.append(__closed_row
+                      % dict(klass = klass,
+                             subject = __htmlescape(vote.subject),
+                             noticed = __htmlescape(vote.timefmt(noticed)),
+                             closed = __htmlescape(vote.timefmt(closed))))
+    if closed:
+        closed = __closed_table % '\n'.join(closed)
+    else:
+        closed = ''
+
+    temp = target + '.temp'
+    with open(temp, 'wt') as page:
+        page.write(__page_template % (current, closed))
+    os.rename(temp, target)
+
+
+if __name__ == '__main__':
+    status_page = os.path.join(os.path.dirname(__file__), 'votes.html')
+    votes_path = os.path.join(os.path.dirname(__file__), 'votes.sqlite')
+    refresh_page(status_page, FeedDatabase(votes_path))

Propchange: incubator/public/trunk/incuvoter/votestatus.py
------------------------------------------------------------------------------
    svn:executable = *



---------------------------------------------------------------------
To unsubscribe, e-mail: cvs-unsubscribe@incubator.apache.org
For additional commands, e-mail: cvs-help@incubator.apache.org


Mime
View raw message