community-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject svn commit: r1688788 - in /comdev/reporter.apache.org: mailglomper.py parseversions.py scandist.py
Date Thu, 02 Jul 2015 09:06:41 GMT
Author: sebb
Date: Thu Jul  2 09:06:41 2015
New Revision: 1688788

URL: http://svn.apache.org/r1688788
Log:
Add som files currently only on the reporter host

Added:
    comdev/reporter.apache.org/mailglomper.py   (with props)
    comdev/reporter.apache.org/parseversions.py   (with props)
    comdev/reporter.apache.org/scandist.py   (with props)

Added: comdev/reporter.apache.org/mailglomper.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/mailglomper.py?rev=1688788&view=auto
==============================================================================
--- comdev/reporter.apache.org/mailglomper.py (added)
+++ comdev/reporter.apache.org/mailglomper.py Thu Jul  2 09:06:41 2015
@@ -0,0 +1,67 @@
+import re, json, os, sys, urllib, time, email.utils
+
+from datetime import datetime
+
+
+mls = {}
+try:
+        with open("data/maildata_extended.json",'r') as f:
+                mls = json.loads(f.read())
+        print("Read JSON")
+except:
+        pass
+
+currentMonth = datetime.now().month
+currentYear = datetime.now().year
+after = time.time() - (86400*92)
+wayafter = time.time() - (86400*92*2)
+months = []
+for i in range(0,7):
+        date = "%04u%02u" % (currentYear, currentMonth)
+        currentMonth -= 1
+        if currentMonth == 0:
+                currentMonth = 12
+                currentYear -= 1
+        months.append(date)
+
+        now = int(time.time())
+
+data = urllib.urlopen("http://mail-archives.us.apache.org/mod_mbox/").read()
+print("Fetched %u bytes of main data" % len(data))
+y = 0
+for mlist in re.finditer(r"<a href='([-a-z0-9]+)/'", data):
+        ml = mlist.group(1)
+        y += 1
+        mls[ml] = {}
+        mls[ml]['quarterly'] = [0, 0];
+        mls[ml]['weekly'] = {}
+        for date in months:
+                        
+                try:
+                        mldata = urllib.urlopen("http://mail-archives.us.apache.org/mod_mbox/%s/%s.mbox"
% (ml, date)).read()
+                        if mldata:
+                                for c in re.finditer(r"Date: (.+)", mldata):
+                                        try:
+                                                d = email.utils.parsedate(c.group(1))
+                                                timestamp = int(time.mktime(d))
+                                                rounded = timestamp - (timestamp % 604800)
+ 604800
+                                                mls[ml]['weekly'][rounded] = (mls[ml]['weekly'][rounded]
if rounded in mls[ml]['weekly'] else 0) + 1
+                                                if timestamp >= after:
+                                                        mls[ml]['quarterly'][0] += 1
+                                                elif timestamp >= wayafter:
+                                                        mls[ml]['quarterly'][1] += 1
+                                        except:
+                                                pass
+                                                
+                except Exception as err:
+                        print(err)
+        print("%s: %u" % (ml, mls[ml]['quarterly'][0]))
+        if y == 50:
+                y = 0
+                with open("data/maildata_extended.json",'w+') as f:
+                        f.write(json.dumps(mls))
+with open("data/maildata_extended.json",'w+') as f:
+        f.write(json.dumps(mls))
+print("Dumped JSON")
+
+

Propchange: comdev/reporter.apache.org/mailglomper.py
------------------------------------------------------------------------------
    svn:eol-style = native

Added: comdev/reporter.apache.org/parseversions.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/parseversions.py?rev=1688788&view=auto
==============================================================================
--- comdev/reporter.apache.org/parseversions.py (added)
+++ comdev/reporter.apache.org/parseversions.py Thu Jul  2 09:06:41 2015
@@ -0,0 +1,41 @@
+import os, sys, json, urllib2, re, time, base64
+
+def getReleaseData(project):
+    try:
+        with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "r") as
f:
+            x = json.loads(f.read())
+            f.close()
+        return x;
+    except:
+        return {}
+       
+projects = {
+       'trafficserver': 'TS',
+       'accumulo': 'ACCUMULO'
+}
+
+jirapass = ""
+with open("/var/www/reporter.apache.org/data/jirapass.txt", "r") as f:
+    jirapass = f.read().strip()
+    f.close()
+
+for project in projects:
+       jiraname = projects[project]
+       base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1]
+       rdata = getReleaseData(project)
+       try:
+           req = req = urllib2.Request("https://issues.apache.org/jira/rest/api/2/project/%s/versions"
% jiraname)
+           req.add_header("Authorization", "Basic %s" % base64string)
+           cdata = json.loads(urllib2.urlopen(req).read())
+           for entry in cdata:
+              if ('name' in entry and 'releaseDate' in entry and 'released' in entry and
entry['released']):
+                     
+                     date = time.mktime(time.strptime(entry['releaseDate'], "%Y-%m-%d"))
+                     print("Updating version for %s - %s: %u" % (project, entry['name'],
date))
+                     rdata[entry['name']] = date
+       except Exception as err:
+           print(err)
+       
+       with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "w") as
f:
+              f.write(json.dumps(rdata))
+              f.close()
\ No newline at end of file

Propchange: comdev/reporter.apache.org/parseversions.py
------------------------------------------------------------------------------
    svn:eol-style = native

Added: comdev/reporter.apache.org/scandist.py
URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/scandist.py?rev=1688788&view=auto
==============================================================================
--- comdev/reporter.apache.org/scandist.py (added)
+++ comdev/reporter.apache.org/scandist.py Thu Jul  2 09:06:41 2015
@@ -0,0 +1,321 @@
+#!/usr/bin/env python
+
+#################################################
+# scandist.py: a minimalistic svnwcsub program  #
+#################################################
+
+from threading import Thread
+from datetime import datetime
+import os
+import sys
+
+
+# SMTP Lib
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+
+version = 2
+if sys.hexversion < 0x03000000:
+    print("Using Python 2...")
+    import json, httplib, urllib, urllib2, ConfigParser as configparser, re, base64, sys,
os, time, atexit, signal, logging, socket, subprocess
+    socket._fileobject.default_bufsize = 0
+else:
+    print("Using Python 3")
+    version = 3
+    import json, http.client, urllib.request, urllib.parse, configparser, re, base64, sys,
os, time, atexit, signal, logging, subprocess
+
+targets = {}
+###########################################################
+# Daemon class, curtesy of an anonymous good-hearted soul #
+###########################################################
+class daemon:
+        """A generic daemon class.
+
+        Usage: subclass the daemon class and override the run() method."""
+
+        def __init__(self, pidfile): self.pidfile = pidfile
+        
+        def daemonize(self):
+                """Deamonize class. UNIX double fork mechanism."""
+
+                try: 
+                        pid = os.fork() 
+                        if pid > 0:
+                                # exit first parent
+                                sys.exit(0) 
+                except OSError as err: 
+                        sys.stderr.write('fork #1 failed: {0}\n'.format(err))
+                        sys.exit(1)
+        
+                # decouple from parent environment
+                os.chdir('/') 
+                os.setsid() 
+                os.umask(0) 
+        
+                # do second fork
+                try: 
+                        pid = os.fork() 
+                        if pid > 0:
+
+                                # exit from second parent
+                                sys.exit(0) 
+                except OSError as err: 
+                        sys.stderr.write('fork #2 failed: {0}\n'.format(err))
+                        sys.exit(1) 
+        
+                # redirect standard file descriptors
+                sys.stdout.flush()
+                sys.stderr.flush()
+                si = open(os.devnull, 'r')
+                so = open(os.devnull, 'a+')
+                se = open(os.devnull, 'a+')
+
+                os.dup2(si.fileno(), sys.stdin.fileno())
+                os.dup2(so.fileno(), sys.stdout.fileno())
+                os.dup2(se.fileno(), sys.stderr.fileno())
+        
+                # write pidfile
+                atexit.register(self.delpid)
+
+                pid = str(os.getpid())
+                with open(self.pidfile,'w+') as f:
+                        f.write(pid + '\n')
+        
+        def delpid(self):
+                os.remove(self.pidfile)
+
+        def start(self):
+                """Start the daemon."""
+
+                # Check for a pidfile to see if the daemon already runs
+                try:
+                        with open(self.pidfile,'r') as pf:
+
+                                pid = int(pf.read().strip())
+                except IOError:
+                        pid = None
+        
+                if pid:
+                        message = "pidfile {0} already exist. " + \
+                                        "Daemon already running?\n"
+                        sys.stderr.write(message.format(self.pidfile))
+                        sys.exit(1)
+                
+                # Start the daemon
+                self.daemonize()
+                self.run()
+
+        def stop(self):
+                """Stop the daemon."""
+
+                # Get the pid from the pidfile
+                try:
+                        with open(self.pidfile,'r') as pf:
+                                pid = int(pf.read().strip())
+                except IOError:
+                        pid = None
+        
+                if not pid:
+                        message = "pidfile {0} does not exist. " + \
+                                        "Daemon not running?\n"
+                        sys.stderr.write(message.format(self.pidfile))
+                        return # not an error in a restart
+
+                # Try killing the daemon process        
+                try:
+                        while 1:
+                                os.kill(pid, signal.SIGTERM)
+                                time.sleep(0.1)
+                except OSError as err:
+                        e = str(err.args)
+                        if e.find("No such process") > 0:
+                                if os.path.exists(self.pidfile):
+                                        os.remove(self.pidfile)
+                        else:
+                                print (str(err.args))
+                                sys.exit(1)
+
+        def restart(self):
+                """Restart the daemon."""
+                self.stop()
+                self.start()
+
+        def run(self):
+                """You should override this method when you subclass Daemon.
+                
+                It will be called after the process has been daemonized by 
+                start() or restart()."""
+
+
+
+####################
+# Helper functions #
+####################
+
+
+# read_chunk: iterator for reading chunks from the stream
+# since this is all handled via urllib now, this is quite rudimentary
+def read_chunk(req):
+    while True:
+        try:
+            line = req.readline().strip()
+            if line:
+                yield line
+            else:
+                print("No more lines?")
+                break
+        except Exception as info:
+            
+            break
+    return
+
+ 
+#########################
+# Main listener program #
+#########################
+
+
+
+# PubSub class: handles connecting to a pubsub service and checking commits
+class PubSubClient(Thread):
+    def run(self):
+        global targets
+        while True:
+            
+            self.req = None
+            while not self.req:
+                try:
+                    if version == 3:
+                        self.req = urllib.request.urlopen(self.url, None, 30)
+                    else:
+                        self.req = urllib2.urlopen(self.url, None, 30)
+                    
+                except:
+                    
+                    time.sleep(30)
+                    continue
+                
+            for line in read_chunk(self.req):
+                if version == 3:
+                    line = str( line, encoding='ascii' ).rstrip('\r\n,').replace('\x00','')
# strip away any old pre-0.9 commas from gitpubsub chunks and \0 in svnpubsub chunks
+                else:
+                    line = str( line ).rstrip('\r\n,').replace('\x00','') # strip away any
old pre-0.9 commas from gitpubsub chunks and \0 in svnpubsub chunks
+                try:
+                    obj = json.loads(line)
+                    if "commit" in obj and "repository" in obj['commit']:
+                        obj['commit']['whence'] = time.time()
+                    
+                            
+                        # If it's our public svn repo, then...
+                        if 'changed' in obj['commit']:
+                        
+                            #Grab some vars
+                            commit = obj['commit']
+                            body = commit['log']
+                            svnuser = commit['committer']
+                            path, action = commit['changed'].popitem()
+                            revision = commit['id']
+                            email = svnuser + "@apache.org"
+                            
+                            # Figure out the root thingermajig
+                            match = re.match(r"^release/([a-z0-9]+)", path)
+                            if match:
+                                if match.group(1) != "incubator":
+                                    targets[svnuser] = match.group(1)
+                            
+                    
+
+                except ValueError as detail:
+                    continue
+            
+
+
+
+
+################         
+# Main program #
+################
+def main():
+    global targets
+    if debug:
+        print("Foreground test mode enabled, no updates will be made")
+        
+  
+    
+    # Start the svn thread
+    svn_thread = PubSubClient()
+    svn_thread.url = "http://hades.apache.org:2069/commits/*"
+    svn_thread.start()
+    
+    while True:
+        now = time.time()
+        then = now - 86400
+       
+        time.sleep(600)
+        targetstwo = targets
+        targets = {}
+        for user in targetstwo:
+            email = user + "@apache.org"
+            project = targetstwo[user]
+            sender = 'no-reply@reporter.apache.org'
+            receivers = [email, 'humbedooh@apache.org']
+            print("Notifying %s of new data pushed to %s" % (email, project))
+            message = """From: Apache Reporter Service <no-reply@reporter.apache.org>
+To: %s <%s>
+Reply-To: dev@community.apache.org
+Subject: Please add your release data for '%s'
+
+Hi,
+This is an automated email from reporter.apache.org.
+I see that you just pushed something to our release repository for the %s project.
+
+If you are a PMC member of this project, we ask that you log on to:
+https://reporter.apache.org/addrelease.html?%s
+and add your release data (version and date) to the database.
+
+If you are not a PMC member, please have a PMC member add this information.
+
+While this is not a requirement, we ask that you still add this data to the
+reporter database, so that people using the Apache Reporter Service will be
+able to see the latest release data for this project.
+
+With regards,
+The Apache Reporter Service.
+            """ % (user, email, project, project, project)
+            
+            try:
+               smtpObj = smtplib.SMTP('localhost')
+               smtpObj.sendmail(sender, receivers, message)         
+               print "Successfully sent email"
+            except SMTPException:
+               print "Error: unable to send email"
+               
+
+##############
+# Daemonizer #
+##############
+class MyDaemon(daemon):
+    def run(self):
+        main()
+ 
+if __name__ == "__main__":
+        daemon = MyDaemon('/tmp/scandist.pid')
+        if len(sys.argv) >= 2:
+                if 'start' == sys.argv[1]:
+                    daemon.start()
+                elif 'stop' == sys.argv[1]:
+                    daemon.stop()
+                elif 'restart' == sys.argv[1]:
+                    daemon.restart()
+                elif 'foreground' == sys.argv[1]:
+                    debug = True
+                    main()
+                else:
+                    print("Unknown command")
+                    sys.exit(2)
+                sys.exit(0)
+        else:
+                print("usage: %s start|stop|restart|foreground" % sys.argv[0])
+                sys.exit(2)
+

Propchange: comdev/reporter.apache.org/scandist.py
------------------------------------------------------------------------------
    svn:eol-style = native



Mime
View raw message