Return-Path: X-Original-To: apmail-community-commits-archive@minotaur.apache.org Delivered-To: apmail-community-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2343B186F8 for ; Thu, 2 Jul 2015 09:03:38 +0000 (UTC) Received: (qmail 56199 invoked by uid 500); 2 Jul 2015 09:03:37 -0000 Delivered-To: apmail-community-commits-archive@community.apache.org Received: (qmail 56176 invoked by uid 500); 2 Jul 2015 09:03:37 -0000 Mailing-List: contact commits-help@community.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@community.apache.org Delivered-To: mailing list commits@community.apache.org Received: (qmail 56167 invoked by uid 99); 2 Jul 2015 09:03:37 -0000 Received: from eris.apache.org (HELO hades.apache.org) (140.211.11.105) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 02 Jul 2015 09:03:37 +0000 Received: from hades.apache.org (localhost [127.0.0.1]) by hades.apache.org (ASF Mail Server at hades.apache.org) with ESMTP id BB2E1AC0330 for ; Thu, 2 Jul 2015 09:03:37 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1688786 - in /comdev/reporter.apache.org: data/parsepmcs.py data/reformat.py readjira.py site/addrelease.py site/chi.py site/getjson.py site/jiraversions.py Date: Thu, 02 Jul 2015 09:03:37 -0000 To: commits@community.apache.org From: sebb@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20150702090337.BB2E1AC0330@hades.apache.org> Author: sebb Date: Thu Jul 2 09:03:37 2015 New Revision: 1688786 URL: http://svn.apache.org/r1688786 Log: Fix EOLs Modified: comdev/reporter.apache.org/data/parsepmcs.py (props changed) comdev/reporter.apache.org/data/reformat.py (props changed) comdev/reporter.apache.org/readjira.py (contents, props changed) comdev/reporter.apache.org/site/addrelease.py (contents, props changed) comdev/reporter.apache.org/site/chi.py (props changed) comdev/reporter.apache.org/site/getjson.py (contents, props changed) comdev/reporter.apache.org/site/jiraversions.py (contents, props changed) Propchange: comdev/reporter.apache.org/data/parsepmcs.py ------------------------------------------------------------------------------ svn:eol-style = native Propchange: comdev/reporter.apache.org/data/reformat.py ------------------------------------------------------------------------------ svn:eol-style = native Modified: comdev/reporter.apache.org/readjira.py URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/readjira.py?rev=1688786&r1=1688785&r2=1688786&view=diff ============================================================================== --- comdev/reporter.apache.org/readjira.py (original) +++ comdev/reporter.apache.org/readjira.py Thu Jul 2 09:03:37 2015 @@ -1,43 +1,43 @@ -#!/usr/bin/env python - -import re, os, json, urllib2, base64, time -from os import listdir -from os.path import isfile, join - -mypath = "/var/www/reporter.apache.org/data/JIRA" -myfiles = [ f for f in listdir(mypath) if isfile(join(mypath,f)) ] - -jirapass = "" -with open("/var/www/reporter.apache.org/data/jirapass.txt", "r") as f: - jirapass = f.read().strip() - f.close() - -def getJIRAS(project): - refresh = True - if refresh: - base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1] - - try: - req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20""" + project + """%20AND%20created%20%3E=%20-91d""") - req.add_header("Authorization", "Basic %s" % base64string) - cdata = json.loads(urllib2.urlopen(req).read()) - req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20""" + project + """%20AND%20resolved%20%3E=%20-91d""") - req.add_header("Authorization", "Basic %s" % base64string) - rdata = json.loads(urllib2.urlopen(req).read()) - with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as f: - f.write(json.dumps([cdata['total'], rdata['total'], project])) - f.close() - return cdata['total'], rdata['total'], project - except Exception as err: - with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as f: - f.write(json.dumps([0,0,None])) - f.close() - return 0,0, None - -for project in myfiles: - jiraname = project.replace(".json", "") - if jiraname != "projects": - print("Refreshing JIRA stats for " + jiraname) - getJIRAS(jiraname) - time.sleep(2) +#!/usr/bin/env python + +import re, os, json, urllib2, base64, time +from os import listdir +from os.path import isfile, join + +mypath = "/var/www/reporter.apache.org/data/JIRA" +myfiles = [ f for f in listdir(mypath) if isfile(join(mypath,f)) ] + +jirapass = "" +with open("/var/www/reporter.apache.org/data/jirapass.txt", "r") as f: + jirapass = f.read().strip() + f.close() + +def getJIRAS(project): + refresh = True + if refresh: + base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1] + + try: + req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20""" + project + """%20AND%20created%20%3E=%20-91d""") + req.add_header("Authorization", "Basic %s" % base64string) + cdata = json.loads(urllib2.urlopen(req).read()) + req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20""" + project + """%20AND%20resolved%20%3E=%20-91d""") + req.add_header("Authorization", "Basic %s" % base64string) + rdata = json.loads(urllib2.urlopen(req).read()) + with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as f: + f.write(json.dumps([cdata['total'], rdata['total'], project])) + f.close() + return cdata['total'], rdata['total'], project + except Exception as err: + with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as f: + f.write(json.dumps([0,0,None])) + f.close() + return 0,0, None + +for project in myfiles: + jiraname = project.replace(".json", "") + if jiraname != "projects": + print("Refreshing JIRA stats for " + jiraname) + getJIRAS(jiraname) + time.sleep(2) \ No newline at end of file Propchange: comdev/reporter.apache.org/readjira.py ------------------------------------------------------------------------------ svn:eol-style = native Modified: comdev/reporter.apache.org/site/addrelease.py URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/site/addrelease.py?rev=1688786&r1=1688785&r2=1688786&view=diff ============================================================================== --- comdev/reporter.apache.org/site/addrelease.py (original) +++ comdev/reporter.apache.org/site/addrelease.py Thu Jul 2 09:03:37 2015 @@ -1,63 +1,63 @@ -#!/usr/bin/env python -import os, sys, re, json, subprocess, urllib, datetime, time -import base64, urllib2, cgi - -form = cgi.FieldStorage(); -user = os.environ['HTTP_X_AUTHENTICATED_USER'] if 'HTTP_X_AUTHENTICATED_USER' in os.environ else "nobody" -date = int(form['date'].value) if ('date' in form and len(form['date'].value) > 0) else None -version = form['version'].value if ('version' in form and len(form['version'].value) > 0) else None -committee = form['committee'].value if 'committee' in form else None -dojson = form['json'].value if 'json' in form else None - -def getPMCs(uid): - groups = [] - ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '(|(memberUid=%s)(member=uid=%s,ou=people,dc=apache,dc=org))' % (uid, uid), 'cn']) - picked = {} - for match in re.finditer(r"dn: cn=([a-zA-Z0-9]+),ou=pmc,ou=committees,ou=groups,dc=apache,dc=org", ldapdata): - group = match.group(1) - if group != "incubator": - groups.append(group) - return groups - - -def isMember(uid): - members = [] - ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '-b', 'cn=member,ou=groups,dc=apache,dc=org']) - for match in re.finditer(r"memberUid: ([-a-z0-9_.]+)", ldapdata): - group = match.group(1) - members.append(group) - if uid in members: - return True - return False - -def getReleaseData(committee): - try: - with open("/var/www/reporter.apache.org/data/releases/%s.json" % committee, "r") as f: - x = json.loads(f.read()) - f.close() - return x; - except: - return {} - -saved = False -committees = getPMCs(user) -if date and version and committee: - if committee in committees or isMember(user): - rdata = getReleaseData(committee) - rdata[version] = date - with open("/var/www/reporter.apache.org/data/releases/%s.json" % committee, "w") as f: - f.write(json.dumps(rdata)) - f.close() - saved = True - if dojson: - print("Content-Type: application/json\r\n\r\n") - print(json.dumps({'versions': rdata})) - else: - print("Content-Type: text/html\r\n\r\n

Data submitted!

You may see the updated committee data at: https://reporter.apache.org/?%s." % (committee, committee)) - -if not saved: - if dojson: - print("Content-Type: application/json\r\n\r\n{\"error\": \"Not saved\"}") - else: - print("Content-Type: text/plain\r\n\r\nCould not save. Make sure you have filled out all fields and have access to this committee data! For further inquiries, please contact dev@community.apache.org") +#!/usr/bin/env python +import os, sys, re, json, subprocess, urllib, datetime, time +import base64, urllib2, cgi + +form = cgi.FieldStorage(); +user = os.environ['HTTP_X_AUTHENTICATED_USER'] if 'HTTP_X_AUTHENTICATED_USER' in os.environ else "nobody" +date = int(form['date'].value) if ('date' in form and len(form['date'].value) > 0) else None +version = form['version'].value if ('version' in form and len(form['version'].value) > 0) else None +committee = form['committee'].value if 'committee' in form else None +dojson = form['json'].value if 'json' in form else None + +def getPMCs(uid): + groups = [] + ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '(|(memberUid=%s)(member=uid=%s,ou=people,dc=apache,dc=org))' % (uid, uid), 'cn']) + picked = {} + for match in re.finditer(r"dn: cn=([a-zA-Z0-9]+),ou=pmc,ou=committees,ou=groups,dc=apache,dc=org", ldapdata): + group = match.group(1) + if group != "incubator": + groups.append(group) + return groups + + +def isMember(uid): + members = [] + ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '-b', 'cn=member,ou=groups,dc=apache,dc=org']) + for match in re.finditer(r"memberUid: ([-a-z0-9_.]+)", ldapdata): + group = match.group(1) + members.append(group) + if uid in members: + return True + return False + +def getReleaseData(committee): + try: + with open("/var/www/reporter.apache.org/data/releases/%s.json" % committee, "r") as f: + x = json.loads(f.read()) + f.close() + return x; + except: + return {} + +saved = False +committees = getPMCs(user) +if date and version and committee: + if committee in committees or isMember(user): + rdata = getReleaseData(committee) + rdata[version] = date + with open("/var/www/reporter.apache.org/data/releases/%s.json" % committee, "w") as f: + f.write(json.dumps(rdata)) + f.close() + saved = True + if dojson: + print("Content-Type: application/json\r\n\r\n") + print(json.dumps({'versions': rdata})) + else: + print("Content-Type: text/html\r\n\r\n

Data submitted!

You may see the updated committee data at: https://reporter.apache.org/?%s." % (committee, committee)) + +if not saved: + if dojson: + print("Content-Type: application/json\r\n\r\n{\"error\": \"Not saved\"}") + else: + print("Content-Type: text/plain\r\n\r\nCould not save. Make sure you have filled out all fields and have access to this committee data! For further inquiries, please contact dev@community.apache.org") \ No newline at end of file Propchange: comdev/reporter.apache.org/site/addrelease.py ------------------------------------------------------------------------------ svn:eol-style = native Propchange: comdev/reporter.apache.org/site/chi.py ------------------------------------------------------------------------------ svn:eol-style = native Modified: comdev/reporter.apache.org/site/getjson.py URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/site/getjson.py?rev=1688786&r1=1688785&r2=1688786&view=diff ============================================================================== --- comdev/reporter.apache.org/site/getjson.py (original) +++ comdev/reporter.apache.org/site/getjson.py Thu Jul 2 09:03:37 2015 @@ -1,311 +1,311 @@ -#!/usr/bin/env python -import os, sys, re, json, subprocess, urllib, time -import base64, urllib2, cgi - -form = cgi.FieldStorage(); -oproject = form['only'].value if ('only' in form and len(form['only'].value) > 0) else None - - -jmap = { - 'trafficserver': ['TS'], - 'cordova': ['CB'], - 'corinthia': ['COR'] -} - -pmap = { - 'community': 'comdev', - 'ws': 'webservices' -} - -ldapmap = { - 'webservices': 'ws' -} - -jirapass = "" -with open("/var/www/reporter.apache.org/data/jirapass.txt", "r") as f: - jirapass = f.read().strip() - f.close() - -def getPMCs(uid): - groups = [] - ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '(|(memberUid=%s)(member=uid=%s,ou=people,dc=apache,dc=org))' % (uid, uid), 'cn']) - picked = {} - for match in re.finditer(r"dn: cn=([a-zA-Z0-9]+),ou=pmc,ou=committees,ou=groups,dc=apache,dc=org", ldapdata): - group = match.group(1) - if group != "incubator": - - groups.append(group) - return groups - - -def isMember(uid): - members = [] - ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '-b', 'cn=member,ou=groups,dc=apache,dc=org']) - for match in re.finditer(r"memberUid: ([-a-z0-9_.]+)", ldapdata): - group = match.group(1) - members.append(group) - if uid in members: - return True - return False - -def getJIRAProjects(project): - project = project.replace("Apache ", "").strip().lower() - refresh = True - x = {} - jiras = [] - try: - mtime = 0 - try: - st=os.stat("/var/www/reporter.apache.org/data/JIRA/projects.json") - mtime=st.st_mtime - except: - pass - if mtime >= (time.time() - 86400): - refresh = False - with open("/var/www/reporter.apache.org/data/JIRA/projects.json", "r") as f: - x = json.loads(f.read()) - f.close() - else: - base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1] - - try: - req = req = urllib2.Request("https://issues.apache.org/jira/rest/api/2/project.json") - req.add_header("Authorization", "Basic %s" % base64string) - x = json.loads(urllib2.urlopen(req).read()) - with open("/var/www/reporter.apache.org/data/JIRA/projects.json", "w") as f: - f.write(json.dumps(x)) - f.close() - except: - pass - except: - pass - - for entry in x: - if entry['name'].replace("Apache ", "").strip().lower() == project: - jiras.append(entry['key']) - elif 'projectCategory' in entry and entry['projectCategory']['name'].replace("Apache ", "").strip().lower() == project: - jiras.append(entry['key']) - return jiras - -def getJIRAS(project): - refresh = True - try: - st=os.stat("/var/www/reporter.apache.org/data/JIRA/%s.json" % project) - mtime=st.st_mtime - if mtime >= (time.time() - (2*86400)): - refresh = False - with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "r") as f: - x = json.loads(f.read()) - f.close() - return x[0], x[1], x[2] - except: - pass - - if refresh: - base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1] - - try: - req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20""" + project + """%20AND%20created%20%3E=%20-91d""") - req.add_header("Authorization", "Basic %s" % base64string) - cdata = json.loads(urllib2.urlopen(req).read()) - req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20""" + project + """%20AND%20resolved%20%3E=%20-91d""") - req.add_header("Authorization", "Basic %s" % base64string) - rdata = json.loads(urllib2.urlopen(req).read()) - with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as f: - f.write(json.dumps([cdata['total'], rdata['total'], project])) - f.close() - return cdata['total'], rdata['total'], project - except Exception as err: - with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as f: - f.write(json.dumps([0,0,None])) - f.close() - return 0,0, None - -def getProjectData(project): - try: - y = [] - with open("/var/www/projects.apache.org/site/json/projects/%s.json" % project, "r") as f: - x = json.loads(f.read()) - f.close() - with open("/var/www/projects.apache.org/site/json/foundation/pmcs.json", "r") as f: - p = json.loads(f.read()) - f.close() - for xproject in p: - y.append(xproject) - if xproject == project: - x['name'] = p[project]['name'] - with open("/var/www/projects.apache.org/site/json/foundation/chairs.json", "r") as f: - c = json.loads(f.read()) - f.close() - for xproject in c: - if xproject.lower() == x['name'].lower(): - x['chair'] = c[xproject] - z = {} - with open("/var/www/reporter.apache.org/data/health.json", "r") as f: - h = json.loads(f.read()) - f.close() - z = {} - for entry in h: - if entry['group'] == project: - z = entry - - return x, y, z; - except: - x = {} - y = [] - with open("/var/www/projects.apache.org/site/json/foundation/pmcs.json", "r") as f: - p = json.loads(f.read()) - f.close() - for xproject in p: - y.append(xproject) - if xproject == project: - x['name'] = p[project]['name'] - - with open("/var/www/projects.apache.org/site/json/foundation/chairs.json", "r") as f: - c = json.loads(f.read()) - f.close() - for xproject in c: - if 'name' in x and xproject == x['name']: - x['chair'] = c[xproject] - z = {} - with open("/var/www/reporter.apache.org/data/health.json", "r") as f: - h = json.loads(f.read()) - f.close() - z = {} - for entry in h: - if entry['group'] == project: - z = entry - return x,y,z - -def getReleaseData(project): - try: - with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "r") as f: - x = json.loads(f.read()) - f.close() - return x; - except: - return {} - - - -pchanges = {} -cchanges = {} - -with open("/var/www/reporter.apache.org/data/pmcs.json", "r") as f: - pchanges = json.loads(f.read()) - f.close() - -with open("/var/www/reporter.apache.org/data/projects.json", "r") as f: - cchanges = json.loads(f.read()) - f.close() - - -user = os.environ['HTTP_X_AUTHENTICATED_USER'] if 'HTTP_X_AUTHENTICATED_USER' in os.environ else "" -m = re.match(r"^([-a-zA-Z0-9_.]+)$", user) -groups = [] - -cdata = {} -if m: - uid = m.group(1) - groups = getPMCs(uid) - include = os.environ['QUERY_STRING'] if 'QUERY_STRING' in os.environ else None - if include and isMember(uid) and not include in groups and len(include) > 1: - groups.append(include) - if oproject and len(oproject) > 0 and isMember(uid): - groups = [oproject] - mlstats = {} - with open("/var/www/reporter.apache.org/data/mailinglists.json", "r") as f: - ml = json.loads(f.read()) - f.close() - for entry in ml: - tlp = entry.split(".")[0] - if tlp in pmap: - tlp = pmap[tlp] - if tlp in groups: - mlstats[tlp] = mlstats[tlp] if tlp in mlstats else {} - mlstats[tlp][entry] = ml[entry] - emails = {} - with open("/var/www/reporter.apache.org/data/maildata_extended.json", "r") as f: - mld = json.loads(f.read()) - f.close() - for entry in mld: - tlp = entry.split("-")[0] - if tlp in pmap: - tlp = pmap[tlp] - if tlp in groups: - emails[tlp] = emails[tlp] if tlp in emails else {} - emails[tlp][entry] = mld[entry] - jdata = {} - ddata = {} - rdata = {} - allpmcs = [] - keys = {} - count = {} - health = {} - for group in groups: - jiras = [] - count[group] = [0,0] - xgroup = group - if group in ldapmap: - xgroup = ldapmap[group] - if xgroup in pchanges: - count[group][0] = len(pchanges[xgroup]) - if xgroup in cchanges: - count[group][1] = len(cchanges[xgroup]) - jdata[group] = [0,0, None] - ddata[group], allpmcs, phealth = getProjectData(group) - health[group] = phealth - rdata[group] = getReleaseData(group) - jiraname = group.upper() - if group in jmap: - for jiraname in jmap[group]: - x,y, p = getJIRAS(jiraname) - jdata[group][0] += x - jdata[group][1] += y - jdata[group][2] = p - elif group in ddata and 'name' in ddata[group]: - jiras = getJIRAProjects(ddata[group]['name']) - keys[group] = jiras - for jiraname in jiras: - x,y, p= getJIRAS(jiraname) - jdata[group][0] += x - jdata[group][1] += y - jdata[group][2] = p - elif jiraname: - x,y, p= getJIRAS(jiraname) - jdata[group][0] += x - jdata[group][1] += y - jdata[group][2] = p - - cdata[group] = cdata[xgroup] if xgroup in cdata else {'pmc': {}, 'committer': {}} - for pmc in pchanges: - if pmc == xgroup: - for member in pchanges[pmc]: - if pchanges[pmc][member][1] > 0: - cdata[group]['pmc'][member] = pchanges[pmc][member] - for pmc in cchanges: - if pmc == xgroup: - for member in cchanges[pmc]: - if cchanges[pmc][member][1] > 0: - cdata[group]['committer'][member] = cchanges[pmc][member] - if not isMember(uid): - allpmcs = [] - output = { - 'count': count, - 'pmcs': groups, - 'all': allpmcs, - 'mail': mlstats, - 'delivery': emails, - 'jira': jdata, - 'changes': cdata, - 'pdata': ddata, - 'releases': rdata, - 'keys': keys, - 'health': health - } - dump = json.dumps(output) - print ("Content-Type: application/json\r\nContent-Length: %u\r\n\r\n" % (len(dump)+1)) - print(dump) -else: - print ("Content-Type: text/html\r\n\r\n") - print("Unknown or invalid user id presented") +#!/usr/bin/env python +import os, sys, re, json, subprocess, urllib, time +import base64, urllib2, cgi + +form = cgi.FieldStorage(); +oproject = form['only'].value if ('only' in form and len(form['only'].value) > 0) else None + + +jmap = { + 'trafficserver': ['TS'], + 'cordova': ['CB'], + 'corinthia': ['COR'] +} + +pmap = { + 'community': 'comdev', + 'ws': 'webservices' +} + +ldapmap = { + 'webservices': 'ws' +} + +jirapass = "" +with open("/var/www/reporter.apache.org/data/jirapass.txt", "r") as f: + jirapass = f.read().strip() + f.close() + +def getPMCs(uid): + groups = [] + ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '(|(memberUid=%s)(member=uid=%s,ou=people,dc=apache,dc=org))' % (uid, uid), 'cn']) + picked = {} + for match in re.finditer(r"dn: cn=([a-zA-Z0-9]+),ou=pmc,ou=committees,ou=groups,dc=apache,dc=org", ldapdata): + group = match.group(1) + if group != "incubator": + + groups.append(group) + return groups + + +def isMember(uid): + members = [] + ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '-b', 'cn=member,ou=groups,dc=apache,dc=org']) + for match in re.finditer(r"memberUid: ([-a-z0-9_.]+)", ldapdata): + group = match.group(1) + members.append(group) + if uid in members: + return True + return False + +def getJIRAProjects(project): + project = project.replace("Apache ", "").strip().lower() + refresh = True + x = {} + jiras = [] + try: + mtime = 0 + try: + st=os.stat("/var/www/reporter.apache.org/data/JIRA/projects.json") + mtime=st.st_mtime + except: + pass + if mtime >= (time.time() - 86400): + refresh = False + with open("/var/www/reporter.apache.org/data/JIRA/projects.json", "r") as f: + x = json.loads(f.read()) + f.close() + else: + base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1] + + try: + req = req = urllib2.Request("https://issues.apache.org/jira/rest/api/2/project.json") + req.add_header("Authorization", "Basic %s" % base64string) + x = json.loads(urllib2.urlopen(req).read()) + with open("/var/www/reporter.apache.org/data/JIRA/projects.json", "w") as f: + f.write(json.dumps(x)) + f.close() + except: + pass + except: + pass + + for entry in x: + if entry['name'].replace("Apache ", "").strip().lower() == project: + jiras.append(entry['key']) + elif 'projectCategory' in entry and entry['projectCategory']['name'].replace("Apache ", "").strip().lower() == project: + jiras.append(entry['key']) + return jiras + +def getJIRAS(project): + refresh = True + try: + st=os.stat("/var/www/reporter.apache.org/data/JIRA/%s.json" % project) + mtime=st.st_mtime + if mtime >= (time.time() - (2*86400)): + refresh = False + with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "r") as f: + x = json.loads(f.read()) + f.close() + return x[0], x[1], x[2] + except: + pass + + if refresh: + base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1] + + try: + req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20""" + project + """%20AND%20created%20%3E=%20-91d""") + req.add_header("Authorization", "Basic %s" % base64string) + cdata = json.loads(urllib2.urlopen(req).read()) + req = req = urllib2.Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project%20=%20""" + project + """%20AND%20resolved%20%3E=%20-91d""") + req.add_header("Authorization", "Basic %s" % base64string) + rdata = json.loads(urllib2.urlopen(req).read()) + with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as f: + f.write(json.dumps([cdata['total'], rdata['total'], project])) + f.close() + return cdata['total'], rdata['total'], project + except Exception as err: + with open("/var/www/reporter.apache.org/data/JIRA/%s.json" % project, "w") as f: + f.write(json.dumps([0,0,None])) + f.close() + return 0,0, None + +def getProjectData(project): + try: + y = [] + with open("/var/www/projects.apache.org/site/json/projects/%s.json" % project, "r") as f: + x = json.loads(f.read()) + f.close() + with open("/var/www/projects.apache.org/site/json/foundation/pmcs.json", "r") as f: + p = json.loads(f.read()) + f.close() + for xproject in p: + y.append(xproject) + if xproject == project: + x['name'] = p[project]['name'] + with open("/var/www/projects.apache.org/site/json/foundation/chairs.json", "r") as f: + c = json.loads(f.read()) + f.close() + for xproject in c: + if xproject.lower() == x['name'].lower(): + x['chair'] = c[xproject] + z = {} + with open("/var/www/reporter.apache.org/data/health.json", "r") as f: + h = json.loads(f.read()) + f.close() + z = {} + for entry in h: + if entry['group'] == project: + z = entry + + return x, y, z; + except: + x = {} + y = [] + with open("/var/www/projects.apache.org/site/json/foundation/pmcs.json", "r") as f: + p = json.loads(f.read()) + f.close() + for xproject in p: + y.append(xproject) + if xproject == project: + x['name'] = p[project]['name'] + + with open("/var/www/projects.apache.org/site/json/foundation/chairs.json", "r") as f: + c = json.loads(f.read()) + f.close() + for xproject in c: + if 'name' in x and xproject == x['name']: + x['chair'] = c[xproject] + z = {} + with open("/var/www/reporter.apache.org/data/health.json", "r") as f: + h = json.loads(f.read()) + f.close() + z = {} + for entry in h: + if entry['group'] == project: + z = entry + return x,y,z + +def getReleaseData(project): + try: + with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "r") as f: + x = json.loads(f.read()) + f.close() + return x; + except: + return {} + + + +pchanges = {} +cchanges = {} + +with open("/var/www/reporter.apache.org/data/pmcs.json", "r") as f: + pchanges = json.loads(f.read()) + f.close() + +with open("/var/www/reporter.apache.org/data/projects.json", "r") as f: + cchanges = json.loads(f.read()) + f.close() + + +user = os.environ['HTTP_X_AUTHENTICATED_USER'] if 'HTTP_X_AUTHENTICATED_USER' in os.environ else "" +m = re.match(r"^([-a-zA-Z0-9_.]+)$", user) +groups = [] + +cdata = {} +if m: + uid = m.group(1) + groups = getPMCs(uid) + include = os.environ['QUERY_STRING'] if 'QUERY_STRING' in os.environ else None + if include and isMember(uid) and not include in groups and len(include) > 1: + groups.append(include) + if oproject and len(oproject) > 0 and isMember(uid): + groups = [oproject] + mlstats = {} + with open("/var/www/reporter.apache.org/data/mailinglists.json", "r") as f: + ml = json.loads(f.read()) + f.close() + for entry in ml: + tlp = entry.split(".")[0] + if tlp in pmap: + tlp = pmap[tlp] + if tlp in groups: + mlstats[tlp] = mlstats[tlp] if tlp in mlstats else {} + mlstats[tlp][entry] = ml[entry] + emails = {} + with open("/var/www/reporter.apache.org/data/maildata_extended.json", "r") as f: + mld = json.loads(f.read()) + f.close() + for entry in mld: + tlp = entry.split("-")[0] + if tlp in pmap: + tlp = pmap[tlp] + if tlp in groups: + emails[tlp] = emails[tlp] if tlp in emails else {} + emails[tlp][entry] = mld[entry] + jdata = {} + ddata = {} + rdata = {} + allpmcs = [] + keys = {} + count = {} + health = {} + for group in groups: + jiras = [] + count[group] = [0,0] + xgroup = group + if group in ldapmap: + xgroup = ldapmap[group] + if xgroup in pchanges: + count[group][0] = len(pchanges[xgroup]) + if xgroup in cchanges: + count[group][1] = len(cchanges[xgroup]) + jdata[group] = [0,0, None] + ddata[group], allpmcs, phealth = getProjectData(group) + health[group] = phealth + rdata[group] = getReleaseData(group) + jiraname = group.upper() + if group in jmap: + for jiraname in jmap[group]: + x,y, p = getJIRAS(jiraname) + jdata[group][0] += x + jdata[group][1] += y + jdata[group][2] = p + elif group in ddata and 'name' in ddata[group]: + jiras = getJIRAProjects(ddata[group]['name']) + keys[group] = jiras + for jiraname in jiras: + x,y, p= getJIRAS(jiraname) + jdata[group][0] += x + jdata[group][1] += y + jdata[group][2] = p + elif jiraname: + x,y, p= getJIRAS(jiraname) + jdata[group][0] += x + jdata[group][1] += y + jdata[group][2] = p + + cdata[group] = cdata[xgroup] if xgroup in cdata else {'pmc': {}, 'committer': {}} + for pmc in pchanges: + if pmc == xgroup: + for member in pchanges[pmc]: + if pchanges[pmc][member][1] > 0: + cdata[group]['pmc'][member] = pchanges[pmc][member] + for pmc in cchanges: + if pmc == xgroup: + for member in cchanges[pmc]: + if cchanges[pmc][member][1] > 0: + cdata[group]['committer'][member] = cchanges[pmc][member] + if not isMember(uid): + allpmcs = [] + output = { + 'count': count, + 'pmcs': groups, + 'all': allpmcs, + 'mail': mlstats, + 'delivery': emails, + 'jira': jdata, + 'changes': cdata, + 'pdata': ddata, + 'releases': rdata, + 'keys': keys, + 'health': health + } + dump = json.dumps(output) + print ("Content-Type: application/json\r\nContent-Length: %u\r\n\r\n" % (len(dump)+1)) + print(dump) +else: + print ("Content-Type: text/html\r\n\r\n") + print("Unknown or invalid user id presented") Propchange: comdev/reporter.apache.org/site/getjson.py ------------------------------------------------------------------------------ svn:eol-style = native Modified: comdev/reporter.apache.org/site/jiraversions.py URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/site/jiraversions.py?rev=1688786&r1=1688785&r2=1688786&view=diff ============================================================================== --- comdev/reporter.apache.org/site/jiraversions.py (original) +++ comdev/reporter.apache.org/site/jiraversions.py Thu Jul 2 09:03:37 2015 @@ -1,73 +1,73 @@ -#!/usr/bin/env python -import os, sys, json, urllib2, re, time, base64, cgi, subprocess - -form = cgi.FieldStorage(); -user = os.environ['HTTP_X_AUTHENTICATED_USER'] if 'HTTP_X_AUTHENTICATED_USER' in os.environ else "nobody" -project = form['project'].value if ('project' in form and len(form['project'].value) > 0) else None -jiraname = form['jiraname'].value if ('jiraname' in form and len(form['jiraname'].value) > 0) else None -prepend = form['prepend'].value if ('prepend' in form and len(form['prepend'].value) > 0) else None - -def getPMCs(uid): - groups = [] - ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '(|(memberUid=%s)(member=uid=%s,ou=people,dc=apache,dc=org))' % (uid, uid), 'cn']) - picked = {} - for match in re.finditer(r"dn: cn=([a-zA-Z0-9]+),ou=pmc,ou=committees,ou=groups,dc=apache,dc=org", ldapdata): - group = match.group(1) - if group != "incubator": - - groups.append(group) - return groups - - -def getReleaseData(project): - try: - with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "r") as f: - x = json.loads(f.read()) - f.close() - return x; - except: - return {} - - -def isMember(uid): - members = [] - ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '-b', 'cn=member,ou=groups,dc=apache,dc=org']) - for match in re.finditer(r"memberUid: ([-a-z0-9_.]+)", ldapdata): - group = match.group(1) - members.append(group) - if uid in members: - return True - return False - - -jirapass = "" -with open("/var/www/reporter.apache.org/data/jirapass.txt", "r") as f: - jirapass = f.read().strip() - f.close() - -groups = getPMCs(user) -if (isMember(user) or project in groups) and jiraname: - jiraname = jiraname.upper() - base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1] - rdata = getReleaseData(project) - try: - req = req = urllib2.Request("https://issues.apache.org/jira/rest/api/2/project/%s/versions" % jiraname) - req.add_header("Authorization", "Basic %s" % base64string) - cdata = json.loads(urllib2.urlopen(req).read()) - for entry in cdata: - if ('name' in entry and 'releaseDate' in entry and 'released' in entry and entry['released']): - date = time.mktime(time.strptime(entry['releaseDate'], "%Y-%m-%d")) - if prepend: - entry['name'] = "%s-%s" % (prepend, entry['name']) - rdata[entry['name']] = date - except Exception as err: - pass - with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "w") as f: - f.write(json.dumps(rdata)) - f.close() - - print("Content-Type: application/json\r\n\r\n") - print(json.dumps({'status': 'Fetched', 'versions': rdata})) - -else: - print("Content-Type: application/json\r\n\r\n{\"status\": \"Data missing\"}\r\n") +#!/usr/bin/env python +import os, sys, json, urllib2, re, time, base64, cgi, subprocess + +form = cgi.FieldStorage(); +user = os.environ['HTTP_X_AUTHENTICATED_USER'] if 'HTTP_X_AUTHENTICATED_USER' in os.environ else "nobody" +project = form['project'].value if ('project' in form and len(form['project'].value) > 0) else None +jiraname = form['jiraname'].value if ('jiraname' in form and len(form['jiraname'].value) > 0) else None +prepend = form['prepend'].value if ('prepend' in form and len(form['prepend'].value) > 0) else None + +def getPMCs(uid): + groups = [] + ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '(|(memberUid=%s)(member=uid=%s,ou=people,dc=apache,dc=org))' % (uid, uid), 'cn']) + picked = {} + for match in re.finditer(r"dn: cn=([a-zA-Z0-9]+),ou=pmc,ou=committees,ou=groups,dc=apache,dc=org", ldapdata): + group = match.group(1) + if group != "incubator": + + groups.append(group) + return groups + + +def getReleaseData(project): + try: + with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "r") as f: + x = json.loads(f.read()) + f.close() + return x; + except: + return {} + + +def isMember(uid): + members = [] + ldapdata = subprocess.check_output(['ldapsearch', '-x', '-LLL', '-b', 'cn=member,ou=groups,dc=apache,dc=org']) + for match in re.finditer(r"memberUid: ([-a-z0-9_.]+)", ldapdata): + group = match.group(1) + members.append(group) + if uid in members: + return True + return False + + +jirapass = "" +with open("/var/www/reporter.apache.org/data/jirapass.txt", "r") as f: + jirapass = f.read().strip() + f.close() + +groups = getPMCs(user) +if (isMember(user) or project in groups) and jiraname: + jiraname = jiraname.upper() + base64string = base64.encodestring('%s:%s' % ('githubbot', jirapass))[:-1] + rdata = getReleaseData(project) + try: + req = req = urllib2.Request("https://issues.apache.org/jira/rest/api/2/project/%s/versions" % jiraname) + req.add_header("Authorization", "Basic %s" % base64string) + cdata = json.loads(urllib2.urlopen(req).read()) + for entry in cdata: + if ('name' in entry and 'releaseDate' in entry and 'released' in entry and entry['released']): + date = time.mktime(time.strptime(entry['releaseDate'], "%Y-%m-%d")) + if prepend: + entry['name'] = "%s-%s" % (prepend, entry['name']) + rdata[entry['name']] = date + except Exception as err: + pass + with open("/var/www/reporter.apache.org/data/releases/%s.json" % project, "w") as f: + f.write(json.dumps(rdata)) + f.close() + + print("Content-Type: application/json\r\n\r\n") + print(json.dumps({'status': 'Fetched', 'versions': rdata})) + +else: + print("Content-Type: application/json\r\n\r\n{\"status\": \"Data missing\"}\r\n") Propchange: comdev/reporter.apache.org/site/jiraversions.py ------------------------------------------------------------------------------ svn:eol-style = native