Author: humbedooh
Date: Fri Aug 9 16:42:38 2019
New Revision: 1864822
URL: http://svn.apache.org/viewvc?rev=1864822&view=rev
Log:
move kibble into wsgi so we can also tweak mailing list stats etc atl
Added:
comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py (with props)
Modified:
comdev/reporter.apache.org/trunk/scripts/pdata.py
Modified: comdev/reporter.apache.org/trunk/scripts/pdata.py
URL:
http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/pdata.py?rev=1864822&r1=1864821&r2=1864822&view=diff
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/pdata.py (original)
+++ comdev/reporter.apache.org/trunk/scripts/pdata.py Fri Aug 9 16:42:38 2019
@@ -23,6 +23,7 @@
import os, sys, re, json, subprocess, time
import base64, requests
+import rapp.kibble
CACHE_TIMEOUT = 14400
@@ -278,6 +279,10 @@ def generate(user, project, runkibble):
lastRead = time.time()
emails = {}
+ mlid = project # mailing list ID, might differ from ldap id
+ for k, v in pmap.items():
+ if v == project:
+ mlid = k
for entry in mld: # e.g. hc-dev, ant-users, ws-dev
tlp = entry.split("-")[0]
@@ -285,9 +290,7 @@ def generate(user, project, runkibble):
if tlp == "empire":
tlp = "empire-db"
nentry = entry.replace("empire-", "empire-db-")
- if tlp in pmap: # convert ml prefix to PMC internal name
- tlp = pmap[tlp]
- if tlp == project:
+ if tlp == mlid:
emails[nentry] = mld[entry]
dates = {}
@@ -373,19 +376,9 @@ def generate(user, project, runkibble):
kibble = None
if runkibble:
try:
- xenv = os.environ.copy()
- if 'SCRIPT_NAME' in xenv:
- del xenv['SCRIPT_NAME']
- cmd = ('%s/site/wizard/kibble.py' % RAOHOME_FULL, project)
- if jdata and jdata[2]:
- cmd += tuple(jdata[2])
- txt = subprocess.check_output(cmd, env = xenv)
- if type(txt) is bytes:
- txt = txt.decode('ascii')
- kibble = json.loads(txt)
- except subprocess.CalledProcessError as e:
- return None
-
+ kibble = rapp.kibble.stats(project, jira = jdata[2], mlid =
mlid)
+ except:
+ pass
output = {
'count': count,
Added: comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py
URL:
http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py?rev=1864822&view=auto
==============================================================================
--- comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py (added)
+++ comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py Fri Aug 9 16:42:38
2019
@@ -0,0 +1,331 @@
+#!/usr/bin/env python3
+""" Script for fetching somekibble data for reports """
+import requests
+import json
+import time
+import re
+import os
+
+RAO_HOME = '/var/www/reporter.apache.org'
+TOKEN = open('%s/data/kibble-token.txt' % RAO_HOME).read().strip()
+
+def stats(project, jira = [], mlid = None):
+ BEFORE = int(time.time() - (91*86400))
+ if not mlid:
+ mlid = project
+
+ # Check for cache?
+ if not re.match(r"^[-_.a-z0-9]+$", project):
+ project = "bogus"
+ cache_file = '/tmp/kibble-cache-%s.json' % project
+ runit = True
+ js = {}
+ if (os.path.exists(cache_file) and os.path.getmtime(cache_file) >
(time.time() - 86400)):
+ output = open(cache_file, "r").read()
+ try:
+ js = json.loads(output)
+ assert('prs' in js)
+ runit = False
+ except:
+ pass
+ if runit:
+ # Issues/PRs
+ issues =
requests.post('https://demo.kibble.apache.org/api/issue/issues',
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Kibble-Token': TOKEN,
+ },
+ json = {
+ "page":"issues",
+ "quick":True,
+ "interval": "week",
+ "subfilter":"/(?:incubator-)?" + project + ".*\\.git",
+ "distinguish":True
+ }
+ ).json();
+ after = [x for x in issues['timeseries'] if x['date'] > BEFORE]
+ before = [x for x in issues['timeseries'] if x['date'] <= BEFORE]
+ pro_before = 0
+ prc_before = 0
+ for month in before:
+ pro_before += month.get('pull requests opened', 0)
+ prc_before += month.get('pull requests closed', 0)
+ pro_after = 0
+ prc_after = 0
+ for month in after:
+ pro_after += month.get('pull requests opened', 0)
+ prc_after += month.get('pull requests closed', 0)
+ pro_change = '%u%%' % int((pro_after - pro_before) / (pro_before or 1)
* 100)
+ prc_change = '%u%%' % int((prc_after - prc_before) / (prc_before or 1)
* 100)
+
+
+ iso_before = 0
+ isc_before = 0
+ for month in before:
+ iso_before += month.get('issues opened', 0)
+ isc_before += month.get('issues closed', 0)
+ iso_after = 0
+ isc_after = 0
+ for month in after:
+ iso_after += month.get('issues opened', 0)
+ isc_after += month.get('issues closed', 0)
+ iso_change = '%u%%' % int((iso_after - iso_before) / (iso_before or 1)
* 100)
+ isc_change = '%u%%' % int((isc_after - isc_before) / (isc_before or 1)
* 100)
+
+ github_ts = issues['timeseries']
+
+ # Busiest GH issues/PRs
+ bissues = requests.post('https://demo.kibble.apache.org/api/issue/top',
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Kibble-Token': TOKEN,
+ },
+ json = {
+ "quick":True,
+ "interval": "month",
+ "from": BEFORE,
+ "to": int(time.time()),
+ "subfilter":"/(?:incubator-)?" + project + ".*\\.git",
+ }
+ ).json();
+
+ # JIRA ??
+ jio_before = 0
+ jic_before = 0
+ jio_after = 0
+ jic_after = 0
+ jira_ts = []
+ if jira:
+ issues =
requests.post('https://demo.kibble.apache.org/api/issue/issues',
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Kibble-Token': TOKEN,
+ },
+ json = {
+ "quick":True,
+ "interval": "week",
+ "subfilter":"/browse/(" + "|".join(jira) + ")$",
+ }
+ ).json();
+ after = [x for x in issues['timeseries'] if x['date'] > BEFORE]
+ before = [x for x in issues['timeseries'] if x['date'] <= BEFORE]
+
+ for month in before:
+ jio_before += month.get('issues opened', 0)
+ jic_before += month.get('issues closed', 0)
+ for month in after:
+ jio_after += month.get('issues opened', 0)
+ jic_after += month.get('issues closed', 0)
+ jira_ts = issues['timeseries']
+
+ jio_change = '%u%%' % int((jio_after - jio_before) / (jio_before or 1)
* 100)
+ jic_change = '%u%%' % int((jic_after - jic_before) / (jic_before or 1)
* 100)
+
+
+ # Busiest JIRAs
+ bjiras = []
+ if jira:
+ bjiras =
requests.post('https://demo.kibble.apache.org/api/issue/top',
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Kibble-Token': TOKEN,
+ },
+ json = {
+ "quick":True,
+ "interval": "month",
+ "from": BEFORE,
+ "to": int(time.time()),
+ "subfilter":"/browse/(" + "|".join(jira) + ")$",
+ }
+ ).json();
+
+
+ # Commits
+ commits =
requests.post('https://demo.kibble.apache.org/api/code/commits',
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Kibble-Token': TOKEN,
+ },
+ json = {
+ "page":"issues",
+ "quick":True,
+ "interval": "week",
+ "subfilter":"/(?:incubator-)?" + project + ".*\\.git",
+ }
+ ).json();
+ after = [x for x in commits['timeseries'] if x['date'] > BEFORE]
+ before = [x for x in commits['timeseries'] if x['date'] <= BEFORE]
+ cmt_before = 0
+ for month in before:
+ cmt_before += month.get('commits', 0)
+ cmt_after = 0
+ for month in after:
+ cmt_after += month.get('commits', 0)
+ cmt_change = '%u%%' % int((cmt_after - cmt_before) / (cmt_before or 1)
* 100)
+ commit_ts = commits['timeseries']
+
+ # Committers
+ authors_b =
requests.post('https://demo.kibble.apache.org/api/code/committers',
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Kibble-Token': TOKEN,
+ },
+ json = {
+ "page":"issues",
+ "quick":True,
+ "from": int(BEFORE - (90*86400)),
+ "to": BEFORE,
+ "interval": "99999d",
+ "subfilter":"/(?:incubator-)?" + project + ".*\\.git",
+ }
+ ).json()
+ authors_a =
requests.post('https://demo.kibble.apache.org/api/code/committers',
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Kibble-Token': TOKEN,
+ },
+ json = {
+ "page":"issues",
+ "quick":True,
+ "from": BEFORE,
+ "to": int(time.time()),
+ "interval": "99999d",
+ "subfilter":"/(?:incubator-)?" + project + ".*\\.git",
+ }
+ ).json();
+ after = authors_a['timeseries']
+ before = authors_b['timeseries']
+ cmtp_before = 0
+ for month in before:
+ cmtp_before += month.get('authors', 0)
+ cmtp_after = 0
+ for month in after:
+ cmtp_after += month.get('authors', 0)
+ cmtp_change = '%u%%' % int((cmtp_after - cmtp_before) / (cmtp_before
or 1) * 100)
+
+
+ # Dev+user list traffic?
+ dev_ts = []
+ user_ts = []
+ if True:
+ ml = r'dev@%s\.apache\.org' % mlid
+ emails =
requests.post('https://demo.kibble.apache.org/api/mail/timeseries-single',
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Kibble-Token': TOKEN,
+ },
+ json = {
+ "quick":True,
+ "interval": "week",
+ "subfilter":"\?%s$" % ml,
+ }
+ ).json()
+ dev_ts = emails['timeseries']
+
+ ml = r'users?@%s\.apache\.org' % mlid
+ emails =
requests.post('https://demo.kibble.apache.org/api/mail/timeseries-single',
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Kibble-Token': TOKEN,
+ },
+ json = {
+ "quick":True,
+ "interval": "week",
+ "subfilter":"\?%s$" % ml,
+ }
+ ).json()
+ user_ts = emails['timeseries']
+
+
+ # Most discussed email topics
+ # This requires 4 months of data because data is compiled into
+ # monthly segments, so mid-month requests return skewed data.
+ topics =
requests.post('https://demo.kibble.apache.org/api/mail/top-topics',
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Kibble-Token': TOKEN,
+ },
+ json = {
+ "quick":True,
+ "interval": "month",
+ "from": BEFORE - (86400*30),
+ "to": int(time.time()),
+ "subfilter":"(dev|users?)@" + project + "\.apache\.org",
+ }
+ ).json();
+
+ js = {
+ 'prs': {
+ 'before': {
+ 'opened': pro_before,
+ 'closed': prc_before,
+ },
+ 'after': {
+ 'opened': pro_after,
+ 'closed': prc_after,
+ },
+ 'change': {
+ 'opened': pro_change,
+ 'closed': prc_change,
+ }
+ },
+ 'issues': {
+ 'before': {
+ 'opened': iso_before,
+ 'closed': isc_before,
+ },
+ 'after': {
+ 'opened': iso_after,
+ 'closed': isc_after,
+ },
+ 'change': {
+ 'opened': iso_change,
+ 'closed': isc_change,
+ }
+ },
+ 'jira': {
+ 'before': {
+ 'opened': jio_before,
+ 'closed': jic_before,
+ },
+ 'after': {
+ 'opened': jio_after,
+ 'closed': jic_after,
+ },
+ 'change': {
+ 'opened': jio_change,
+ 'closed': jic_change,
+ }
+ },
+ 'commits' : {
+ 'before': {
+ 'commits': cmt_before,
+ 'authors': cmtp_before,
+ },
+ 'after': {
+ 'commits': cmt_after,
+ 'authors': cmtp_after,
+ },
+ 'change': {
+ 'commits': cmt_change,
+ 'authors': cmtp_change,
+ }
+ },
+ 'busiest': {
+ 'email': topics['topN']['items'][:10],
+ 'github': bissues['topN']['items'][:10],
+ 'jira': bjiras['topN']['items'][:10] if bjiras else [],
+ },
+ 'timeseries': {
+ 'github': github_ts,
+ 'jira': jira_ts,
+ 'commits': commit_ts,
+ 'devlist': dev_ts,
+ 'userlist': user_ts,
+ }
+ }
+ output = json.dumps(js, indent = 2)
+ with open(cache_file, 'w') as f:
+ f.write(output)
+ f.close()
+ return js
\ No newline at end of file
Propchange: comdev/reporter.apache.org/trunk/scripts/rapp/kibble.py
------------------------------------------------------------------------------
svn:executable = *
