Author: humbedooh Date: Thu Aug 8 11:07:48 2019 New Revision: 1864689 URL: http://svn.apache.org/viewvc?rev=1864689&view=rev Log: Add in new API endpoints for WSGI app.
Added: comdev/reporter.apache.org/trunk/scripts/rapp/ comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py comdev/reporter.apache.org/trunk/scripts/rapp/overview.py comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py Modified: comdev/reporter.apache.org/trunk/scripts/pdata.py comdev/reporter.apache.org/trunk/scripts/wsgi.py Modified: comdev/reporter.apache.org/trunk/scripts/pdata.py URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/pdata.py?rev=1864689&r1=1864688&r2=1864689&view=diff ============================================================================== --- comdev/reporter.apache.org/trunk/scripts/pdata.py (original) +++ comdev/reporter.apache.org/trunk/scripts/pdata.py Thu Aug 8 11:07:48 2019 @@ -41,6 +41,9 @@ MEMBER_INFO = 'https://whimsy.apache.org PROJECTS = 'https://whimsy.apache.org/public/public_ldap_projects.json' DESCRIPTIONS = 'https://projects.apache.org/json/foundation/committees.json' +def has_cache(filename, ttl = 14400): + return (os.path.exists(filename) and os.path.getmtime(filename) > (time.time() - ttl)) + jmap = { 'trafficserver': ['TS'], 'cordova': ['CB'], Added: comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py?rev=1864689&view=auto ============================================================================== --- comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py (added) +++ comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py Thu Aug 8 11:07:48 2019 @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +""" script for working with drafts """ +import os +import sys +import time +import json +import re +import pdata +import committee_info + +DRAFTS_DIR = '/tmp/rapp-drafts' +EDITOR_TYPE = 'unified' + +if not os.path.isdir(DRAFTS_DIR): + os.makedirs(DRAFTS_DIR, exist_ok = True) + +def has_access(user, project): + member = pdata.isASFMember(user) + pmc = project in pdata.getPMCs(user) + return (member or pmc) + +def index(environ, user): + """ Listy currently existing drafts for a project """ + project = environ.get('QUERY_STRING') + drafts = {} + + if has_access(user, project): + whence = int(time.time() - (60*86400)) # Max 2 months ago! + for filename in [x for x in os.listdir(DRAFTS_DIR) if x.startswith(EDITOR_TYPE) and x.endswith('.draft')]: + e, p, t, u = filename.split('-', 3) + t = int(t) + # If a file is way old, try deleting it. + if t < whence: + try: + os.unlink("%s/%s" % (DRAFTS_DIR, filename)) + except: + pass + elif p == project and t >= whence: + u = u.replace('.draft', '') + drafts[t] = {'filename': filename, 'creator': u, 'yours': user == u} + + return { 'drafts': drafts } + +def fetch(environ, user): + """ Fetch a draft if access is right... """ + filename = environ.get('QUERY_STRING') + m = re.match(r"[^-./]+-([^-./]+)-\d+-[^-./]+\.draft$", filename) + if not m: + return {'error': "Invalid filename!"} + if os.path.exists(os.path.join(DRAFTS_DIR, filename)): + project = m.group(1) + if has_access(user, project): + report = open(os.path.join(DRAFTS_DIR, filename), "r").read() + return {'report': report} + return {} + +def delete(environ, user): + """ Delete a draft if access is right... """ + filename = environ.get('QUERY_STRING') + m = re.match(r"[^-./]+-[^-./]+-\d+-([^-/]+)\.draft$", filename) + if not m: + return {'error': "Invalid filename!"} + if os.path.exists(os.path.join(DRAFTS_DIR, filename)): + u = m.group(1) + if user == u: + try: + os.unlink(os.path.join(DRAFTS_DIR, filename)) + return {'message': 'Draft deleted'} + except: + pass + return {'error': "Could not delete draft!"} + +def save(environ, user): + """ Save a draft """ + try: + request_body_size = int(environ.get('CONTENT_LENGTH', 0)) + except (ValueError): + request_body_size = 0 + if request_body_size: + request_body = environ['wsgi.input'].read(request_body_size) + try: + js = json.loads(request_body.decode('utf-8')) + except: + js = {} + if js: + project = js.get('project') + if has_access(user, project): + report = js.get('report') + now = int(time.time()) + filename = '%s-%s-%s-%s.draft' % (EDITOR_TYPE, project, now, user) + try: + with open(os.path.join(DRAFTS_DIR, filename), "w") as f: + f.write(report) + f.close() + return { + 'okay': True, + 'filename': filename, + } + except: + return { + 'okay': False, + 'error': 'Could not save draft (permission issue?? disk full?)', + } + else: + return { + 'okay': False, + 'error': 'You do not have access to this project', + } + else: + return { + 'okay': False, + 'error': "Invalid data!", + } Added: comdev/reporter.apache.org/trunk/scripts/rapp/overview.py URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/overview.py?rev=1864689&view=auto ============================================================================== --- comdev/reporter.apache.org/trunk/scripts/rapp/overview.py (added) +++ comdev/reporter.apache.org/trunk/scripts/rapp/overview.py Thu Aug 8 11:07:48 2019 @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +import os +import cgi +import json +import pdata +import time +import re +import committee_info + +CACHE_TIMEOUT = 14400 + +def run(environ, user): + committers = pdata.loadJson(pdata.COMMITTER_INFO)['people'] + pmcSummary = committee_info.PMCsummary() + project = environ.get('QUERY_STRING') + output = {'okay': False, 'error': 'Unknown user ID provided!'} + + dumps = {} + groups = [] + if user: + groups = pdata.getPMCs(user) + if project and user and re.match(r"[-a-z0-9]+", project): + groups = [project] + + for xproject in groups: + + # Try cache first? (max 6 hours old) + wanted_file = "/tmp/pdata-%s.json" % xproject + if xproject == project: + wanted_file = "/tmp/pdata-kibbled-%s.json" % xproject + if (os.path.exists(wanted_file) and os.path.getmtime(wanted_file) > (time.time() - CACHE_TIMEOUT)): + mpdata = json.load(open(wanted_file, "r")) + # If cache failed, generate fom scratch + else: + mpdata = pdata.generate(user, xproject, xproject == project) + if not mpdata: + break + open(wanted_file, "w").write(json.dumps(mpdata)) + # Weave results into combined object, mindful of kibble data + for k, v in mpdata.items(): + if k not in dumps: + dumps[k] = {} + if (k != 'kibble'): + dumps[k][xproject] = v + if k == 'kibble' and v: + dumps['kibble'] =v + + # Set personalized vars, dump + if dumps and user: + ddata, allpmcs, health = pdata.getProjectData() + dumps['you'] = committers[user] + dumps['all'] = sorted(allpmcs) + dumps['pmcs'] = sorted(groups) + dumps['pmcsummary'] = pmcSummary + output = dumps + + return output Added: comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py?rev=1864689&view=auto ============================================================================== --- comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py (added) +++ comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py Thu Aug 8 11:07:48 2019 @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +""" script for publishing a report to whimsy """ +import os +import sys +import time +import json +import requests +import re +import pdata +import committee_info + +WHIMSY_SUBMIT = 'https://whimsy.apache.org/board/agenda/json/post' +WHIMSY_AGENDA = 'https://whimsy.apache.org/board/agenda/latest.json' +WHIMSY_COMMENTS = 'https://whimsy.apache.org/board/agenda/json/historical-comments' + +def get_whimsy(url, env, ttl = 14400): + cached = True + xurl = re.sub(r"[^a-z0-9]+", "-", url.replace('.json', '')) + wanted_file = '/tmp/%s.json' % xurl + if pdata.has_cache(wanted_file, ttl = ttl): + js = json.load(open(wanted_file)) + else: + try: + print("Fetching %s => %s..." % (url, wanted_file)) + js = requests.get(url, headers = {'Authorization': env.get('HTTP_AUTHORIZATION')}, timeout = 5).json() + with open(wanted_file, "w") as f: + json.dump(js, f) + f.close() + cached = False + except: # fall back to cache on failure! + js = json.load(open(wanted_file)) + + return js, cached + +def has_access(user, project): + member = pdata.isASFMember(user) + pmc = project in pdata.getPMCs(user) + return (member or pmc) + +def guess_title(project): + """ Guess the whimsy name of a project """ + pmcSummary = committee_info.PMCsummary() + + # Figure out the name as written in whimsy.. + pname = project + if project in pmcSummary: + pname = pmcSummary[project]['name'].replace('Apache ', '') + + return pname + +def agenda_forced(environ, user): + """ Force whimsy agenda refresh... """ + get_whimsy(WHIMSY_AGENDA, environ, ttl = 0) + return agenda(environ, user) + +def agenda(environ, user): + """ Returns data on the board report for a project, IF present and/or filed in the current agenda """ + project = environ.get('QUERY_STRING') + report = None + if has_access(user, project): + agenda, cached = get_whimsy(WHIMSY_AGENDA, environ, ttl = 3600) + for entry in agenda: + ml = entry.get('mail_list') # mailing list id, usually correct + rid = entry.get('roster', '').replace('https://whimsy.apache.org/roster/committee/', '') # ldap id per roster + if ml and (ml == project or rid == project): + report = entry + break + + comments, cached = get_whimsy(WHIMSY_COMMENTS, environ) + title = report and report.get('title') or guess_title(project) + if title in comments: + comments = comments[title] + else: + comments = {} + + return { + 'can_access': True, + 'found': report and True or False, + 'filed': report and report.get('report') and True or False, + 'report': report, + 'comments': comments, + } + + return { + 'can_access': False, + 'found': False, + } + + + comments, cached = get_whimsy(WHIMSY_COMMENTS, environ) + +def comments(environ, user): + """ Display board feedback from previous reports ... """ + project = environ.get('QUERY_STRING') + comments, cached = get_whimsy(WHIMSY_COMMENTS, environ) + + pmcSummary = committee_info.PMCsummary() + + # Figure out the name as written in whimsy.. + pname = project + if project in pmcSummary: + pname = pmcSummary[project]['name'].replace('Apache ', '') + cmt = {} + + # If we can access, fetch comments + if comments and pname in comments and has_access(user, project): + comments = comments[pname] + else: + comments = {} + + js = { + "pid": project, + "pname": pname, + "comments": comments, + "used_cache": cached, + } + return js + + +def publish(environ, user): + try: + request_body_size = int(environ.get('CONTENT_LENGTH', 0)) + except (ValueError): + request_body_size = 0 + if request_body_size: + request_body = environ['wsgi.input'].read(request_body_size) + try: + js = json.loads(request_body.decode('utf-8')) + except: + js = {} + if js: + agenda = js.get('agenda') + project = js.get('project') + report = js.get('report') + digest = js.get('digest') + attach = js.get('attach') + print(project, agenda) + if agenda and project and report: + message = "Publishing report for %s via Reporter" % project + payload = { + 'agenda': agenda, + 'project': project, + 'report': report, + 'message': message, + } + if digest and attach: + del payload['project'] + payload['attach'] = attach + payload['message'] = "Updating report for %s via Reporter." % project + payload['digest'] = digest + try: + rv = requests.post(WHIMSY_SUBMIT, headers = { + 'Authorization': environ.get('HTTP_AUTHORIZATION'), + "Content-Type": "application/json" + }, json = payload, timeout = 10) + rv.raise_for_status() + print(rv.text) + return {'okay': True, 'message': "Posted to board agenda!"} + except: + pass + return {} + + \ No newline at end of file Modified: comdev/reporter.apache.org/trunk/scripts/wsgi.py URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/wsgi.py?rev=1864689&r1=1864688&r2=1864689&view=diff ============================================================================== --- comdev/reporter.apache.org/trunk/scripts/wsgi.py (original) +++ comdev/reporter.apache.org/trunk/scripts/wsgi.py Thu Aug 8 11:07:48 2019 @@ -1,62 +1,42 @@ #!/usr/bin/env python2.7 -import os -import cgi import json -import pdata import time +import base64 import re -import committee_info CACHE_TIMEOUT = 14400 +import rapp.overview +import rapp.whimsy +import rapp.drafts + +webmap = { + '/api/overview': rapp.overview.run, + '/api/whimsy/comments': rapp.whimsy.comments, + '/api/whimsy/agenda': rapp.whimsy.agenda, + '/api/whimsy/agenda/refresh': rapp.whimsy.agenda_forced, + '/api/whimsy/publish': rapp.whimsy.publish, + '/api/drafts/index': rapp.drafts.index, + '/api/drafts/save': rapp.drafts.save, + '/api/drafts/fetch': rapp.drafts.fetch, + '/api/drafts/delete': rapp.drafts.delete, +} def app(environ, start_fn): - committers = pdata.loadJson(pdata.COMMITTER_INFO)['people'] - pmcSummary = committee_info.PMCsummary() - project = environ.get('QUERY_STRING') - user = environ.get('HTTP_X_AUTHENTICATED_USER', 'humbedooh') - - output = {'okay': False, 'error': 'Unknown user ID provided!'} - - dumps = {} - groups = [] - if user: - groups = pdata.getPMCs(user) - if project and user and re.match(r"[-a-z0-9]+", project): - groups = [project] - - for xproject in groups: - - # Try cache first? (max 6 hours old) - wanted_file = "/tmp/pdata-%s.json" % xproject - if xproject == project: - wanted_file = "/tmp/pdata-kibbled-%s.json" % xproject - if (os.path.exists(wanted_file) and os.path.getmtime(wanted_file) > (time.time() - CACHE_TIMEOUT)): - mpdata = json.load(open(wanted_file, "r")) - # If cache failed, generate fom scratch - else: - mpdata = pdata.generate(user, xproject, xproject == project) - if not mpdata: - break - open(wanted_file, "w").write(json.dumps(mpdata)) - # Weave results into combined object, mindful of kibble data - for k, v in mpdata.items(): - if k not in dumps: - dumps[k] = {} - if (k != 'kibble'): - dumps[k][xproject] = v - if k == 'kibble' and v: - dumps['kibble'] =v - - # Set personalized vars, dump - if dumps and user: - ddata, allpmcs, health = pdata.getProjectData() - dumps['you'] = committers[user] - dumps['all'] = sorted(allpmcs) - dumps['pmcs'] = sorted(groups) - dumps['pmcsummary'] = pmcSummary - output = dumps + now = time.time() + bauth = re.match(r"Basic (.+)", environ.get('HTTP_AUTHORIZATION', 'foo')) + if bauth: + bdec = base64.b64decode(bauth.group(1)).decode('utf-8') + m = re.match("^(.+?):(.+)$", bdec) + if m: + user = m.group(1) + uri = environ.get('PATH_INFO', '/') + if uri in webmap: + output = webmap[uri](environ, user) + else: + output = {'okay': False, 'message': 'Unknown URI %s' % uri} + output['took'] = int((time.time() - now) * 1000) out = json.dumps(output, indent = 2, sort_keys = True).encode('ascii') start_fn('200 OK', [('Content-Type', 'application/json'), ('Content-Length', str(len(out)))]) return [out]