Author: humbedooh Date: Fri Aug 9 17:32:31 2019 New Revision: 1864826 URL: http://svn.apache.org/viewvc?rev=1864826&view=rev Log: Updates: - fetch calendar from whimsy to determine which agenda in use - better (future?) checks for partial content - list agenda file in forgotten API
Modified: comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py Modified: comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py?rev=1864826&r1=1864825&r2=1864826&view=diff ============================================================================== --- comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py (original) +++ comdev/reporter.apache.org/trunk/scripts/rapp/drafts.py Fri Aug 9 17:32:31 2019 @@ -118,7 +118,8 @@ def save(environ, user): def forgotten(environ, user): """ Query for which TLP reports have drafts but haven't filed to agenda yet """ - agenda, cached = rapp.whimsy.get_whimsy(rapp.whimsy.WHIMSY_AGENDA, environ) + which_agenda, aurl = rapp.whimsy.latest_agenda(environ) + agenda, cached = rapp.whimsy.get_whimsy(aurl, environ) drafts = sorted([x for x in os.listdir(DRAFTS_DIR) if x.startswith(EDITOR_TYPE) and x.endswith('.draft')]) lost = {} for entry in agenda: @@ -129,6 +130,7 @@ def forgotten(environ, user): if entry.get('report'): lost[rid] = { 'filed': True, + 'agenda': which_agenda, 'attach': entry['attach'] } else: @@ -150,6 +152,7 @@ def forgotten(environ, user): last_author = u.replace('.draft', '') lost[rid] = { 'filed': False, + 'agenda': which_agenda, 'has_draft': has_draft, 'last_draft': last_report, 'last_author': last_author, Modified: comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py URL: http://svn.apache.org/viewvc/comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py?rev=1864826&r1=1864825&r2=1864826&view=diff ============================================================================== --- comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py (original) +++ comdev/reporter.apache.org/trunk/scripts/rapp/whimsy.py Fri Aug 9 17:32:31 2019 @@ -11,8 +11,11 @@ import pdata import committee_info WHIMSY_SUBMIT = 'https://whimsy.apache.org/board/agenda/json/post' -WHIMSY_AGENDA = 'https://whimsy.apache.org/board/agenda/latest.json' +WHIMSY_AGENDA_IP = 'https://whimsy.apache.org/board/agenda/%s.json' +WHIMSY_AGENDA_RE = r'https://whimsy\.apache\.org/board/agenda/(latest|\d+-\d+-\d+)\.json' + WHIMSY_COMMENTS = 'https://whimsy.apache.org/board/agenda/json/historical-comments' +WHIMSY_CALENDAR = 'https://whimsy.apache.org/board/agenda/calendar.json' def get_whimsy(url, env, ttl = 14400): cached = True @@ -23,10 +26,13 @@ def get_whimsy(url, env, ttl = 14400): else: try: print("Fetching %s => %s..." % (url, wanted_file)) - js = requests.get(url, headers = {'Authorization': env.get('HTTP_AUTHORIZATION')}, timeout = 5).json() + rv = requests.get(url, headers = {'Authorization': env.get('HTTP_AUTHORIZATION')}, timeout = 5) + js = rv.json() - # Some weaving may be required here if we're not member or chair - if url == WHIMSY_AGENDA and len(js) < 50 and ttl <= 3600: + # If we get a partial response (206), we need to weave the partial + # into the full object in cache. + if re.match(WHIMSY_AGENDA_RE, url) and (len(js) < 50 or rv.status_code == 206): + print("Got partial response, weaving!!") try: ojs = json.load(open(wanted_file)) except: @@ -51,7 +57,7 @@ def get_whimsy(url, env, ttl = 14400): js = ojs # Extend old cache if needed - elif url == WHIMSY_COMMENTS: + elif url == WHIMSY_COMMENTS and rv.status_code == 206: try: ojs = json.load(open(wanted_file)) except: @@ -70,6 +76,15 @@ def get_whimsy(url, env, ttl = 14400): return js, cached +def latest_agenda(environ): + calendar, cached = get_whimsy(WHIMSY_CALENDAR, environ) + latest = calendar['agendas'][-1] + ymd = re.match(r"board_agenda_(\d\d\d\d_\d\d_\d\d)\.txt", latest) + if ymd: + return latest, WHIMSY_AGENDA_IP % ymd.group(1).replace('_', '-') + else: + return latest, WHIMSY_AGENDA_IP % 'latest' + def has_access(user, project): member = pdata.isASFMember(user) pmc = project in pdata.getPMCs(user) @@ -88,15 +103,17 @@ def guess_title(project): def agenda_forced(environ, user): """ Force whimsy agenda refresh... """ - get_whimsy(WHIMSY_AGENDA, environ, ttl = 0) + txtfile, url = latest_agenda(environ) + get_whimsy(url, environ, ttl = 0) return agenda(environ, user) def agenda(environ, user): """ Returns data on the board report for a project, IF present and/or filed in the current agenda """ project = environ.get('QUERY_STRING') report = None + txtfile, url = latest_agenda(environ) if has_access(user, project): - agenda, cached = get_whimsy(WHIMSY_AGENDA, environ, ttl = 3600) + agenda, cached = get_whimsy(url, environ, ttl = 3600) for entry in agenda: ml = entry.get('mail_list') # mailing list id, usually correct rid = entry.get('roster', '').replace('https://whimsy.apache.org/roster/committee/', '') # ldap id per roster