Author: sebb Date: Sat Aug 9 12:28:16 2025 New Revision: 1927713 Log: Use session cookie to avoid repeated authn Simplify code by using requests
Modified: comdev/reporter.apache.org/trunk/scripts/readjira.py Modified: comdev/reporter.apache.org/trunk/scripts/readjira.py ============================================================================== --- comdev/reporter.apache.org/trunk/scripts/readjira.py Sat Aug 9 12:10:02 2025 (r1927712) +++ comdev/reporter.apache.org/trunk/scripts/readjira.py Sat Aug 9 12:28:16 2025 (r1927713) @@ -15,9 +15,8 @@ """ import errtee # this is used, even though it is not referenced -import os, json, base64, time, sys -from urllib.request import urlopen, Request -from urllib.error import HTTPError +import os, json, time, sys +import requests from os import listdir from os.path import isfile, join, dirname, abspath from inspect import getsourcefile @@ -25,6 +24,14 @@ from inspect import getsourcefile # MYHOME = "/var/www/reporter.apache.org" # Assume we are one directory below RAO (scripts) MYHOME = dirname(dirname(abspath(getsourcefile(lambda:0)))) # automatically work out home location so can run the code anywhere + +JIRA_REST = 'https://issues.apache.org/jira/rest/' +JIRA_AUTH = JIRA_REST + 'auth/1/session' # POST or DELETE +JIRA_PROJ = JIRA_REST + 'api/2/project.json' +JIRA_QURY = JIRA_REST + "api/2/search?jql=project='%s'+AND+%s%%3E=-91d&fields=key" + +PROJECT_JSON = f"{MYHOME}/data/JIRA/jira_projects.json" + mypath = "%s/data/JIRA" % MYHOME print("Scanning mypath=%s" % mypath) myfiles = [ f for f in listdir(mypath) if f.endswith(".json") and isfile(join(mypath,f)) ] @@ -34,9 +41,7 @@ with open("/usr/local/etc/tokens/jira.tx jirapass = f.read().strip() f.close() -__AUTHSTRING = '%s:%s' % ('githubbot', jirapass) -base64string = base64.b64encode(__AUTHSTRING.encode(encoding='utf-8')).decode(encoding='utf-8') - +JIRA_SLEEP = 0.5 # how long to wait between API requests JIRATIMEOUT=90 # This may need to be adjusted jiraerrors = 0 # count how many consecutive errors occurred def clearErrors(): @@ -49,44 +54,65 @@ def handleError(): if jiraerrors > 5: sys.exit(f"Too many errors - {jiraerrors} - quitting. See /var/log/www-data/readjira.log for details.") +def getCookie(): + hdrs = {"content-type": "application/json"} + data = {"username": "githubbot", "password": jirapass } # needs to be JSON + res = requests.post(JIRA_AUTH, headers=hdrs, json=data, timeout=JIRATIMEOUT) + res.raise_for_status() + session = res.json()['session'] + if session['name'] == 'JSESSIONID': + return session['value'] + else: + sys.exit("Failed to establish session cookie - quitting.") + +hdrs = {"cookie": f"JSESSIONID={getCookie()}; Path=/; HttpOnly;"} + +def logout(): + res = requests.delete(JIRA_AUTH, headers=hdrs, timeout=JIRATIMEOUT) + res.raise_for_status() + def getProjects(): """Update the list of projects in data/JIRA/jira_projects.json""" - PROJECT_JSON = "%s/data/JIRA/jira_projects.json" % MYHOME - x = {} - print("Refresh %s" % PROJECT_JSON) + print(f"Refresh {PROJECT_JSON}") try: - req = Request("https://issues.apache.org/jira/rest/api/2/project.json";) - req.add_header("Authorization", "Basic %s" % base64string) - x = json.loads(urlopen(req, timeout=JIRATIMEOUT).read().decode('utf-8')) + res = requests.get(JIRA_PROJ, headers=hdrs, timeout=JIRATIMEOUT) + res.raise_for_status() with open(PROJECT_JSON, "w") as f: - json.dump(x, f, indent=1, sort_keys=True) - f.close() - print("Created %s" % PROJECT_JSON) - clearErrors() + json.dump(res.json(), f, indent=1, sort_keys=True) + print(f"Created {PROJECT_JSON}") except Exception as e: - print("Err: could not refresh %s: %s" % (PROJECT_JSON, e)) - handleError() + print(f"Err: could not refresh {PROJECT_JSON}: {e}") + sys.exit("Failed to update project list - quitting. See /var/log/www-data/readjira.log for details.") def getJIRAS(project): - file = "%s/data/JIRA/%s.json" % (MYHOME, project) + file = f"{MYHOME}/data/JIRA/{project}.json" + created = JIRA_QURY % (project, 'created') + resolved = JIRA_QURY % (project, 'resolved') + res = None try: - req = Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project='""" + project + """'+AND+created%3E=-91d&fields=key""") # >= - req.add_header("Authorization", "Basic %s" % base64string) - cdata = json.loads(urlopen(req, timeout=JIRATIMEOUT).read().decode('utf-8')) - req = Request("""https://issues.apache.org/jira/rest/api/2/search?jql=project='""" + project + """'+AND+resolved%3E=-91d&fields=key""") # >= - req.add_header("Authorization", "Basic %s" % base64string) - rdata = json.loads(urlopen(req, timeout=JIRATIMEOUT).read().decode('utf-8')) + time.sleep(JIRA_SLEEP) # don't overload Jira + res = requests.get(created, headers=hdrs, timeout=JIRATIMEOUT) + res.raise_for_status() + cdata = res.json() + time.sleep(JIRA_SLEEP) # don't overload Jira + res = requests.get(resolved, headers=None, timeout=JIRATIMEOUT) + res.raise_for_status() + rdata = res.json() with open(file, "w") as f: json.dump([cdata['total'], rdata['total'], project], f, indent=1, sort_keys=True) clearErrors() return cdata['total'], rdata['total'], project - except Exception as err: - response = '' + except requests.exceptions.RequestException as err: # If a project cannot be found, the API returns code 400 with JSON text of the form: # {"errorMessages":["The value 'DUMMY' does not exist for the field 'project'."],"errors":{}} # For some errors (e.g. auth fail), the API returns 401 or 403, in which case the body is HTML rather than JSON - if isinstance(err, HTTPError): - response = err.read().decode('utf-8') # needed to check for missing project + ct = res.headers['content-type'] + if ct.startswith('application/json'): + response = res.json()['errorMessages'][0] + elif 'www-authenticate' in res.headers: + response = 'Authentication failure' + else: + response = str(res.headers) print("Failed to get data for %s: %s %s" % (project, err, response)) if "does not exist for the field 'project'" in response: print("Removing %s" % file) @@ -102,6 +128,7 @@ for project in myfiles: if jiraname != "jira_projects": print("Refreshing JIRA stats for " + jiraname) getJIRAS(jiraname) - time.sleep(2) + +logout() print("Done") \ No newline at end of file
