This is an automated email from the ASF dual-hosted git repository.
sebb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/attic.git
The following commit(s) were added to refs/heads/main by this push:
new 8b614d8 No longer needed [skip ci]
8b614d8 is described below
commit 8b614d8ba217a93129f4f0a9eace080a93254c86
Author: Sebb <[email protected]>
AuthorDate: Thu May 1 22:17:57 2025 +0100
No longer needed [skip ci]
---
infrajiratext.py | 128 -----------------------------------
retire.py | 202 -------------------------------------------------------
urlutils.py | 84 -----------------------
3 files changed, 414 deletions(-)
diff --git a/infrajiratext.py b/infrajiratext.py
deleted file mode 100755
index 7289009..0000000
--- a/infrajiratext.py
+++ /dev/null
@@ -1,128 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-
-Find project resources for retired projects and generate JIRA issue text for
INFRA:
-- source control system (SVN or Git)
-- dist.apache.org releases and dev
-- mailing lists (Whimsy)
-- JIRA https://issues.apache.org/jira/rest/api/2/project.json - detect if
present
-- LDAP project
-- https://cwiki.apache.org/confluence/display/VXQUERY (e.g.)
-
-"""
-
-import sys
-import subprocess
-from urlutils import loadjson, urlexists
-
-DEV="https://dist.apache.org/repos/dist/dev/";
-REL="https://dist.apache.org/repos/dist/release/";
-JIRA='https://issues.apache.org/jira/rest/api/2/project'
-CWIKI='https://cwiki.apache.org/confluence/display/'
-EMAIL='https://whimsy.apache.org/public/committee-retired.json'
-GITBOX='https://gitbox.apache.org/repositories.json' # ['projects']
-
-
-# =====================================
-
-retired = loadjson(EMAIL)['retired']
-gitbox = loadjson(GITBOX)['projects']
-
-def check_wiki(pid):
- url = CWIKI + pid.upper()
- if urlexists(url):
- print("Make CWIKI readonly: %s" % url)
-
-def check_mail(pid):
- try:
- mlists = retired[pid]['mlists']
- if len(mlists) > 0:
- print("Mailing lists
https://lists.apache.org/list.html?%s.apache.org :" % pid)
- for mlist in mlists:
- print("- %s@%s.apache.org" % (mlist,pid))
- print()
- except KeyError:
- pass
-
-def svnfiles(path):
- res = subprocess.run(["svn", "ls", path], capture_output=True)
- if res.returncode == 0:
- stdout = res.stdout
- if len(stdout) == 0:
- return 0
- if stdout == b'.htaccess\n': # just the htaccess file is OK
- return 0
- return 1
- else:
- stderr = res.stderr
- if b"W160013" in stderr: # no such item
- return -1
- print(stderr)
- return -1
-
-def check_dist(pid):
- dev = svnfiles(DEV + pid)
- rel = svnfiles(REL + pid)
- if dev > 0 or rel > 0:
- if dev > 0:
- print("Remove the distribution SVN developer directory:")
- print('- ' + DEV + pid)
- if rel > 0:
- print("Remove the distribution SVN release directory contents,
and")
- print("set up redirection to
https://attic.apache.org/projects/%s.html"; % pid)
- print('- ' + REL + pid)
- print()
-
-# There is no anonymous access to LDAP now
-# Assume there is a project LDAP group
-def check_ldap(pid):
- BASE='ou=project,ou=groups,dc=apache,dc=org'
- print("Remove the LDAP project group: dn: cn=%s,%s" % (pid, BASE))
-
-def check_jira(pid):
- jira = loadjson(JIRA)
- for project in jira:
- key = project['key']
- catname = ''
- if 'projectCategory' in project:
- catname = project['projectCategory']['name']
- if pid.upper() == key:
- if '(Retired)' in project['name'] or catname == 'Retired':
- pass
- else:
- print("Make the JIRA project
https://issues.apache.org/jira/projects/%s read-only and flag as retired" % key)
- elif catname.lower() == pid:
- print("Make the JIRA project
https://issues.apache.org/jira/projects/%s read-only and flag as retired" % key)
-
-# check for SVN and Git
-def check_scs(pid):
- svn = "https://svn.apache.org/repos/asf/%s"; % pid
- if svnfiles(svn) > 0:
- print("Make SVN readonly: %s" % svn) # TODO what if this is the site
SVN?
-# print("Update config file so commits go to attic ???")
- if pid in gitbox:
- print("Make the following git repos read-only
https://gitbox.apache.org/repos/asf#%s :" % pid)
- for repo in gitbox[pid]['repositories']:
- print("- %s" % repo)
- print("Please do NOT rename the repos.")
-
-if len(sys.argv) == 1:
- print("Please provide a list of project ids")
- sys.exit(1)
-
-for arg in sys.argv[1:]:
- if not arg in retired:
- print("%s is not listed as a retired TLP" % arg)
- continue
-# with open('jira_%s.tmp' % arg, 'w') as f:
-# with redirect_stdout(f):
- print("%s project has retired." % retired[arg]['display_name'])
- print("The following resources were detected for archive/removal
please:\n")
- check_dist(arg)
- check_mail(arg)
- check_jira(arg)
- check_scs(arg)
- check_wiki(arg)
- check_ldap(arg)
- print("Turn off any automated builds")
diff --git a/retire.py b/retire.py
deleted file mode 100755
index fdb2970..0000000
--- a/retire.py
+++ /dev/null
@@ -1,202 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-
-Script to create retirement files for a project.
-
-Input:
-- https://whimsy.apache.org/public/committee-retired.json
-- https://lists.apache.org/api/preferences.lua
-- xdocs/projects/_template.xml
-- _template.jira
-
-Output:
-- xdocs/flagged/<pid> (created)
-- xdocs/projects/<pid>.xml (created)
-- xdocs/stylesheets/project.xml (updated)
-- cwiki_retired/<wiki_id>.txt (created)
-- <pid>.jira.tmp (created) - this is for pasting into an Attic JIRA issue
-
-N.B. The generated pid.xml file may need tweaking
-
-"""
-
-import sys
-import os.path
-from os.path import dirname, abspath, join
-from inspect import getsourcefile
-from string import Template
-import os
-import re
-import subprocess
-from collections import defaultdict
-from urlutils import loadyaml, loadjson
-
-if len(sys.argv) == 1:
- print("Please provide a list of project ids")
- sys.exit(1)
-
-CWIKI='https://cwiki.apache.org/confluence/display/'
-
-CWIKI_INFO='https://cwiki.apache.org/confluence/rest/api/space/?type=global&limit=1000'
-WIKI_KEYS = defaultdict(list) # key = project name, value = list of possible
spaces
-for entry in loadjson(CWIKI_INFO)['results']:
- WIKI_KEYS[entry['name'].lower().replace('apache ',
'')].append(entry['key'])
-
-JIRA='https://issues.apache.org/jira/rest/api/2/project'
-
-SVNURL='https://svn.apache.org/repos/asf/'
-SVNKEYS = {}
-svnargs = ['svn', 'ls', SVNURL]
-output = subprocess.run(svnargs, stdout=subprocess.PIPE,
check=True).stdout.decode("us-ascii")
-for name in output.split('\n'):
- if len(name) > 0:
- SVNKEYS[name.rstrip('/')] = ''
-
-GITREPOS='https://gitbox.apache.org/repositories.json'
-GITKEYS = {}
-for entry in loadjson(GITREPOS)['projects']:
- GITKEYS[entry] = ''
-
-MYHOME = dirname(abspath(getsourcefile(lambda:0)))
-PROJECTS = join((MYHOME), 'xdocs', 'projects')
-SYLESHEETS = join((MYHOME), 'xdocs', 'stylesheets')
-FLAGGED = join((MYHOME), 'xdocs', 'flagged')
-CWIKI_RETIRED = join((MYHOME), 'cwiki_retired')
-
-# get details of the retired projects
-RETIREES =
loadyaml('https://whimsy.apache.org/public/committee-retired.json')['retired']
-lists = {}
-for host,names in
loadyaml('https://lists.apache.org/api/preferences.lua')['lists'].items():
- proj = host.replace('.apache.org','')
- if proj in RETIREES:
- lists[proj] = list(names.keys())
-
-def list_jira(pid):
- jira = loadjson(JIRA)
- jiras = []
- for project in jira:
- key = project['key']
- catname = ''
- if 'projectCategory' in project:
- catname = project['projectCategory']['name']
- if pid.upper() == key:
- jiras.append(key)
- elif catname.lower() == pid:
- jiras.append(key)
- return jiras
-
-# updates xdocs/stylesheets/project.xml
-# <li><a href="/projects/abdera.html">Abdera</a></li>
-def update_stylesheet(pid):
- xmlfile = join(SYLESHEETS,'project.xml')
- xmlfilet = join(SYLESHEETS,'project.xml.t')
- print("Updating %s" % xmlfile)
- found = False
- with open(xmlfile,'r', encoding='utf-8') as r, open(xmlfilet,'w',
encoding='utf-8') as w:
- for l in r:
- if not found:
- m = re.search(r'^(\s+<li><a
href="/projects/)([^.]+)(.html">)[^<]+(</a></li>)', l)
- if m:
- stem = m.group(2)
- if stem == pid:
- print("Already present in projects.xml")
- print(l)
- w.close()
- os.remove(xmlfilet)
- return
- if stem > pid: # Found insertion point
- found = True
- name = RETIREES[pid]['display_name']
- w.write("%s%s%s%s%s\n" % (m.group(1), pid, m.group(3),
name, m.group(4)))
- w.write(l) # write the original line
- if found:
- print("Successfully added %s to %s" % (pid, xmlfile))
- os.system("diff %s %s" % (xmlfile, xmlfilet))
- os.rename(xmlfilet, xmlfile)
- else:
- print("Could not find where to add %s" % pid)
-
-# create JIRA template
-def create_jira_template(pid):
- outfile = join(MYHOME, "%s.jira.tmp" % pid)
- print("Creating %s" % outfile)
- with open(join(MYHOME, '_template.jira'), 'r', encoding='utf-8') as t:
- template = Template(t.read())
- out = template.substitute(tlpid = pid)
- with open(outfile, 'w', encoding='utf-8') as o:
- o.write(out)
-
-# create the project.xml file from the template
-def create_project(pid):
- outfile = join(PROJECTS, "%s.xml" % pid)
- print("Creating %s" % outfile)
- with open(join(PROJECTS, '_template.xml'), 'r', encoding='utf-8') as t:
- template = Template(t.read())
- meta = RETIREES[pid]
- mnames = lists[pid]
- mnames.remove('dev')
- jiras = list_jira(pid)
- # Is there a Wiki?
- cwikis = find_wiki(pid)
- if len(cwikis) > 0:
- if len(cwikis) == 1 and cwikis[0].lower() == pid:
- cwiki ='<cwiki/>'
- else:
- keys = ','.join(sorted(cwikis))
- cwiki =f'<cwiki keys="{keys}"/>'
- else:
- cwiki = ''
- svn = git = ''
- # May have both SVN and Git
- if pid in SVNKEYS:
- svn = '<svn/>'
- if pid in GITKEYS:
- git = '<git/>'
- out = template.substitute(tlpid = pid,
- FullName = meta['display_name'],
- Month_Year = meta['retired'],
- mail_names = ",".join(sorted(mnames)),
- jira_names = ",".join(sorted(jiras)),
- cwiki = cwiki,
- svn = svn,
- git = git,
- description = meta.get('description', 'TBA'))
- with open(outfile, 'w', encoding='utf-8') as o:
- o.write(out)
- os.system("git add %s" % outfile)
- print("Check XML file for customisations such as JIRA and mailing lists")
-
-def find_wiki(pid):
- found = []
- for k, v in WIKI_KEYS.items():
- if k == pid or (k.startswith(pid)): # and not k == valid project name
- found += v
- return found
-
-def check_wiki(pid):
- for wiki in find_wiki(pid):
- flagname = wiki.lower()
- flagfile = join(CWIKI_RETIRED, f"{flagname}.txt")
- with open(flagfile, 'w', encoding='utf-8') as w:
- if not flagname == pid:
- w.write(pid)
- w.write("\n")
- os.system("git add %s" % flagfile)
-
-for arg in sys.argv[1:]:
- print("Processing "+arg)
- if not arg in RETIREES:
- print("%s does not appear to be a retired project" % arg)
- continue
- flagdir = join(FLAGGED, arg)
- if os.path.exists(flagdir):
- print("flagged/%s already exists" % arg)
- continue
- create_jira_template(arg)
- os.mkdir(flagdir)
- open(join(flagdir, "git.keep"), 'a').close()
- os.system("git add %s" % flagdir)
- create_project(arg)
- update_stylesheet(arg)
- check_wiki(arg)
diff --git a/urlutils.py b/urlutils.py
deleted file mode 100644
index 8b3775a..0000000
--- a/urlutils.py
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/usr/bin/env python3
-
-"""
- Utility URL methods
-"""
-
-import hashlib
-import sys
-import errno
-import time
-import yaml
-import json
-from os import environ
-from os.path import getmtime, join
-from urllib.request import urlopen, Request
-from urllib.error import HTTPError
-
-MAXAGE=int(environ.get('CACHE_AGE', '600')) # 5 min in seconds
-CACHE=environ.get('CACHE')
-DEBUG=environ.get('CACHE_DEBUG')
-
-def isFileStale(filename):
- """ is file older than max age (default 5 minutes) """
- try:
- t = getmtime(filename)
- except OSError as e:
- if not e.errno == errno.ENOENT:
- raise e
- return True
- diff = time.time() - t
- return diff > MAXAGE
-
-def hashurl(url):
- """ create hash from url """
- return hashlib.sha224(url.encode()).hexdigest()
-
-def geturl(url):
- """ Get url contents -- no caching """
- req = Request(url)
- resp = urlopen(req)
- return resp.read()
-
-def urlexists(url):
- """ Does URL exist? """
- req = Request(url)
- try:
- urlopen(req)
- return True
- except HTTPError:
- return False
-
-def urlcache(url):
- """ Get url contents -- optional caching """
- if CACHE:
- # basename seems to work OK with URLs
- cache = join(CACHE, hashurl(url)+".tmp")
- if isFileStale(cache):
- if DEBUG:
- print("Caching %s" % url, file=sys.stderr)
- data = geturl(url)
- with open(cache,'wb') as w:
- w.write(data)
- return data
- else:
- if DEBUG:
- print("Using cache for %s" % url, file=sys.stderr)
- with open(cache,'r') as r:
- return r.read()
- else:
- if DEBUG:
- print("Fetching %s" % url, file=sys.stderr)
- return geturl(url)
-
-def loadyaml(url):
- return yaml.safe_load(urlcache(url))
-
-def loadjson(url):
- return json.loads(urlcache(url))
-
-# Test code
-if __name__ == '__main__':
- for arg in sys.argv[1:]:
- print(arg)
- print(loadyaml(arg))
