Diff:
---
calm/calm.py | 4 +
calm/mksetupini.py | 2 +
calm/package.py | 19 ++--
calm/pkg2html.py | 114 ++++++++++++++++++---
test/testdata/htdocs.expected/dirtree.expected | 4 +-
test/testdata/htdocs.expected/doc/.htaccess | 1 +
.../doc/keychain/keychain-2.6.8.README | 58 +++++++++++
.../doc/keychain/keychain-2.7.1.README | 85 +++++++++++++++
test/testdata/htdocs.expected/packages_docs.inc | 4 +
.../testdata/htdocs.expected/summary/keychain.html | 1 +
test/testdata/process_arch/htdocs.expected | 4 +-
11 files changed, 274 insertions(+), 22 deletions(-)
diff --git a/calm/calm.py b/calm/calm.py
index 6bbf43a..600a074 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -365,6 +365,10 @@ def _process_maintainer_uploads(scan_result, args, state,
all_packages, m, based
logging.error("error while validating merged %s packages for %s" %
(arch, name))
valid = False
+ # issue any 'upload time only' validation warnings
+ for arch in common_constants.ARCHES:
+ package.packages_warnings(args, merged_packages[arch],
scan_result[arch].packages, scan_result['noarch'].packages,
scan_result['src'].packages)
+
# if an error occurred ...
if not valid:
# ... discard move list and merged_packages
diff --git a/calm/mksetupini.py b/calm/mksetupini.py
index 66449e1..70eee22 100755
--- a/calm/mksetupini.py
+++ b/calm/mksetupini.py
@@ -66,6 +66,8 @@ def do_main(args):
logging.error("package set has errors, not writing setup.ini")
return 1
+ package.packages_warnings(args, packages, packages)
+
# write setup.ini
package.write_setup_ini(args, packages, args.arch)
diff --git a/calm/package.py b/calm/package.py
index 4d9055b..292cbac 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -907,12 +907,6 @@ def validate_packages(args, packages,
valid_provides_extra=None, missing_obsolet
else:
if len(packages[p].versions()):
packages[p].best_version = sorted(packages[p].versions(),
key=lambda v: SetupVersion(v), reverse=True)[0]
-
- # warn if no non-test ('curr') version exists
- if (('missing-curr' not in
packages[p].version_hints[packages[p].best_version].get('disable-check', ''))
and
- ('missing-curr' not in getattr(args, 'disable_check',
[]))):
- logging.warning("package '%s' doesn't have any non-test
versions (i.e. no curr: version)" % (p))
-
else:
# the package must have some versions
logging.error("package '%s' doesn't have any versions" % (p))
@@ -1239,6 +1233,19 @@ def validate_package_maintainers(args, packages):
return error
+#
+# certain validation warnings we only want to issue once, when the package is
+# uploaded
+#
+def packages_warnings(args, packages, *modified):
+ for m in modified:
+ for p in sorted(m):
+ # warn if no non-test ('curr') version exists
+ if (('missing-curr' not in
packages[p].version_hints[packages[p].best_version].get('disable-check', ''))
and
+ ('missing-curr' not in getattr(args, 'disable_check', []))):
+ logging.warning("package '%s' doesn't have any non-test
versions (i.e. no curr: version)" % (p))
+
+
#
# write setup.ini
#
diff --git a/calm/pkg2html.py b/calm/pkg2html.py
index 5f34bba..617221a 100755
--- a/calm/pkg2html.py
+++ b/calm/pkg2html.py
@@ -40,14 +40,17 @@
#
import argparse
+import functools
import glob
import html
import logging
+import lzma
import math
import os
import re
import string
import sys
+import tarfile
import textwrap
import time
import types
@@ -329,6 +332,16 @@ def update_package_listings(args, packages):
repo_browse_url = '/cgit/cygwin-packages/%s/'
% pn
details_table['packaging repository'] = '<a
href="%s">%s.git</a>' % (repo_browse_url, pn)
+ if po.kind == package.Kind.binary:
+ doc_path = os.path.join(args.htdocs, 'doc', pn)
+ if os.path.exists(doc_path):
+ links = []
+
+ for readme in sorted(os.listdir(doc_path)):
+ links.append('<a href="../doc/%s/%s">%s</a>' %
(pn, readme, readme))
+
+ details_table['readme'] = ', '.join(links)
+
# output details table
print('<table class="pkgdetails">', file=f)
for d, v in details_table.items():
@@ -379,22 +392,23 @@ def update_package_listings(args, packages):
write_packages_inc(args, packages, 'src_packages.inc',
package.Kind.source, 'src_package_list.html')
+# callback function for open_amifc to touch including file
+def touch_including(including_file, changed):
+ if changed:
+ # touch the including file for the benefit of 'XBitHack full'
+ if os.path.exists(including_file):
+ logging.info("touching %s for the benefit of 'XBitHack full'" %
(including_file))
+ utils.touch(including_file)
+
+
#
# write package index page fragment for inclusion
#
def write_packages_inc(args, packages, name, kind, includer):
packages_inc = os.path.join(args.htdocs, name)
if not args.dryrun:
-
- def touch_including(changed):
- if changed:
- # touch the including file for the benefit of 'XBitHack full'
- package_list = os.path.join(args.htdocs, includer)
- if os.path.exists(package_list):
- logging.info("touching %s for the benefit of 'XBitHack
full'" % (package_list))
- utils.touch(package_list)
-
- with utils.open_amifc(packages_inc, cb=touch_including) as index:
+ package_list = os.path.join(args.htdocs, includer)
+ with utils.open_amifc(packages_inc,
cb=functools.partial(touch_including, package_list)) as index:
os.fchmod(index.fileno(), 0o644)
# This list contains all packages in any arch. Source packages
@@ -458,8 +472,57 @@ def write_packages_inc(args, packages, name, kind,
includer):
print('</table>', file=index)
+#
+# write package doc catalogue fragment for inclusion
+#
+doc_inc_overrides = {'X': 'https://x.cygwin.com/'}
+
+
+def write_doc_inc(args):
+ packages_inc = os.path.join(args.htdocs, 'packages_docs.inc')
+ if not args.dryrun:
+ htaccess = os.path.join(args.htdocs, 'doc', '.htaccess')
+ if not os.path.exists(htaccess) or args.force:
+ with utils.open_amifc(htaccess) as f:
+ # README files are text
+ print('AddType text/plain README', file=f)
+
+ package_docs = os.path.join(args.htdocs, 'package_docs.html')
+ with utils.open_amifc(packages_inc,
cb=functools.partial(touch_including, package_docs)) as index:
+ os.fchmod(index.fileno(), 0o644)
+
+ print('<div class="multicolumn-list">', file=index)
+
+ dir_list = os.listdir(os.path.join(args.htdocs, 'doc'))
+
+ for d in sorted(set(dir_list).union(doc_inc_overrides.keys()),
key=package.sort_key):
+ if d.startswith('.'):
+ continue
+
+ if d in doc_inc_overrides:
+ print('<p><a href="%s">%s</a></p>' %
(doc_inc_overrides[d], d),
+ file=index)
+ else:
+ links = []
+ different_name = False
+
+ for f in sorted(os.listdir(os.path.join(args.htdocs,
'doc', d))):
+ links.append('<a href="doc/%s/%s">%s</a>' % (d, f,
f.replace('.README', '')))
+
+ if f.replace('.README', '') != d:
+ different_name = True
+
+ if (len(links) > 1) or different_name:
+ print('<p>%s: %s</p>' % (d, ', '.join(links)),
file=index)
+ elif links:
+ print('<p>%s</p>' % (', '.join(links)), file=index)
+
+ print('</div>', file=index)
+
+
def write_arch_listing(args, packages, arch):
update_summary = set()
+ update_doc_inc = False
base = os.path.join(args.htdocs, arch)
ensure_dir_exists(args, base)
@@ -570,7 +633,23 @@ def write_arch_listing(args, packages, arch):
if i.issym() or i.islnk():
print(' -> %s' % i.linkname,
file=f, end='')
print('', file=f)
- except Exception as e:
+
+ # extract Cygwin-specific READMEs
+ if
i.name.startswith('usr/share/doc/Cygwin/') and i.name.endswith('README'):
+ logging.error("extracting %s to
cygwin-specific documents directory" % (i.name))
+
+ readme_text =
a.extractfile(i).read()
+ # redact email addresses
+ readme_text =
re.sub(rb'<(.*)@(.*)>', rb'<\1 at \2>', readme_text)
+
+ doc_dir =
os.path.join(args.htdocs, 'doc', p)
+ ensure_dir_exists(args, doc_dir)
+ with open(os.path.join(doc_dir,
os.path.basename(i.name)), mode='wb') as readme:
+ readme.write(readme_text)
+
+ update_doc_inc = True
+
+ except (tarfile.TarError, lzma.LZMAError) as e:
print('package is corrupted', file=f)
logging.error("exception %s while reading %s"
% (type(e).__name__, tf))
logging.debug('', exc_info=True)
@@ -612,6 +691,10 @@ def write_arch_listing(args, packages, arch):
logging.debug('rmdir %s' % dirpath)
os.rmdir(os.path.join(dirpath))
+ # update the package documents list
+ if update_doc_inc:
+ write_doc_inc(args)
+
return update_summary
@@ -620,7 +703,6 @@ if __name__ == "__main__":
relarea_default = common_constants.FTP
parser = argparse.ArgumentParser(description='Write HTML package listings')
- parser.add_argument('--arch', action='store', required=True,
choices=common_constants.ARCHES)
parser.add_argument('--force', action='store_true', help="overwrite
existing files")
parser.add_argument('--htdocs', action='store', metavar='DIR',
help="htdocs output directory (default: " + htdocs_default + ")",
default=htdocs_default)
parser.add_argument('--releasearea', action='store', metavar='DIR',
help="release directory (default: " + relarea_default + ")",
default=relarea_default, dest='rel_area')
@@ -633,5 +715,9 @@ if __name__ == "__main__":
logging.basicConfig(format=os.path.basename(sys.argv[0]) + ': %(message)s')
- packages, _ = package.read_packages(args.rel_area, args.arch)
- update_package_listings(args, packages, args.arch)
+# packages = {}
+# for arch in common_constants.ARCHES:
+# packages[arch], _ = package.read_packages(args.rel_area, arch)
+#
+# update_package_listings(args, packages)
+ write_doc_inc(args)
diff --git a/test/testdata/htdocs.expected/dirtree.expected
b/test/testdata/htdocs.expected/dirtree.expected
index ce68626..e24bab7 100644
--- a/test/testdata/htdocs.expected/dirtree.expected
+++ b/test/testdata/htdocs.expected/dirtree.expected
@@ -1,4 +1,6 @@
-{'.': ['packages.inc', 'src_packages.inc'],
+{'.': ['packages.inc', 'packages_docs.inc', 'src_packages.inc'],
+ 'doc': ['.htaccess'],
+ 'doc/keychain': ['keychain-2.6.8.README', 'keychain-2.7.1.README'],
'summary': ['arc-src.html',
'arc.html',
'base-cygwin.html',
diff --git a/test/testdata/htdocs.expected/doc/.htaccess
b/test/testdata/htdocs.expected/doc/.htaccess
new file mode 100644
index 0000000..522580e
--- /dev/null
+++ b/test/testdata/htdocs.expected/doc/.htaccess
@@ -0,0 +1 @@
+AddType text/plain README
diff --git a/test/testdata/htdocs.expected/doc/keychain/keychain-2.6.8.README
b/test/testdata/htdocs.expected/doc/keychain/keychain-2.6.8.README
new file mode 100644
index 0000000..32dbe29
--- /dev/null
+++ b/test/testdata/htdocs.expected/doc/keychain/keychain-2.6.8.README
@@ -0,0 +1,58 @@
+Keychain
+------------------------------------------
+Keychain is a manager for ssh-agent, typically run from ~/.bash_profile.
+It allows your shells and cron jobs to share a single ssh-agent process.
+
+Runtime requirements:
+ cygwin-1.5.18 or newer
+ openssh-4.5p1-1 or newer
+ bash-3.1-9 or newer
+ grep-2.5.1a-3 or newer
+ gawk-3.1.5-4 or newer
+ coreutils-6.4-1 or newer
+
+Canonical homepage:
+ http://www.gentoo.org/proj/en/keychain/
+
+Canonical download:
+ http://dev.gentoo.org/~agriffis/keychain/
+
+License:
+ GPL
+
+Language:
+ sh
+
+------------------------------------
+
+Build instructions:
+ unpack keychain-2.6.8-1-src.tar.bz2
+ if you use setup to install this src package, it will be
+ unpacked under /usr/src automatically
+ cd /usr/src
+ ./keychain-2.6.8-1.sh all
+
+This will create:
+ /usr/src/keychain-2.6.8-1.tar.bz2
+ /usr/src/keychain-2.6.8-1-src.tar.bz2
+
+Or use './keychain-2.6.8-1.sh prep' to get a patched source directory
+
+To find out the files included in the binary distribution, you can
+use "cygcheck -l keychain", or browse the listing for the appropriate version
+at <http://cygwin.com/packages/>.
+
+------------------
+
+Port Notes:
+
+---------- keychain-2.6.8-1 -- 11/08/2006 -----------
+New Maintainer: Jonathan C. Allen
+keychain 2.6.8 released
+
+For more information about this package, see the upstream documentation in
+/usr/share/doc/keychain-2.6.8.
+
+Cygwin port maintained by: Jonathan C. Allen <jca at sdf dot lonestar dot org>
+Please address all questions to the Cygwin mailing list:
+<cygwin at cygwin dot com>
diff --git a/test/testdata/htdocs.expected/doc/keychain/keychain-2.7.1.README
b/test/testdata/htdocs.expected/doc/keychain/keychain-2.7.1.README
new file mode 100644
index 0000000..364fa9c
--- /dev/null
+++ b/test/testdata/htdocs.expected/doc/keychain/keychain-2.7.1.README
@@ -0,0 +1,85 @@
+keychain
+------------------------------------------
+
+Key manager for OpenSSH
+
+Keychain is an OpenSSH key manager, typically run from
+~/.bash_profile. When keychain is run, it checks for a running
+ssh-agent, otherwise it starts one. It saves the ssh-agent environment
+variables to ~/.keychain/$HOSTNAME-sh, so that subsequent logins
+and non-interactive shells such as cron jobs can source the file and
+make passwordless ssh connections. In addition, when keychain runs, it
+verifies that the key files specified on the command-line are known to
+ssh-agent, otherwise it loads them, prompting you for a password if
+necessary.
+
+Runtime requirements:
+ openssh
+ bash
+ grep
+ gawk
+ coreutils (uname whoami)
+ cygwin (kill ps)
+
+Build requirements:
+ cygwin
+
+Canonical homepage:
+ http://freecode.com/projects/keychain
+ https://github.com/funtoo/keychain
+ http://packages.debian.org/keychain
+ http://packages.gentoo.org/package/net-misc/keychain
+
+ NOT RELATED:
+ http://code.google.com/p/keychain
+
+Canonical download:
+ http://www.funtoo.org/archive/keychain/
+
+Version control repository:
+ https://github.com/funtoo/keychain
+ git clone git://github.com/funtoo/keychain.git
+
+Mailing list:
+
+Forum / Wiki:
+
+IRC channel:
+
+Upstream contact:
+ Maintainer: [email protected] <drobbins at funtoo.org>
+ Bugs: https://github.com/funtoo/keychain/issues
+
+License:
+ GPL
+
+Language:
+ Bourne Shell
+
+------------------------------------------
+
+Build instructions:
+ unpack keychain-*-src.tar.bz2
+ if you use setup to install this source package, it will be
+ unpacked under /usr/src automatically
+ cd /usr/src
+ ./keychain*.sh all
+
+------------------------------------------
+
+Files included in the binary distribution:
+ See Cygwin package archive
+
+------------------------------------------
+
+Port Notes:
+
+----- version 2.7.1-1 -----
+- New maintainer 2012-10-10 Jari Aalto
+
+---------- keychain-2.6.8-1 -- 11/08/2006 -----------
+- New Maintainer Jonathan C. Allen
+
+Cygwin port maintained by: Jari Aalto <jari.aalto at cante.net>
+Copyright (C) 2012 Jari Aalto; Licensed under GPL v2 or later
+Copyright (C) 2006-2012 Jonathan C. Allen
diff --git a/test/testdata/htdocs.expected/packages_docs.inc
b/test/testdata/htdocs.expected/packages_docs.inc
new file mode 100644
index 0000000..7b4652e
--- /dev/null
+++ b/test/testdata/htdocs.expected/packages_docs.inc
@@ -0,0 +1,4 @@
+<div class="multicolumn-list">
+<p>keychain: <a href="doc/keychain/keychain-2.6.8.README">keychain-2.6.8</a>,
<a href="doc/keychain/keychain-2.7.1.README">keychain-2.7.1</a></p>
+<p><a href="https://x.cygwin.com/";>X</a></p>
+</div>
diff --git a/test/testdata/htdocs.expected/summary/keychain.html
b/test/testdata/htdocs.expected/summary/keychain.html
index 436ddb0..c8869fa 100644
--- a/test/testdata/htdocs.expected/summary/keychain.html
+++ b/test/testdata/htdocs.expected/summary/keychain.html
@@ -27,6 +27,7 @@ necessary</p></td></tr>
<tr><td><p><span class="detail">maintainer(s)</span>:</p></td><td><p>Jari Aalto
<span class="smaller">(Use <a href="/lists.html#cygwin">the mailing list</a>
to report bugs or ask questions.
<a href="/problems.html#personal-email">Do not contact the maintainer(s)
directly</a>.)</span></p></td></tr>
+<tr><td><p><span class="detail">readme</span>:</p></td><td><p><a
href="../doc/keychain/keychain-2.6.8.README">keychain-2.6.8.README</a>, <a
href="../doc/keychain/keychain-2.7.1.README">keychain-2.7.1.README</a></p></td></tr>
</table><br>
<table class="pkgtable">
<tr><th>Version</th><th>Arch</th><th>Package
Size</th><th>Date</th><th>Files</th><th>Status</th></tr>
diff --git a/test/testdata/process_arch/htdocs.expected
b/test/testdata/process_arch/htdocs.expected
index 57c3bc9..067d41c 100644
--- a/test/testdata/process_arch/htdocs.expected
+++ b/test/testdata/process_arch/htdocs.expected
@@ -1,4 +1,6 @@
-{'.': ['calm.db', 'packages.inc', 'reports_list.inc', 'src_packages.inc'],
+{'.': ['calm.db', 'packages.inc', 'packages_docs.inc', 'reports_list.inc',
'src_packages.inc'],
+ 'doc': ['.htaccess'],
+ 'doc/keychain': ['keychain-2.6.8.README', 'keychain-2.7.1.README'],
'reports': ['deprecated_so.html',
'maintainer_activity.html',
'perl_rebuilds.html',
[calm - Cygwin server-side packaging maintenance script] branch master, updated. 20250329-2-ga3c8059
Jon Turney via Cygwin-apps-cvs Sat, 31 May 2025 06:34:25 -0700
