Author: brane Date: Mon Nov 4 05:50:33 2019 New Revision: 1869353 URL: http://svn.apache.org/viewvc?rev=1869353&view=rev Log: On the swig-py3 branch: Sync from trunk up to r1869352.
Modified: subversion/branches/swig-py3/ (props changed) subversion/branches/swig-py3/CHANGES subversion/branches/swig-py3/COMMITTERS subversion/branches/swig-py3/INSTALL subversion/branches/swig-py3/tools/dist/release.py subversion/branches/swig-py3/tools/dist/templates/download.ezt subversion/branches/swig-py3/tools/dist/templates/rc-release-ann.ezt subversion/branches/swig-py3/tools/dist/templates/stable-release-ann.ezt subversion/branches/swig-py3/tools/hook-scripts/mailer/mailer.py Propchange: subversion/branches/swig-py3/ ------------------------------------------------------------------------------ Merged /subversion/trunk:r1868692-1869352 Modified: subversion/branches/swig-py3/CHANGES URL: http://svn.apache.org/viewvc/subversion/branches/swig-py3/CHANGES?rev=1869353&r1=1869352&r2=1869353&view=diff ============================================================================== --- subversion/branches/swig-py3/CHANGES (original) +++ subversion/branches/swig-py3/CHANGES Mon Nov 4 05:50:33 2019 @@ -9,7 +9,7 @@ Version 1.14.0 https://svn.apache.org/repos/asf/subversion/tags/1.14.0 Version 1.13.0 -(16 Oct 2019, from /branches/1.13.x) +(30 Oct 2019, from /branches/1.13.x) https://svn.apache.org/repos/asf/subversion/tags/1.13.0 User-visible changes: Modified: subversion/branches/swig-py3/COMMITTERS URL: http://svn.apache.org/viewvc/subversion/branches/swig-py3/COMMITTERS?rev=1869353&r1=1869352&r2=1869353&view=diff ============================================================================== --- subversion/branches/swig-py3/COMMITTERS [UTF-8] (original) +++ subversion/branches/swig-py3/COMMITTERS [UTF-8] Mon Nov 4 05:50:33 2019 @@ -63,6 +63,7 @@ Blanket commit access: luke1410 Stefan Hett <luke1...@posteo.de> troycurtisjr Troy Curtis, Jr <troycurti...@gmail.com> hartmannathan Nathan Hartman <hartman.nat...@gmail.com> + futatuki Yasuhito Futatsuki <futat...@yf.bsdclub.org> [[END ACTIVE FULL COMMITTERS. LEAVE THIS LINE HERE; SCRIPTS LOOK FOR IT.]] @@ -215,7 +216,6 @@ giorgio_valoti Giorgio Valoti <giorgio holden Holden Karau <hol...@pigscanfly.ca> (scheme-bindings br.) moklo Morten Kloster <mor...@gmail.com> (diff-improvements br.) vmpn Vladimir Berezniker <v...@hitechman.com> (javahl-ra br.) - futatuki Yasuhito Futatsuki <futat...@yf.bsdclub.org> (swig-py3 br.) Subprojects that are complete, abandoned or have moved elsewhere: Modified: subversion/branches/swig-py3/INSTALL URL: http://svn.apache.org/viewvc/subversion/branches/swig-py3/INSTALL?rev=1869353&r1=1869352&r2=1869353&view=diff ============================================================================== --- subversion/branches/swig-py3/INSTALL (original) +++ subversion/branches/swig-py3/INSTALL Mon Nov 4 05:50:33 2019 @@ -419,6 +419,7 @@ I. INTRODUCTION test suite is written in Python, as is part of Subversion's build system. + Note that Python 3.x on Windows is not supported. 11. Perl 5.8 or newer (Windows only) (OPTIONAL) Modified: subversion/branches/swig-py3/tools/dist/release.py URL: http://svn.apache.org/viewvc/subversion/branches/swig-py3/tools/dist/release.py?rev=1869353&r1=1869352&r2=1869353&view=diff ============================================================================== --- subversion/branches/swig-py3/tools/dist/release.py (original) +++ subversion/branches/swig-py3/tools/dist/release.py Mon Nov 4 05:50:33 2019 @@ -41,7 +41,10 @@ import sys import glob import fnmatch import shutil -import urllib2 +try: + from urllib.request import urlopen # Python 3 +except: + from urllib2 import urlopen # Python 2 import hashlib import tarfile import logging @@ -102,7 +105,7 @@ tool_versions['trunk'] = tool_versions[' # ### TODO: derive this from svn_version.h; see ../../build/getversion.py recommended_release = '1.13' # For clean-dist, a whitelist of artifacts to keep, by version. -supported_release_lines = frozenset({"1.9", "1.10", "1.12", "1.13"}) +supported_release_lines = frozenset({"1.9", "1.10", "1.13"}) # Some constants svn_repos = os.getenv('SVN_RELEASE_SVN_REPOS', @@ -269,7 +272,7 @@ def get_tmplfile(filename): return open(os.path.join(get_tmpldir(), filename)) except IOError: # Hmm, we had a problem with the local version, let's try the repo - return urllib2.urlopen(svn_repos + '/trunk/tools/dist/templates/' + filename) + return urlopen(svn_repos + '/trunk/tools/dist/templates/' + filename) def get_nullfile(): return open(os.path.devnull, 'w') @@ -295,15 +298,22 @@ def run_script(verbose, script, hide_std run_command(l.split(), verbose, hide_stderr) def download_file(url, target, checksum): - response = urllib2.urlopen(url) - target_file = open(target, 'w+') + """Download the file at URL to the local path TARGET. + If CHECKSUM is a string, verify the checksum of the downloaded + file and raise RuntimeError if it does not match. If CHECKSUM + is None, do not verify the downloaded file. + """ + assert checksum is None or isinstance(checksum, str) + + response = urlopen(url) + target_file = open(target, 'w+b') target_file.write(response.read()) target_file.seek(0) m = hashlib.sha256() m.update(target_file.read()) target_file.close() checksum2 = m.hexdigest() - if checksum != checksum2: + if checksum is not None and checksum != checksum2: raise RuntimeError("Checksum mismatch for '%s': "\ "downloaded: '%s'; expected: '%s'" % \ (target, checksum, checksum2)) @@ -369,7 +379,8 @@ class RollDep(object): def _test_version(self, cmd): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + universal_newlines=True) (stdout, stderr) = proc.communicate() rc = proc.wait() if rc: return '' @@ -728,7 +739,7 @@ def compare_changes(repos, branch, revis mergeinfo_cmd = ['svn', 'mergeinfo', '--show-revs=eligible', repos + '/trunk/CHANGES', repos + '/' + branch + '/' + 'CHANGES'] - stdout = subprocess.check_output(mergeinfo_cmd) + stdout = subprocess.check_output(mergeinfo_cmd, universal_newlines=True) if stdout: # Treat this as a warning since we are now putting entries for future # minor releases in CHANGES on trunk. @@ -746,7 +757,7 @@ def check_copyright_year(repos, branch, file_url = (repos + '/' + branch + '/' + branch_relpath + '@' + str(revision)) cat_cmd = ['svn', 'cat', file_url] - stdout = subprocess.check_output(cat_cmd) + stdout = subprocess.check_output(cat_cmd, universal_newlines=True) m = _copyright_re.search(stdout) if m: year = m.group('year') @@ -955,10 +966,10 @@ def roll_tarballs(args): # They are deprecated, however, so we don't publicly link them in # the announcements any more. m = hashlib.sha1() - m.update(open(filepath, 'r').read()) + m.update(open(filepath, 'rb').read()) open(filepath + '.sha1', 'w').write(m.hexdigest()) m = hashlib.sha512() - m.update(open(filepath, 'r').read()) + m.update(open(filepath, 'rb').read()) open(filepath + '.sha512', 'w').write(m.hexdigest()) # Nightlies do not get tagged so do not need the header @@ -966,7 +977,15 @@ def roll_tarballs(args): shutil.copy(os.path.join(get_workdir(args.base_dir), 'subversion', 'include', 'svn_version.h'), os.path.join(get_target(args), - 'svn_version.h.dist-%s' % str(args.version))) + 'svn_version.h.dist-%s' + % (str(args.version),))) + + # Download and "tag" the KEYS file (in case a signing key is removed + # from a committer's LDAP profile down the road) + basename = 'subversion-%s.KEYS' % (str(args.version),) + filepath = os.path.join(get_tempdir(args.base_dir), basename) + download_file(KEYS, filepath, None) + shutil.move(filepath, get_target(args)) # And we're done! @@ -1077,7 +1096,8 @@ def bump_versions_on_branch(args): args.version.patch + 1)) HEAD = subprocess.check_output(['svn', 'info', '--show-item=revision', - '--', branch_url]).strip() + '--', branch_url], + universal_newlines=True).strip() HEAD = int(HEAD) def file_object_for(relpath): fd = tempfile.NamedTemporaryFile() @@ -1124,7 +1144,8 @@ def clean_dist(args): '''Clean the distribution directory of release artifacts of no-longer-supported minor lines.''' - stdout = subprocess.check_output(['svn', 'list', dist_release_url]) + stdout = subprocess.check_output(['svn', 'list', dist_release_url], + universal_newlines=True) def minor(version): """Return the minor release line of the parameter, which must be @@ -1165,7 +1186,8 @@ def clean_dist(args): def move_to_dist(args): 'Move candidate artifacts to the distribution directory.' - stdout = subprocess.check_output(['svn', 'list', dist_dev_url]) + stdout = subprocess.check_output(['svn', 'list', dist_dev_url], + universal_newlines=True) filenames = [] for entry in stdout.split('\n'): @@ -1188,8 +1210,12 @@ def move_to_dist(args): def write_news(args): 'Write text for the Subversion website.' - data = { 'date' : datetime.date.today().strftime('%Y%m%d'), - 'date_pres' : datetime.date.today().strftime('%Y-%m-%d'), + if args.news_release_date: + release_date = datetime.datetime.strptime(args.news_release_date, '%Y-%m-%d') + else: + release_date = datetime.date.today() + data = { 'date' : release_date.strftime('%Y%m%d'), + 'date_pres' : release_date.strftime('%Y-%m-%d'), 'major-minor' : args.version.branch, 'version' : str(args.version), 'version_base' : args.version.base, @@ -1349,9 +1375,9 @@ def get_siginfo(args, quiet=False): % (n, filename, key_end)) sys.exit(1) - fd, fn = tempfile.mkstemp() - os.write(fd, key_start + key) - os.close(fd) + fd, fn = tempfile.mkstemp(text=True) + with os.fdopen(fd, 'w') as key_file: + key_file.write(key_start + key) verified = gpg.verify_file(open(fn, 'rb'), filename[:-4]) os.unlink(fn) @@ -1373,6 +1399,7 @@ def get_siginfo(args, quiet=False): gpg_output = subprocess.check_output( ['gpg', '--fixed-list-mode', '--with-colons', '--fingerprint', id], stderr=subprocess.STDOUT, + universal_newlines=True, ) gpg_output = gpg_output.splitlines() @@ -1440,7 +1467,7 @@ def get_keys(args): 'Import the LDAP-based KEYS file to gpg' # We use a tempfile because urlopen() objects don't have a .fileno() with tempfile.SpooledTemporaryFile() as fd: - fd.write(urllib2.urlopen(KEYS).read()) + fd.write(urlopen(KEYS).read()) fd.flush() fd.seek(0) subprocess.check_call(['gpg', '--import'], stdin=fd) @@ -1521,7 +1548,8 @@ def write_changelog(args): separator_line = ('-' * 72) + '\n' mergeinfo = subprocess.check_output(['svn', 'mergeinfo', '--show-revs', - 'eligible', '--log', branch_url, previous]) + 'eligible', '--log', branch_url, previous], + universal_newlines=True) log_messages_dict = { # This is a dictionary mapping revision numbers to their respective # log messages. The expression in the "key:" part of the dict @@ -1796,6 +1824,9 @@ def main(): subparser.set_defaults(func=write_news) subparser.add_argument('--announcement-url', help='''The URL to the archived announcement email.''') + subparser.add_argument('--news-release-date', + help='''The release date for the news, as YYYY-MM-DD. + Default: today.''') subparser.add_argument('--edit-html-file', help='''Insert the text into this file news.html, index.html).''') Modified: subversion/branches/swig-py3/tools/dist/templates/download.ezt URL: http://svn.apache.org/viewvc/subversion/branches/swig-py3/tools/dist/templates/download.ezt?rev=1869353&r1=1869352&r2=1869353&view=diff ============================================================================== --- subversion/branches/swig-py3/tools/dist/templates/download.ezt (original) +++ subversion/branches/swig-py3/tools/dist/templates/download.ezt Mon Nov 4 05:50:33 2019 @@ -4,10 +4,12 @@ <th>File</th> <th>Checksum (SHA512)</th> <th>Signatures</th> + <th>PGP Public Keys</th> </tr> [for fileinfo]<tr> <td><a href="[[]preferred]subversion/[fileinfo.filename]">[fileinfo.filename]</a></td> <td>[<a href="https://www.apache.org/dist/subversion/[fileinfo.filename].sha512">SHA-512</a>]</td> - <td>[<a href="https://www.apache.org/dist/subversion/[fileinfo.filename].asc">PGP</a>]</td> + <td>[<a href="https://www.apache.org/dist/subversion/[fileinfo.filename].asc">PGP signatures</a>]</td> + <td>[<a href="https://www.apache.org/dist/subversion/subversion-[version].KEYS">PGP keyring</a>]</td> </tr>[end] </table> Modified: subversion/branches/swig-py3/tools/dist/templates/rc-release-ann.ezt URL: http://svn.apache.org/viewvc/subversion/branches/swig-py3/tools/dist/templates/rc-release-ann.ezt?rev=1869353&r1=1869352&r2=1869353&view=diff ============================================================================== --- subversion/branches/swig-py3/tools/dist/templates/rc-release-ann.ezt (original) +++ subversion/branches/swig-py3/tools/dist/templates/rc-release-ann.ezt Mon Nov 4 05:50:33 2019 @@ -23,6 +23,10 @@ PGP Signatures are available at: For this release, the following people have provided PGP signatures: [siginfo] +These public keys are available at: + + https://www.apache.org/dist/subversion/subversion-[version].KEYS + This is a pre-release for what will eventually become version [major-minor-patch] of the Apache Subversion open source version control system. It may contain known issues, a complete list of [major-minor-patch]-blocking issues can be found Modified: subversion/branches/swig-py3/tools/dist/templates/stable-release-ann.ezt URL: http://svn.apache.org/viewvc/subversion/branches/swig-py3/tools/dist/templates/stable-release-ann.ezt?rev=1869353&r1=1869352&r2=1869353&view=diff ============================================================================== --- subversion/branches/swig-py3/tools/dist/templates/stable-release-ann.ezt (original) +++ subversion/branches/swig-py3/tools/dist/templates/stable-release-ann.ezt Mon Nov 4 05:50:33 2019 @@ -34,6 +34,10 @@ PGP Signatures are available at: For this release, the following people have provided PGP signatures: [siginfo] +These public keys are available at: + + https://www.apache.org/dist/subversion/subversion-[version].KEYS + Release notes for the [major-minor].x release series may be found at: https://subversion.apache.org/docs/release-notes/[major-minor].html Modified: subversion/branches/swig-py3/tools/hook-scripts/mailer/mailer.py URL: http://svn.apache.org/viewvc/subversion/branches/swig-py3/tools/hook-scripts/mailer/mailer.py?rev=1869353&r1=1869352&r2=1869353&view=diff ============================================================================== --- subversion/branches/swig-py3/tools/hook-scripts/mailer/mailer.py (original) +++ subversion/branches/swig-py3/tools/hook-scripts/mailer/mailer.py Mon Nov 4 05:50:33 2019 @@ -123,7 +123,7 @@ def main(pool, cmd, config_fname, repos_ else: raise UnknownSubcommand(cmd) - messenger.generate() + return messenger.generate() def remove_leading_slashes(path): @@ -285,15 +285,61 @@ class SMTPOutput(MailedOutput): self.write(self.mail_headers(group, params)) def finish(self): - if self.cfg.is_set('general.smtp_ssl') and self.cfg.general.smtp_ssl == 'yes': - server = smtplib.SMTP_SSL(self.cfg.general.smtp_hostname) - else: - server = smtplib.SMTP(self.cfg.general.smtp_hostname) - if self.cfg.is_set('general.smtp_username'): - server.login(self.cfg.general.smtp_username, - self.cfg.general.smtp_password) - server.sendmail(self.from_addr, self.to_addrs, self.buffer.getvalue()) - server.quit() + """ + Send email via SMTP or SMTP_SSL, logging in if username is + specified. + + Errors such as invalid recipient, which affect a particular email, + are reported to stderr and raise MessageSendFailure. If the caller + has other emails to send, it may continue doing so. + + Errors caused by bad configuration, such as login failures, for + which too many occurrences could lead to SMTP server lockout, are + reported to stderr and re-raised. These should be considered fatal + (to minimize the chances of said lockout). + """ + + try: + if self.cfg.is_set('general.smtp_ssl') and self.cfg.general.smtp_ssl == 'yes': + server = smtplib.SMTP_SSL(self.cfg.general.smtp_hostname) + else: + server = smtplib.SMTP(self.cfg.general.smtp_hostname) + except Exception as detail: + sys.stderr.write("mailer.py: Failed to instantiate SMTP object: %s\n" % (detail,)) + # Any error to instantiate is fatal + raise + + try: + if self.cfg.is_set('general.smtp_username'): + try: + server.login(self.cfg.general.smtp_username, + self.cfg.general.smtp_password) + except smtplib.SMTPException as detail: + sys.stderr.write("mailer.py: SMTP login failed with username %s and/or password: %s\n" + % (self.cfg.general.smtp_username, detail,)) + # Any error at login is fatal + raise + + server.sendmail(self.from_addr, self.to_addrs, self.buffer.getvalue()) + + except smtplib.SMTPRecipientsRefused as detail: + sys.stderr.write("mailer.py: SMTP recipient(s) refused: %s: %s\n" + % (self.to_addrs, detail,)) + raise MessageSendFailure from detail + + except smtplib.SMTPSenderRefused as detail: + sys.stderr.write("mailer.py: SMTP sender refused: %s: %s\n" + % (self.from_addr, detail,)) + raise MessageSendFailure from detail + + except smtplib.SMTPException as detail: + # All other errors are fatal; this includes: + # SMTPHeloError, SMTPDataError, SMTPNotSupportedError + sys.stderr.write("mailer.py: SMTP error occurred: %s\n" % (detail,)) + raise + + finally: + server.quit() class StandardOutput(OutputBase): @@ -421,21 +467,26 @@ class Commit(Messenger): ### rather than rebuilding it each time. subpool = svn.core.svn_pool_create(self.pool) + ret = 0 # build a renderer, tied to our output stream renderer = TextCommitRenderer(self.output) for (group, param_tuple), (params, paths) in self.groups.items(): - self.output.start(group, params) - - # generate the content for this group and set of params - generate_content(renderer, self.cfg, self.repos, self.changelist, - group, params, paths, subpool) + try: + self.output.start(group, params) - self.output.finish() + # generate the content for this group and set of params + generate_content(renderer, self.cfg, self.repos, self.changelist, + group, params, paths, subpool) + + self.output.finish() + except MessageSendFailure: + ret = 1 svn.core.svn_pool_clear(subpool) svn.core.svn_pool_destroy(subpool) + return ret class PropChange(Messenger): @@ -456,35 +507,40 @@ class PropChange(Messenger): def generate(self): actions = { 'A': 'added', 'M': 'modified', 'D': 'deleted' } + ret = 0 for (group, param_tuple), params in self.groups.items(): - self.output.start(group, params) - self.output.write('Author: %s\n' - 'Revision: %s\n' - 'Property Name: %s\n' - 'Action: %s\n' - '\n' - % (self.author, self.repos.rev, self.propname, - actions.get(self.action, 'Unknown (\'%s\')' \ - % self.action))) - if self.action == 'A' or self.action not in actions: - self.output.write('Property value:\n') - propvalue = self.repos.get_rev_prop(self.propname) - self.output.write(propvalue) - elif self.action == 'M': - self.output.write('Property diff:\n') - tempfile1 = tempfile.NamedTemporaryFile() - tempfile1.write(sys.stdin.read()) - tempfile1.flush() - tempfile2 = tempfile.NamedTemporaryFile() - tempfile2.write(self.repos.get_rev_prop(self.propname)) - tempfile2.flush() - self.output.run(self.cfg.get_diff_cmd(group, { - 'label_from' : 'old property value', - 'label_to' : 'new property value', - 'from' : tempfile1.name, - 'to' : tempfile2.name, - })) - self.output.finish() + try: + self.output.start(group, params) + self.output.write('Author: %s\n' + 'Revision: %s\n' + 'Property Name: %s\n' + 'Action: %s\n' + '\n' + % (self.author, self.repos.rev, self.propname, + actions.get(self.action, 'Unknown (\'%s\')' \ + % self.action))) + if self.action == 'A' or self.action not in actions: + self.output.write('Property value:\n') + propvalue = self.repos.get_rev_prop(self.propname) + self.output.write(propvalue) + elif self.action == 'M': + self.output.write('Property diff:\n') + tempfile1 = tempfile.NamedTemporaryFile() + tempfile1.write(sys.stdin.read()) + tempfile1.flush() + tempfile2 = tempfile.NamedTemporaryFile() + tempfile2.write(self.repos.get_rev_prop(self.propname)) + tempfile2.flush() + self.output.run(self.cfg.get_diff_cmd(group, { + 'label_from' : 'old property value', + 'label_to' : 'new property value', + 'from' : tempfile1.name, + 'to' : tempfile2.name, + })) + self.output.finish() + except MessageSendFailure: + ret = 1 + return ret def get_commondir(dirlist): @@ -564,21 +620,26 @@ class Lock(Messenger): self.dirlist[0], self.pool) def generate(self): + ret = 0 for (group, param_tuple), (params, paths) in self.groups.items(): - self.output.start(group, params) - - self.output.write('Author: %s\n' - '%s paths:\n' % - (self.author, self.do_lock and 'Locked' or 'Unlocked')) - - self.dirlist.sort() - for dir in self.dirlist: - self.output.write(' %s\n\n' % dir) - - if self.do_lock: - self.output.write('Comment:\n%s\n' % (self.lock.comment or '')) + try: + self.output.start(group, params) - self.output.finish() + self.output.write('Author: %s\n' + '%s paths:\n' % + (self.author, self.do_lock and 'Locked' or 'Unlocked')) + + self.dirlist.sort() + for dir in self.dirlist: + self.output.write(' %s\n\n' % dir) + + if self.do_lock: + self.output.write('Comment:\n%s\n' % (self.lock.comment or '')) + + self.output.finish() + except MessageSendFailure: + ret = 1 + return ret class DiffSelections: @@ -1394,6 +1455,8 @@ class UnknownMappingSpec(Exception): pass class UnknownSubcommand(Exception): pass +class MessageSendFailure(Exception): + pass if __name__ == '__main__': @@ -1455,8 +1518,9 @@ if the property was added, modified or d if not os.path.exists(config_fname): raise MissingConfig(config_fname) - svn.core.run_app(main, cmd, config_fname, repos_dir, - sys.argv[3:3+expected_args]) + ret = svn.core.run_app(main, cmd, config_fname, repos_dir, + sys.argv[3:3+expected_args]) + sys.exit(1 if ret else 0) # ------------------------------------------------------------------------ # TODO