On Mon, Jan 18, 2021 at 04:30:27PM -0000, futat...@apache.org wrote: > Author: futatuki > Date: Mon Jan 18 16:30:27 2021 > New Revision: 1885656 > > URL: http://svn.apache.org/viewvc?rev=1885656&view=rev > Log: > mailer.py: Restore Python 2 support.
mailer.py's test suite starts failing with Python3 after this change. I don't know yet how this could be fixed. I will try to find out. $ pwd /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests $ svn diff ~/svn/svn-trunk Index: /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/mailer.py =================================================================== --- /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/mailer.py (revision 1885780) +++ /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/mailer.py (working copy) @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/home/stsp/svn/prefix/python/bin/python3 # -*- coding: utf-8 -*- # # $ ./mailer-t1.sh mailer-init.33014/repos ../mailer.py Traceback (most recent call last): File "../mailer.py", line 1583, in <module> sys.argv[3:3+expected_args]) File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324, in run_app return func(application_pool, *args, **kw) File "../mailer.py", line 147, in main return messenger.generate() File "../mailer.py", line 533, in generate group, params, paths, subpool) File "../mailer.py", line 817, in generate_content renderer.render(data) File "../mailer.py", line 1167, in render self._render_diffs(data.diffs, '') File "../mailer.py", line 1223, in _render_diffs to_str(diff.base_path))) File "../mailer.py", line 88, in to_str return x.decode('utf-8') AttributeError: 'str' object has no attribute 'decode' Traceback (most recent call last): File "../mailer.py", line 1583, in <module> sys.argv[3:3+expected_args]) File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324, in run_app return func(application_pool, *args, **kw) File "../mailer.py", line 147, in main return messenger.generate() File "../mailer.py", line 533, in generate group, params, paths, subpool) File "../mailer.py", line 817, in generate_content renderer.render(data) File "../mailer.py", line 1167, in render self._render_diffs(data.diffs, '') File "../mailer.py", line 1227, in _render_diffs to_str(diff.base_path))) File "../mailer.py", line 88, in to_str return x.decode('utf-8') AttributeError: 'str' object has no attribute 'decode' Traceback (most recent call last): File "../mailer.py", line 1583, in <module> sys.argv[3:3+expected_args]) File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324, in run_app return func(application_pool, *args, **kw) File "../mailer.py", line 147, in main return messenger.generate() File "../mailer.py", line 533, in generate group, params, paths, subpool) File "../mailer.py", line 817, in generate_content renderer.render(data) File "../mailer.py", line 1167, in render self._render_diffs(data.diffs, '') File "../mailer.py", line 1217, in _render_diffs w('\nDeleted: %s\n' % to_str(diff.base_path)) File "../mailer.py", line 88, in to_str return x.decode('utf-8') AttributeError: 'str' object has no attribute 'decode' Traceback (most recent call last): File "../mailer.py", line 1583, in <module> sys.argv[3:3+expected_args]) File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324, in run_app return func(application_pool, *args, **kw) File "../mailer.py", line 147, in main return messenger.generate() File "../mailer.py", line 533, in generate group, params, paths, subpool) File "../mailer.py", line 817, in generate_content renderer.render(data) File "../mailer.py", line 1167, in render self._render_diffs(data.diffs, '') File "../mailer.py", line 1223, in _render_diffs to_str(diff.base_path))) File "../mailer.py", line 88, in to_str return x.decode('utf-8') AttributeError: 'str' object has no attribute 'decode' current mailer.py output in: /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current dos2unix: converting file /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current to Unix format... diff -q /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.output /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current Files /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.output and /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current differ $ > * tools/hook-scripts/mailer/mailer.py > (): > - Absorb difference of import module name. > - import codecs, to examine equivalence of codecs. > - Don't import locale. > (to_bytes): New function to absorb differnce between Python 2 and Python 3. > Replace occurence of .encode('utf-8') with this whole in this file. > (to_str): New function to absorb difference between Python 2 and Python 3. > Replace occurence of .decode('utf-8') with this whole in this file. > (_stdin): New variable to hold bytes I/O object for stdin > (_stdout): New variable to hold bytes I/O object for stdout > (OutputBase.make_subject): > - Truncate subject by number of bytes, for compatibility before r1884427. > - Truncate subject on character boundary (don't truncate in the middle of > multi-byte sequence of a UTF-8 character). > (StandardOutput.__init__): Use bytes output interface _stdout for > StandardOutput.write_binary. > (StandardOutput.wirte): > - Override this method only if on Python 3 and encoding of stdout is not > 'utf-8' > - Use sys.stdout.encoding instead of locale.getpreferredencoding(). > (PropChange.generate): Use bytes input interface _stdin. > (Lock.__init__): Use bytes input interface _stdin. > (DiffURLSelections._get_url): Remove extra trailing spaces. > (DiffGenerator.__getitem__): Prepare bytes and str representation of > base_path, 'base_path_bytes' and 'base_path', then use them properly. > > Modified: > subversion/trunk/tools/hook-scripts/mailer/mailer.py > > Modified: subversion/trunk/tools/hook-scripts/mailer/mailer.py > URL: > http://svn.apache.org/viewvc/subversion/trunk/tools/hook-scripts/mailer/mailer.py?rev=1885656&r1=1885655&r2=1885656&view=diff > ============================================================================== > --- subversion/trunk/tools/hook-scripts/mailer/mailer.py (original) > +++ subversion/trunk/tools/hook-scripts/mailer/mailer.py Mon Jan 18 16:30:27 > 2021 > @@ -46,15 +46,21 @@ > > import os > import sys > -import configparser > -from urllib.parse import quote as _url_quote > +if sys.hexversion >= 0x3000000: > + PY3 = True > + import configparser > + from urllib.parse import quote as _url_quote > +else: > + PY3 = False > + import ConfigParser as configparser > + from urllib import quote as _url_quote > import time > import subprocess > from io import BytesIO > import smtplib > import re > import tempfile > -import locale > +import codecs > > # Minimal version of Subversion's bindings required > _MIN_SVN_VERSION = [1, 5, 0] > @@ -73,6 +79,28 @@ if _MIN_SVN_VERSION > [svn.core.SVN_VER_ > % ".".join([str(x) for x in _MIN_SVN_VERSION])) > sys.exit(1) > > +# Absorb difference between Python 2 and Python >= 3 > +if PY3: > + def to_bytes(x): > + return x.encode('utf-8') > + > + def to_str(x): > + return x.decode('utf-8') > + > + # We never use sys.stdin nor sys.stdout TextIOwrapper. > + _stdin = sys.stdin.buffer > + _stdout = sys.stdout.buffer > +else: > + # Python 2 > + def to_bytes(x): > + return x > + > + def to_str(x): > + return x > + > + _stdin = sys.stdin > + _stdout = sys.stdout > + > > SEPARATOR = '=' * 78 > > @@ -151,8 +179,16 @@ class OutputBase: > except ValueError: > truncate_subject = 0 > > - if truncate_subject and len(subject) > truncate_subject: > - subject = subject[:(truncate_subject - 3)] + "..." > + # truncate subject as UTF-8 string. > + # Note: there still exists an issue on combining characters. > + if truncate_subject: > + bsubject = to_bytes(subject) > + if len(bsubject) > truncate_subject: > + idx = truncate_subject - 2 > + while b'\x80' <= bsubject[idx-1:idx] <= b'\xbf': > + idx -= 1 > + subject = to_str(bsubject[:idx-1]) + "..." > + > return subject > > def start(self, group, params): > @@ -177,7 +213,7 @@ class OutputBase: > > def write(self, output): > """Append the literal text string OUTPUT to the output representation.""" > - return self.write_binary(output.encode('utf-8')) > + return self.write_binary(to_bytes(output)) > > def run(self, cmd): > """Override this method, if the default implementation is not sufficient. > @@ -356,7 +392,7 @@ class StandardOutput(OutputBase): > > def __init__(self, cfg, repos, prefix_param): > OutputBase.__init__(self, cfg, repos, prefix_param) > - self.write_binary = sys.stdout.buffer.write > + self.write_binary = _stdout.write > > def start(self, group, params): > self.write("Group: " + (group or "defaults") + "\n") > @@ -365,10 +401,11 @@ class StandardOutput(OutputBase): > def finish(self): > pass > > - def write(self, output): > - """Write text as *default* encoding string""" > - return self.write_binary(output.encode(locale.getpreferredencoding(), > - 'backslashreplace')) > + if (PY3 and (codecs.lookup(sys.stdout.encoding) != > codecs.lookup('utf-8'))): > + def write(self, output): > + """Write text as *default* encoding string""" > + return self.write_binary(output.encode(sys.stdout.encoding, > + 'backslashreplace')) > > > class PipeOutput(MailedOutput): > @@ -431,8 +468,7 @@ class Commit(Messenger): > > self.changelist = sorted(editor.get_changes().items()) > > - log = (repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG) > - or b'').decode('utf-8') > + log = to_str(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG) or b'') > > # collect the set of groups and the unique sets of params for the options > self.groups = { } > @@ -451,7 +487,7 @@ class Commit(Messenger): > # figure out the changed directories > dirs = { } > for path, change in self.changelist: > - path = path.decode('utf-8') > + path = to_str(path) > if change.item_kind == svn.core.svn_node_dir: > dirs[path] = None > else: > @@ -542,7 +578,7 @@ class PropChange(Messenger): > elif self.action == 'M': > self.output.write('Property diff:\n') > tempfile1 = tempfile.NamedTemporaryFile() > - tempfile1.write(sys.stdin.buffer.read()) > + tempfile1.write(_stdin.read()) > tempfile1.flush() > tempfile2 = tempfile.NamedTemporaryFile() > tempfile2.write(self.repos.get_rev_prop(self.propname)) > @@ -604,8 +640,7 @@ class Lock(Messenger): > or 'unlock_subject_prefix')) > > # read all the locked paths from STDIN and strip off the trailing > newlines > - self.dirlist = [x.decode('utf-8').rstrip() > - for x in sys.stdin.buffer.readlines()] > + self.dirlist = [to_str(x).rstrip() for x in _stdin.readlines()] > > # collect the set of groups and the unique sets of params for the options > self.groups = { } > @@ -634,7 +669,7 @@ class Lock(Messenger): > # The lock comment is the same for all paths, so we can just pull > # the comment for the first path in the dirlist and cache it. > self.lock = svn.fs.svn_fs_get_lock(self.repos.fs_ptr, > - self.dirlist[0].encode('utf-8'), > + to_bytes(self.dirlist[0]), > self.pool) > > def generate(self): > @@ -709,7 +744,7 @@ class DiffURLSelections: > # KeyError exceptions. > params = self.params.copy() > params['path'] = _url_quote(change.path) if change.path else None > - params['base_path'] = (_url_quote(change.base_path) > + params['base_path'] = (_url_quote(change.base_path) > if change.base_path else None) > params['rev'] = repos_rev > params['base_rev'] = change.base_rev > @@ -764,8 +799,7 @@ def generate_content(renderer, cfg, repo > author=repos.author, > date=date, > rev=repos.rev, > - log=(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG) > - or b'').decode('utf-8'), > + log=to_str(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG) or b''), > commit_url=commit_url, > added_data=generate_list('A', changelist, paths, True), > replaced_data=generate_list('R', changelist, paths, True), > @@ -873,7 +907,9 @@ class DiffGenerator: > > # figure out if/how to generate a diff > > - base_path = remove_leading_slashes(change.base_path) > + base_path_bytes = remove_leading_slashes(change.base_path) > + base_path = (to_str(base_path_bytes) > + if base_path_bytes is not None else None) > if change.action == svn.repos.CHANGE_ACTION_DELETE: > # it was delete. > kind = 'D' > @@ -884,10 +920,9 @@ class DiffGenerator: > # show the diff? > if self.diffsels.delete: > diff = svn.fs.FileDiff(self.repos.get_root(change.base_rev), > - base_path, None, None, self.pool) > + base_path_bytes, None, None, self.pool) > > - label1 = '%s\t%s\t(r%s)' % (base_path.decode('utf-8'), self.date, > - change.base_rev) > + label1 = '%s\t%s\t(r%s)' % (base_path, self.date, change.base_rev) > label2 = '/dev/null\t00:00:00 1970\t(deleted)' > singular = True > > @@ -906,14 +941,13 @@ class DiffGenerator: > # show the diff? > if self.diffsels.modify: > diff = svn.fs.FileDiff(self.repos.get_root(change.base_rev), > - base_path, > + base_path_bytes, > self.repos.root_this, change.path, > self.pool) > - label1 = '%s\t%s\t(r%s, copy source)' \ > - % (base_path.decode('utf-8'), base_date, > change.base_rev) > - label2 = '%s\t%s\t(r%s)' \ > - % (change.path.decode('utf-8'), self.date, \ > - self.repos.rev) > + label1 = ('%s\t%s\t(r%s, copy source)' > + % (base_path, base_date, change.base_rev)) > + label2 = ('%s\t%s\t(r%s)' > + % (to_str(change.path), self.date, self.repos.rev)) > singular = False > else: > # this file was copied. > @@ -921,12 +955,12 @@ class DiffGenerator: > if self.diffsels.copy: > diff = svn.fs.FileDiff(None, None, self.repos.root_this, > change.path, self.pool) > - label1 = '/dev/null\t00:00:00 1970\t' \ > - '(empty, because file is newly added)' > - label2 = '%s\t%s\t(r%s, copy of r%s, %s)' \ > - % (change.path.decode('utf-8'), self.date, > - self.repos.rev, change.base_rev, > - base_path.decode('utf-8')) > + label1 = ('/dev/null\t00:00:00 1970\t' > + '(empty, because file is newly added)') > + label2 = ('%s\t%s\t(r%s, copy of r%s, %s)' > + % (to_str(change.path), > + self.date, self.repos.rev, change.base_rev, > + base_path)) > singular = False > else: > # the file was added. > @@ -942,7 +976,7 @@ class DiffGenerator: > label1 = '/dev/null\t00:00:00 1970\t' \ > '(empty, because file is newly added)' > label2 = '%s\t%s\t(r%s)' \ > - % (change.path.decode('utf-8'), self.date, > self.repos.rev) > + % (to_str(change.path), self.date, self.repos.rev) > singular = True > > elif not change.text_changed: > @@ -962,9 +996,9 @@ class DiffGenerator: > self.repos.root_this, change.path, > self.pool) > label1 = '%s\t%s\t(r%s)' \ > - % (base_path.decode('utf-8'), base_date, change.base_rev) > + % (base_path, base_date, change.base_rev) > label2 = '%s\t%s\t(r%s)' \ > - % (change.path.decode('utf-8'), self.date, self.repos.rev) > + % (to_str(change.path), self.date, self.repos.rev) > singular = False > > if diff: > @@ -1154,7 +1188,7 @@ class TextCommitRenderer: > props = ' (props changed)' > else: > props = '' > - w(' %s%s%s\n' % (d.path.decode('utf-8'), is_dir, props)) > + w(' %s%s%s\n' % (to_str(d.path), is_dir, props)) > if d.copied: > if is_dir: > text = '' > @@ -1163,7 +1197,7 @@ class TextCommitRenderer: > else: > text = ' unchanged' > w(' - copied%s from r%d, %s%s\n' > - % (text, d.base_rev, d.base_path.decode('utf-8'), is_dir)) > + % (text, d.base_rev, to_str(d.base_path), is_dir)) > > def _render_diffs(self, diffs, section_header): > """Render diffs. Write the SECTION_HEADER if there are actually > @@ -1180,20 +1214,20 @@ class TextCommitRenderer: > w(section_header) > section_header_printed = True > if diff.kind == 'D': > - w('\nDeleted: %s\n' % diff.base_path.decode('utf-8')) > + w('\nDeleted: %s\n' % to_str(diff.base_path)) > elif diff.kind == 'A': > - w('\nAdded: %s\n' % diff.path.decode('utf-8')) > + w('\nAdded: %s\n' % to_str(diff.path)) > elif diff.kind == 'C': > w('\nCopied: %s (from r%d, %s)\n' > - % (diff.path.decode('utf-8'), diff.base_rev, > - diff.base_path.decode('utf-8'))) > + % (to_str(diff.path), diff.base_rev, > + to_str(diff.base_path))) > elif diff.kind == 'W': > w('\nCopied and modified: %s (from r%d, %s)\n' > - % (diff.path.decode('utf-8'), diff.base_rev, > - diff.base_path.decode('utf-8'))) > + % (to_str(diff.path), diff.base_rev, > + to_str(diff.base_path))) > else: > # kind == 'M' > - w('\nModified: %s\n' % diff.path.decode('utf-8')) > + w('\nModified: %s\n' % to_str(diff.path)) > > if diff.diff_url: > w('URL: %s\n' % diff.diff_url) > @@ -1232,7 +1266,7 @@ class Repository: > > self.author = self.get_rev_prop(svn.core.SVN_PROP_REVISION_AUTHOR) > if self.author is not None: > - self.author = self.author.decode('utf-8') > + self.author = to_str(self.author) > > def get_rev_prop(self, propname, rev = None): > if not rev: > @@ -1441,9 +1475,9 @@ class Config: > "Return the path's associated groups." > groups = [] > for group, pattern, exclude_pattern, repos_params, search_logmsg_re in > self._group_re: > - match = pattern.match(path.decode('utf-8')) > + match = pattern.match(to_str(path)) > if match: > - if exclude_pattern and exclude_pattern.match(path.decode('utf-8')): > + if exclude_pattern and exclude_pattern.match(to_str(path)): > continue > params = repos_params.copy() > params.update(match.groupdict()) > @@ -1522,8 +1556,7 @@ if the property was added, modified or d > usage() > > cmd = sys.argv[1] > - repos_dir = svn.core.svn_path_canonicalize(sys.argv[2].encode('utf-8')) > - repos_dir = repos_dir.decode('utf-8') > + repos_dir = to_str(svn.core.svn_path_canonicalize(to_bytes(sys.argv[2]))) > try: > expected_args = cmd_list[cmd] > except KeyError: > > >