On Mon, Jan 18, 2021 at 04:30:27PM -0000, futat...@apache.org wrote:
> Author: futatuki
> Date: Mon Jan 18 16:30:27 2021
> New Revision: 1885656
>
> URL: http://svn.apache.org/viewvc?rev=1885656&view=rev
> Log:
> mailer.py: Restore Python 2 support.
mailer.py's test suite starts failing with Python3 after this change.
I don't know yet how this could be fixed. I will try to find out.
$ pwd
/home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests
$ svn diff ~/svn/svn-trunk
Index: /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/mailer.py
===================================================================
--- /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/mailer.py
(revision 1885780)
+++ /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/mailer.py
(working copy)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/home/stsp/svn/prefix/python/bin/python3
# -*- coding: utf-8 -*-
#
#
$ ./mailer-t1.sh mailer-init.33014/repos ../mailer.py
Traceback (most recent call last):
File "../mailer.py", line 1583, in <module>
sys.argv[3:3+expected_args])
File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324,
in run_app
return func(application_pool, *args, **kw)
File "../mailer.py", line 147, in main
return messenger.generate()
File "../mailer.py", line 533, in generate
group, params, paths, subpool)
File "../mailer.py", line 817, in generate_content
renderer.render(data)
File "../mailer.py", line 1167, in render
self._render_diffs(data.diffs, '')
File "../mailer.py", line 1223, in _render_diffs
to_str(diff.base_path)))
File "../mailer.py", line 88, in to_str
return x.decode('utf-8')
AttributeError: 'str' object has no attribute 'decode'
Traceback (most recent call last):
File "../mailer.py", line 1583, in <module>
sys.argv[3:3+expected_args])
File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324,
in run_app
return func(application_pool, *args, **kw)
File "../mailer.py", line 147, in main
return messenger.generate()
File "../mailer.py", line 533, in generate
group, params, paths, subpool)
File "../mailer.py", line 817, in generate_content
renderer.render(data)
File "../mailer.py", line 1167, in render
self._render_diffs(data.diffs, '')
File "../mailer.py", line 1227, in _render_diffs
to_str(diff.base_path)))
File "../mailer.py", line 88, in to_str
return x.decode('utf-8')
AttributeError: 'str' object has no attribute 'decode'
Traceback (most recent call last):
File "../mailer.py", line 1583, in <module>
sys.argv[3:3+expected_args])
File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324,
in run_app
return func(application_pool, *args, **kw)
File "../mailer.py", line 147, in main
return messenger.generate()
File "../mailer.py", line 533, in generate
group, params, paths, subpool)
File "../mailer.py", line 817, in generate_content
renderer.render(data)
File "../mailer.py", line 1167, in render
self._render_diffs(data.diffs, '')
File "../mailer.py", line 1217, in _render_diffs
w('\nDeleted: %s\n' % to_str(diff.base_path))
File "../mailer.py", line 88, in to_str
return x.decode('utf-8')
AttributeError: 'str' object has no attribute 'decode'
Traceback (most recent call last):
File "../mailer.py", line 1583, in <module>
sys.argv[3:3+expected_args])
File "/home/stsp/svn/prefix/svn-trunk/lib/svn-python/svn/core.py", line 324,
in run_app
return func(application_pool, *args, **kw)
File "../mailer.py", line 147, in main
return messenger.generate()
File "../mailer.py", line 533, in generate
group, params, paths, subpool)
File "../mailer.py", line 817, in generate_content
renderer.render(data)
File "../mailer.py", line 1167, in render
self._render_diffs(data.diffs, '')
File "../mailer.py", line 1223, in _render_diffs
to_str(diff.base_path)))
File "../mailer.py", line 88, in to_str
return x.decode('utf-8')
AttributeError: 'str' object has no attribute 'decode'
current mailer.py output in:
/home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current
dos2unix: converting file
/home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current to
Unix format...
diff -q
/home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.output
/home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current
Files /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.output
and /home/stsp/svn/svn-trunk/tools/hook-scripts/mailer/tests/mailer-t1.current
differ
$
> * tools/hook-scripts/mailer/mailer.py
> ():
> - Absorb difference of import module name.
> - import codecs, to examine equivalence of codecs.
> - Don't import locale.
> (to_bytes): New function to absorb differnce between Python 2 and Python 3.
> Replace occurence of .encode('utf-8') with this whole in this file.
> (to_str): New function to absorb difference between Python 2 and Python 3.
> Replace occurence of .decode('utf-8') with this whole in this file.
> (_stdin): New variable to hold bytes I/O object for stdin
> (_stdout): New variable to hold bytes I/O object for stdout
> (OutputBase.make_subject):
> - Truncate subject by number of bytes, for compatibility before r1884427.
> - Truncate subject on character boundary (don't truncate in the middle of
> multi-byte sequence of a UTF-8 character).
> (StandardOutput.__init__): Use bytes output interface _stdout for
> StandardOutput.write_binary.
> (StandardOutput.wirte):
> - Override this method only if on Python 3 and encoding of stdout is not
> 'utf-8'
> - Use sys.stdout.encoding instead of locale.getpreferredencoding().
> (PropChange.generate): Use bytes input interface _stdin.
> (Lock.__init__): Use bytes input interface _stdin.
> (DiffURLSelections._get_url): Remove extra trailing spaces.
> (DiffGenerator.__getitem__): Prepare bytes and str representation of
> base_path, 'base_path_bytes' and 'base_path', then use them properly.
>
> Modified:
> subversion/trunk/tools/hook-scripts/mailer/mailer.py
>
> Modified: subversion/trunk/tools/hook-scripts/mailer/mailer.py
> URL:
> http://svn.apache.org/viewvc/subversion/trunk/tools/hook-scripts/mailer/mailer.py?rev=1885656&r1=1885655&r2=1885656&view=diff
> ==============================================================================
> --- subversion/trunk/tools/hook-scripts/mailer/mailer.py (original)
> +++ subversion/trunk/tools/hook-scripts/mailer/mailer.py Mon Jan 18 16:30:27
> 2021
> @@ -46,15 +46,21 @@
>
> import os
> import sys
> -import configparser
> -from urllib.parse import quote as _url_quote
> +if sys.hexversion >= 0x3000000:
> + PY3 = True
> + import configparser
> + from urllib.parse import quote as _url_quote
> +else:
> + PY3 = False
> + import ConfigParser as configparser
> + from urllib import quote as _url_quote
> import time
> import subprocess
> from io import BytesIO
> import smtplib
> import re
> import tempfile
> -import locale
> +import codecs
>
> # Minimal version of Subversion's bindings required
> _MIN_SVN_VERSION = [1, 5, 0]
> @@ -73,6 +79,28 @@ if _MIN_SVN_VERSION > [svn.core.SVN_VER_
> % ".".join([str(x) for x in _MIN_SVN_VERSION]))
> sys.exit(1)
>
> +# Absorb difference between Python 2 and Python >= 3
> +if PY3:
> + def to_bytes(x):
> + return x.encode('utf-8')
> +
> + def to_str(x):
> + return x.decode('utf-8')
> +
> + # We never use sys.stdin nor sys.stdout TextIOwrapper.
> + _stdin = sys.stdin.buffer
> + _stdout = sys.stdout.buffer
> +else:
> + # Python 2
> + def to_bytes(x):
> + return x
> +
> + def to_str(x):
> + return x
> +
> + _stdin = sys.stdin
> + _stdout = sys.stdout
> +
>
> SEPARATOR = '=' * 78
>
> @@ -151,8 +179,16 @@ class OutputBase:
> except ValueError:
> truncate_subject = 0
>
> - if truncate_subject and len(subject) > truncate_subject:
> - subject = subject[:(truncate_subject - 3)] + "..."
> + # truncate subject as UTF-8 string.
> + # Note: there still exists an issue on combining characters.
> + if truncate_subject:
> + bsubject = to_bytes(subject)
> + if len(bsubject) > truncate_subject:
> + idx = truncate_subject - 2
> + while b'\x80' <= bsubject[idx-1:idx] <= b'\xbf':
> + idx -= 1
> + subject = to_str(bsubject[:idx-1]) + "..."
> +
> return subject
>
> def start(self, group, params):
> @@ -177,7 +213,7 @@ class OutputBase:
>
> def write(self, output):
> """Append the literal text string OUTPUT to the output representation."""
> - return self.write_binary(output.encode('utf-8'))
> + return self.write_binary(to_bytes(output))
>
> def run(self, cmd):
> """Override this method, if the default implementation is not sufficient.
> @@ -356,7 +392,7 @@ class StandardOutput(OutputBase):
>
> def __init__(self, cfg, repos, prefix_param):
> OutputBase.__init__(self, cfg, repos, prefix_param)
> - self.write_binary = sys.stdout.buffer.write
> + self.write_binary = _stdout.write
>
> def start(self, group, params):
> self.write("Group: " + (group or "defaults") + "\n")
> @@ -365,10 +401,11 @@ class StandardOutput(OutputBase):
> def finish(self):
> pass
>
> - def write(self, output):
> - """Write text as *default* encoding string"""
> - return self.write_binary(output.encode(locale.getpreferredencoding(),
> - 'backslashreplace'))
> + if (PY3 and (codecs.lookup(sys.stdout.encoding) !=
> codecs.lookup('utf-8'))):
> + def write(self, output):
> + """Write text as *default* encoding string"""
> + return self.write_binary(output.encode(sys.stdout.encoding,
> + 'backslashreplace'))
>
>
> class PipeOutput(MailedOutput):
> @@ -431,8 +468,7 @@ class Commit(Messenger):
>
> self.changelist = sorted(editor.get_changes().items())
>
> - log = (repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG)
> - or b'').decode('utf-8')
> + log = to_str(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG) or b'')
>
> # collect the set of groups and the unique sets of params for the options
> self.groups = { }
> @@ -451,7 +487,7 @@ class Commit(Messenger):
> # figure out the changed directories
> dirs = { }
> for path, change in self.changelist:
> - path = path.decode('utf-8')
> + path = to_str(path)
> if change.item_kind == svn.core.svn_node_dir:
> dirs[path] = None
> else:
> @@ -542,7 +578,7 @@ class PropChange(Messenger):
> elif self.action == 'M':
> self.output.write('Property diff:\n')
> tempfile1 = tempfile.NamedTemporaryFile()
> - tempfile1.write(sys.stdin.buffer.read())
> + tempfile1.write(_stdin.read())
> tempfile1.flush()
> tempfile2 = tempfile.NamedTemporaryFile()
> tempfile2.write(self.repos.get_rev_prop(self.propname))
> @@ -604,8 +640,7 @@ class Lock(Messenger):
> or 'unlock_subject_prefix'))
>
> # read all the locked paths from STDIN and strip off the trailing
> newlines
> - self.dirlist = [x.decode('utf-8').rstrip()
> - for x in sys.stdin.buffer.readlines()]
> + self.dirlist = [to_str(x).rstrip() for x in _stdin.readlines()]
>
> # collect the set of groups and the unique sets of params for the options
> self.groups = { }
> @@ -634,7 +669,7 @@ class Lock(Messenger):
> # The lock comment is the same for all paths, so we can just pull
> # the comment for the first path in the dirlist and cache it.
> self.lock = svn.fs.svn_fs_get_lock(self.repos.fs_ptr,
> - self.dirlist[0].encode('utf-8'),
> + to_bytes(self.dirlist[0]),
> self.pool)
>
> def generate(self):
> @@ -709,7 +744,7 @@ class DiffURLSelections:
> # KeyError exceptions.
> params = self.params.copy()
> params['path'] = _url_quote(change.path) if change.path else None
> - params['base_path'] = (_url_quote(change.base_path)
> + params['base_path'] = (_url_quote(change.base_path)
> if change.base_path else None)
> params['rev'] = repos_rev
> params['base_rev'] = change.base_rev
> @@ -764,8 +799,7 @@ def generate_content(renderer, cfg, repo
> author=repos.author,
> date=date,
> rev=repos.rev,
> - log=(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG)
> - or b'').decode('utf-8'),
> + log=to_str(repos.get_rev_prop(svn.core.SVN_PROP_REVISION_LOG) or b''),
> commit_url=commit_url,
> added_data=generate_list('A', changelist, paths, True),
> replaced_data=generate_list('R', changelist, paths, True),
> @@ -873,7 +907,9 @@ class DiffGenerator:
>
> # figure out if/how to generate a diff
>
> - base_path = remove_leading_slashes(change.base_path)
> + base_path_bytes = remove_leading_slashes(change.base_path)
> + base_path = (to_str(base_path_bytes)
> + if base_path_bytes is not None else None)
> if change.action == svn.repos.CHANGE_ACTION_DELETE:
> # it was delete.
> kind = 'D'
> @@ -884,10 +920,9 @@ class DiffGenerator:
> # show the diff?
> if self.diffsels.delete:
> diff = svn.fs.FileDiff(self.repos.get_root(change.base_rev),
> - base_path, None, None, self.pool)
> + base_path_bytes, None, None, self.pool)
>
> - label1 = '%s\t%s\t(r%s)' % (base_path.decode('utf-8'), self.date,
> - change.base_rev)
> + label1 = '%s\t%s\t(r%s)' % (base_path, self.date, change.base_rev)
> label2 = '/dev/null\t00:00:00 1970\t(deleted)'
> singular = True
>
> @@ -906,14 +941,13 @@ class DiffGenerator:
> # show the diff?
> if self.diffsels.modify:
> diff = svn.fs.FileDiff(self.repos.get_root(change.base_rev),
> - base_path,
> + base_path_bytes,
> self.repos.root_this, change.path,
> self.pool)
> - label1 = '%s\t%s\t(r%s, copy source)' \
> - % (base_path.decode('utf-8'), base_date,
> change.base_rev)
> - label2 = '%s\t%s\t(r%s)' \
> - % (change.path.decode('utf-8'), self.date, \
> - self.repos.rev)
> + label1 = ('%s\t%s\t(r%s, copy source)'
> + % (base_path, base_date, change.base_rev))
> + label2 = ('%s\t%s\t(r%s)'
> + % (to_str(change.path), self.date, self.repos.rev))
> singular = False
> else:
> # this file was copied.
> @@ -921,12 +955,12 @@ class DiffGenerator:
> if self.diffsels.copy:
> diff = svn.fs.FileDiff(None, None, self.repos.root_this,
> change.path, self.pool)
> - label1 = '/dev/null\t00:00:00 1970\t' \
> - '(empty, because file is newly added)'
> - label2 = '%s\t%s\t(r%s, copy of r%s, %s)' \
> - % (change.path.decode('utf-8'), self.date,
> - self.repos.rev, change.base_rev,
> - base_path.decode('utf-8'))
> + label1 = ('/dev/null\t00:00:00 1970\t'
> + '(empty, because file is newly added)')
> + label2 = ('%s\t%s\t(r%s, copy of r%s, %s)'
> + % (to_str(change.path),
> + self.date, self.repos.rev, change.base_rev,
> + base_path))
> singular = False
> else:
> # the file was added.
> @@ -942,7 +976,7 @@ class DiffGenerator:
> label1 = '/dev/null\t00:00:00 1970\t' \
> '(empty, because file is newly added)'
> label2 = '%s\t%s\t(r%s)' \
> - % (change.path.decode('utf-8'), self.date,
> self.repos.rev)
> + % (to_str(change.path), self.date, self.repos.rev)
> singular = True
>
> elif not change.text_changed:
> @@ -962,9 +996,9 @@ class DiffGenerator:
> self.repos.root_this, change.path,
> self.pool)
> label1 = '%s\t%s\t(r%s)' \
> - % (base_path.decode('utf-8'), base_date, change.base_rev)
> + % (base_path, base_date, change.base_rev)
> label2 = '%s\t%s\t(r%s)' \
> - % (change.path.decode('utf-8'), self.date, self.repos.rev)
> + % (to_str(change.path), self.date, self.repos.rev)
> singular = False
>
> if diff:
> @@ -1154,7 +1188,7 @@ class TextCommitRenderer:
> props = ' (props changed)'
> else:
> props = ''
> - w(' %s%s%s\n' % (d.path.decode('utf-8'), is_dir, props))
> + w(' %s%s%s\n' % (to_str(d.path), is_dir, props))
> if d.copied:
> if is_dir:
> text = ''
> @@ -1163,7 +1197,7 @@ class TextCommitRenderer:
> else:
> text = ' unchanged'
> w(' - copied%s from r%d, %s%s\n'
> - % (text, d.base_rev, d.base_path.decode('utf-8'), is_dir))
> + % (text, d.base_rev, to_str(d.base_path), is_dir))
>
> def _render_diffs(self, diffs, section_header):
> """Render diffs. Write the SECTION_HEADER if there are actually
> @@ -1180,20 +1214,20 @@ class TextCommitRenderer:
> w(section_header)
> section_header_printed = True
> if diff.kind == 'D':
> - w('\nDeleted: %s\n' % diff.base_path.decode('utf-8'))
> + w('\nDeleted: %s\n' % to_str(diff.base_path))
> elif diff.kind == 'A':
> - w('\nAdded: %s\n' % diff.path.decode('utf-8'))
> + w('\nAdded: %s\n' % to_str(diff.path))
> elif diff.kind == 'C':
> w('\nCopied: %s (from r%d, %s)\n'
> - % (diff.path.decode('utf-8'), diff.base_rev,
> - diff.base_path.decode('utf-8')))
> + % (to_str(diff.path), diff.base_rev,
> + to_str(diff.base_path)))
> elif diff.kind == 'W':
> w('\nCopied and modified: %s (from r%d, %s)\n'
> - % (diff.path.decode('utf-8'), diff.base_rev,
> - diff.base_path.decode('utf-8')))
> + % (to_str(diff.path), diff.base_rev,
> + to_str(diff.base_path)))
> else:
> # kind == 'M'
> - w('\nModified: %s\n' % diff.path.decode('utf-8'))
> + w('\nModified: %s\n' % to_str(diff.path))
>
> if diff.diff_url:
> w('URL: %s\n' % diff.diff_url)
> @@ -1232,7 +1266,7 @@ class Repository:
>
> self.author = self.get_rev_prop(svn.core.SVN_PROP_REVISION_AUTHOR)
> if self.author is not None:
> - self.author = self.author.decode('utf-8')
> + self.author = to_str(self.author)
>
> def get_rev_prop(self, propname, rev = None):
> if not rev:
> @@ -1441,9 +1475,9 @@ class Config:
> "Return the path's associated groups."
> groups = []
> for group, pattern, exclude_pattern, repos_params, search_logmsg_re in
> self._group_re:
> - match = pattern.match(path.decode('utf-8'))
> + match = pattern.match(to_str(path))
> if match:
> - if exclude_pattern and exclude_pattern.match(path.decode('utf-8')):
> + if exclude_pattern and exclude_pattern.match(to_str(path)):
> continue
> params = repos_params.copy()
> params.update(match.groupdict())
> @@ -1522,8 +1556,7 @@ if the property was added, modified or d
> usage()
>
> cmd = sys.argv[1]
> - repos_dir = svn.core.svn_path_canonicalize(sys.argv[2].encode('utf-8'))
> - repos_dir = repos_dir.decode('utf-8')
> + repos_dir = to_str(svn.core.svn_path_canonicalize(to_bytes(sys.argv[2])))
> try:
> expected_args = cmd_list[cmd]
> except KeyError:
>
>
>
