Mark Sapiro pushed to branch master at GNU Mailman / Mailman Core
Commits: 94fbd7dd by Mark Sapiro at 2020-02-06T11:36:54-08:00 Fixed html_to_plaintext to properly encode the new payload. Added test for same and updated existing tests for changes. - - - - - 3af7ed0b by Mark Sapiro at 2020-02-09T17:08:14+00:00 Merge branch 'fix_677' into 'master' Fixed html_to_plaintext to properly encode the new payload. Closes #677 See merge request mailman/mailman!600 - - - - - 4 changed files: - src/mailman/docs/NEWS.rst - src/mailman/handlers/docs/filtering.rst - src/mailman/handlers/mime_delete.py - src/mailman/handlers/tests/test_mimedel.py Changes: ===================================== src/mailman/docs/NEWS.rst ===================================== @@ -11,7 +11,7 @@ Here is a history of user visible changes to Mailman. 3.3.1 ===== -(2019-XX-XX) +(2020-XX-XX) Bugs ---- @@ -41,6 +41,8 @@ Bugs (Closes #665) * Residial bounces are now marked as processed. (Closes #669) * Find members API now searches for the Member's display name too. (Closes #667) +* Content filtering ``convert_html_to_plaintext`` no longer creates messages + that can't be flattened as_bytes. (Closes #677) REST ---- ===================================== src/mailman/handlers/docs/filtering.rst ===================================== @@ -234,8 +234,9 @@ name of the file containing the message payload to filter. ... process(mlist, msg, {}) >>> print(msg.as_string()) From: aper...@example.com + Content-Transfer-Encoding: 7bit MIME-Version: 1.0 - Content-Type: text/plain + Content-Type: text/plain; charset="us-ascii" X-Content-Filtered-By: Mailman/MimeDel ... <BLANKLINE> Converted text/html to text/plain @@ -301,8 +302,9 @@ so the entire inner ``multipart/mixed`` is discarded. X-Content-Filtered-By: Mailman/MimeDel ... <BLANKLINE> --AAA + Content-Transfer-Encoding: 7bit MIME-Version: 1.0 - Content-Type: text/plain + Content-Type: text/plain; charset="us-ascii" <BLANKLINE> Converted text/html to text/plain Filename: ... ===================================== src/mailman/handlers/mime_delete.py ===================================== @@ -274,7 +274,7 @@ def to_plaintext(msg): resources.callback(shutil.rmtree, tempdir) for subpart in typed_subpart_iterator(msg, 'text', 'html'): filename = os.path.join(tempdir, '{}.html'.format(next(counter))) - ctype = msg.get_content_charset('utf-8') + ctype = msg.get_content_charset('us-ascii') with open(filename, 'w', encoding='utf-8') as fp: fp.write(subpart.get_payload(decode=True).decode(ctype, errors='replace')) @@ -288,7 +288,7 @@ def to_plaintext(msg): # Replace the payload of the subpart with the converted text # and tweak the content type. del subpart['content-transfer-encoding'] - subpart.set_payload(stdout) + subpart.set_payload(stdout, charset=ctype) subpart.set_type('text/plain') changedp += 1 return changedp ===================================== src/mailman/handlers/tests/test_mimedel.py ===================================== @@ -45,6 +45,9 @@ from zope.component import getUtility @contextmanager def dummy_script(arg=''): exe = sys.executable + non_ascii = '' + if arg == 'non-ascii': + non_ascii = '‘...’' extra = '' if arg == 'scripterr': extra = 'error' @@ -59,10 +62,10 @@ def dummy_script(arg=''): import sys if len(sys.argv) > 2: sys.exit(1) -print('Converted text/html to text/plain') +print('Converted text/html to text/plain{}') print('Filename:', sys.argv[1]) print(open(sys.argv[1]).readlines()[0]) -""", file=fp) +""".format(non_ascii), file=fp) config.push('dummy script', """\ [mailman] html_to_plain_text_command = {exe} {script} {extra} $filename @@ -261,6 +264,29 @@ PGh0bWw+PGhlYWQ+PC9oZWFkPgo8Ym9keT48L2JvZHk+PC9odG1sPgo= self.assertEqual(payload_lines[0], 'Converted text/html to text/plain') self.assertEqual(payload_lines[2], '<html><head></head>') + def test_convert_html_to_plaintext_encodes_new_payload(self): + # Test that the converted payload with non-ascii is encoded. + msg = mfs("""\ +From: aper...@example.com +Content-Type: text/html; charset=utf-8 +Content-Transfer-Encoding: base64 +MIME-Version: 1.0 + +Q29udmVydGVkIHRleHQvaHRtbCB0byB0ZXh0L3BsYWlu4oCYLi4u4oCZCg== +""") + process = config.handlers['mime-delete'].process + with dummy_script('non-ascii'): + process(self._mlist, msg, {}) + self.assertEqual(msg['content-type'], 'text/plain; charset="utf-8"') + self.assertEqual(msg['content-transfer-encoding'], 'base64') + self.assertTrue( + msg['x-content-filtered-by'].startswith('Mailman/MimeDel')) + payload_lines = ( + msg.get_payload(decode=True).decode('utf-8').splitlines()) + self.assertEqual(payload_lines[0], + 'Converted text/html to text/plain‘...’') + self.assertTrue(payload_lines[1].startswith('Filename')) + def test_convert_html_to_plaintext_error_return(self): # Calling a script which returns an error status is properly logged. msg = mfs("""\ View it on GitLab: https://gitlab.com/mailman/mailman/-/compare/fbec46157004dfbc3105670accff117d74614335...3af7ed0b5b98efb3a4203b8d7a81bea5f20bda46 -- View it on GitLab: https://gitlab.com/mailman/mailman/-/compare/fbec46157004dfbc3105670accff117d74614335...3af7ed0b5b98efb3a4203b8d7a81bea5f20bda46 You're receiving this email because of your account on gitlab.com.
_______________________________________________ Mailman-checkins mailing list Mailman-checkins@python.org Unsubscribe: https://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org