Mark Sapiro pushed to branch master at GNU Mailman / Mailman Core
Commits:
3885139c by Mark Sapiro at 2020-09-26T21:43:04-07:00
Truncate long SAUnicode values to 255 chars for MySQL.
- - - - -
34104ae0 by Mark Sapiro at 2020-09-27T05:25:27+00:00
Merge branch 'import' into 'master'
Truncate long SAUnicode values to 255 chars for MySQL.
Closes #772
See merge request mailman/mailman!708
- - - - -
3 changed files:
- src/mailman/docs/NEWS.rst
- src/mailman/utilities/importer.py
- src/mailman/utilities/tests/test_import.py
Changes:
=====================================
src/mailman/docs/NEWS.rst
=====================================
@@ -56,6 +56,8 @@ Command line
show sections defined only in mailman.cfg in addition to those from
schema.cfg. (Closes #736)
* Added a ``charset`` option to the ``import21`` subcommand. (Closes #769)
+* The ``import21`` subcommand will now truncate long SAUnicode values if the
+ database is MySQL. (Closes #772)
REST
----
=====================================
src/mailman/utilities/importer.py
=====================================
@@ -25,6 +25,9 @@ import logging
import datetime
from contextlib import ExitStack
+from mailman.config import config
+from mailman.database.helpers import is_mysql
+from mailman.database.types import SAUnicode
from mailman.handlers.decorate import decorate_template
from mailman.interfaces.action import Action, FilterAction
from mailman.interfaces.address import IEmailValidator
@@ -192,6 +195,19 @@ enabled: yes
return code
+def maybe_truncate_mysql(value):
+ # For MySQL, column type SAUnicode is VARCHAR(255). In many MySQL
+ # configurations, attempts to store longer values are fatal.
+ if is_mysql(config.db.engine) and len(value) > 255:
+ # This actually is covered but only if db is MySQL.
+ print( # pragma: nocover
+ """Length of value for {} is {} which is too long for MySQL.
+Truncated from {}
+to {}""".format(key, len(value), value, value[:255]), file=sys.stderr)
+ return value[:255] # pragma: nocover
+ return value
+
+
# Attributes in Mailman 2 which have a different type in Mailman 3. Some
# types (e.g. bools) are autodetected from their SA column types.
TYPES = dict(
@@ -265,6 +281,7 @@ def import_config_pck(mlist, config_dict):
:param config_dict: The Mailman 2.1 configuration dictionary.
:type config_dict: dict
"""
+ global key
for key, value in config_dict.items():
# Some attributes must not be directly imported.
if key in EXCLUDES:
@@ -290,6 +307,8 @@ def import_config_pck(mlist, config_dict):
column = getattr(mlist.__class__, key, None)
if column is not None and isinstance(column.type, Boolean):
converter = bool
+ if column is not None and isinstance(column.type, SAUnicode):
+ converter = maybe_truncate_mysql
try:
if converter is not None:
value = converter(value)
=====================================
src/mailman/utilities/tests/test_import.py
=====================================
@@ -26,6 +26,7 @@ from enum import Enum
from importlib_resources import open_binary
from mailman.app.lifecycle import create_list
from mailman.config import config
+from mailman.database.helpers import is_mysql
from mailman.handlers.decorate import decorate
from mailman.interfaces.action import Action, FilterAction
from mailman.interfaces.address import InvalidEmailAddressError
@@ -606,6 +607,22 @@ class TestBasicImport(unittest.TestCase):
self.assertIn('Skipping duplicate header_filter rule',
error_log.readline())
+ def test_long_saunicode(self):
+ # Long SAUnicode fields should truncate for MySql with warning only.
+ long_desc = (
+ 'A very long description exceeding 255 ckaracters to test '
+ 'truncation of SAUnicode field data for MySQL. just add some '
+ 'dots .......................................................'
+ '............................................................'
+ '... and a bit more Thats 255 ending at more')
+ self._pckdict['description'] = long_desc
+ self._import()
+ if is_mysql(config.db.engine):
+ self.assertEqual(long_desc[:255], self._mlist.description)
+ self.assertTrue(self._mlist.description.endswith('a bit more'))
+ else:
+ self.assertEqual(long_desc, self._mlist.description)
+
class TestArchiveImport(unittest.TestCase):
"""Test conversion of the archive policies.
View it on GitLab:
https://gitlab.com/mailman/mailman/-/compare/29fd56936d4deb484c9e926a9b32ebc5d18c79f8...34104ae0960d0234d438cc64e3c150202c25e090
--
View it on GitLab:
https://gitlab.com/mailman/mailman/-/compare/29fd56936d4deb484c9e926a9b32ebc5d18c79f8...34104ae0960d0234d438cc64e3c150202c25e090
You're receiving this email because of your account on gitlab.com.
_______________________________________________
Mailman-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/mailman-checkins.python.org/
Member address: [email protected]
