------------------------------------------------------------
revno: 6620
committer: Barry Warsaw <[EMAIL PROTECTED]>
branch nick: endtoend
timestamp: Sun 2008-03-30 22:55:22 -0400
message:
Make all the SMTP_LOG_* templates, PEP 292 compatible. Make them consistently
put the Message-ID first in the log message. Dict the MsgSafeDict class as
unnecessary now.
Fix the DEFAULT_MSG_FOOTER.
Back out an earlier unnecessary change for virgin queue pipelines.
Add a bunch of debugging to the mail Runner loops.
modified:
mailman/Defaults.py
mailman/Message.py
mailman/SafeDict.py
mailman/Utils.py
mailman/bin/withlist.py
mailman/pipeline/smtp_direct.py
mailman/queue/__init__.py
mailman/tests/test_safedict.py
=== modified file 'mailman/Defaults.py'
--- a/mailman/Defaults.py 2008-03-30 04:06:07 +0000
+++ b/mailman/Defaults.py 2008-03-31 02:55:22 +0000
@@ -551,19 +551,21 @@
# printing of this log message.
SMTP_LOG_EVERY_MESSAGE = (
'smtp',
- '%(msg_message-id)s smtp to %(listname)s for %(#recips)d recips, completed
in %(time).3f seconds')
+ ('${message-id} smtp to $listname for ${#recips} recips, '
+ 'completed in $time seconds'))
# This will only be printed if there were no immediate smtp failures.
# Mutually exclusive with SMTP_LOG_REFUSED.
SMTP_LOG_SUCCESS = (
'post',
- 'post to %(listname)s from %(sender)s, size=%(size)d,
message-id=%(msg_message-id)s, success')
+ '${message-id} post to $listname from $sender, size=$size, success')
# This will only be printed if there were any addresses which encountered an
# immediate smtp failure. Mutually exclusive with SMTP_LOG_SUCCESS.
SMTP_LOG_REFUSED = (
'post',
- 'post to %(listname)s from %(sender)s, size=%(size)d,
message-id=%(msg_message-id)s, %(#refused)d failures')
+ ('${message-id} post to $listname from $sender, size=$size, '
+ '${#refused} failures'))
# This will be logged for each specific recipient failure. Additional %()s
# keys are:
@@ -573,7 +575,8 @@
# failmsg -- the actual smtp message, if available
SMTP_LOG_EACH_FAILURE = (
'smtp-failure',
- 'delivery to %(recipient)s failed with code %(failcode)d: %(failmsg)s')
+ ('${message-id} delivery to $recipient failed with code $failcode: '
+ '$failmsg'))
# These variables control the format and frequency of VERP-like delivery for
# better bounce detection. VERP is Variable Envelope Return Path, defined
@@ -878,8 +881,8 @@
DEFAULT_MSG_FOOTER = u"""\
_______________________________________________
$real_name mailing list
-$fqdn_realname
-${web_page_url}listinfo${cgiext}/${list_name}
+$fqdn_listname
+${web_page_url}listinfo${cgiext}/${fqdn_listname}
"""
# Scrub regular delivery
=== modified file 'mailman/Message.py'
--- a/mailman/Message.py 2008-03-30 23:38:18 +0000
+++ b/mailman/Message.py 2008-03-31 02:55:22 +0000
@@ -270,7 +270,6 @@
recips=self.recips,
nodecorate=True,
reduced_list_headers=True,
- pipeline='virgin',
)
if mlist is not None:
enqueue_kws['listname'] = mlist.fqdn_listname
@@ -307,5 +306,4 @@
nodecorate=True,
reduced_list_headers=True,
envsender=self._sender,
- pipeline='virgin',
**_kws)
=== modified file 'mailman/SafeDict.py'
--- a/mailman/SafeDict.py 2008-02-27 06:26:18 +0000
+++ b/mailman/SafeDict.py 2008-03-31 02:55:22 +0000
@@ -53,34 +53,3 @@
if isinstance(v, str):
self.__setitem__(k, unicode(v, self.cset))
return template % self
-
-
-
-class MsgSafeDict(SafeDict):
- def __init__(self, msg, d=None):
- self.__msg = msg
- if d is None:
- d = {}
- super(MsgSafeDict, self).__init__(d)
-
- def __getitem__(self, key):
- if key.startswith('msg_'):
- return self.__msg.get(key[4:], 'n/a')
- elif key.startswith('allmsg_'):
- missing = []
- all = self.__msg.get_all(key[7:], missing)
- if all is missing:
- return 'n/a'
- return COMMASPACE.join(all)
- else:
- return super(MsgSafeDict, self).__getitem__(key)
-
- def copy(self):
- d = super(MsgSafeDict, self).copy()
- for k in self.__msg.keys():
- vals = self.__msg.get_all(k)
- if len(vals) == 1:
- d['msg_'+k.lower()] = vals[0]
- else:
- d['allmsg_'+k.lower()] = COMMASPACE.join(vals)
- return d
=== modified file 'mailman/Utils.py'
--- a/mailman/Utils.py 2008-03-27 09:14:14 +0000
+++ b/mailman/Utils.py 2008-03-31 02:55:22 +0000
@@ -858,5 +858,3 @@
matched = pattern
break
return matched
-
-
=== modified file 'mailman/bin/withlist.py'
--- a/mailman/bin/withlist.py 2008-03-26 02:28:57 +0000
+++ b/mailman/bin/withlist.py 2008-03-31 02:55:22 +0000
@@ -210,5 +210,8 @@
"The variable 'm' is the $listname mailing list")
else:
banner = interact.DEFAULT_BANNER
- overrides = dict(m=LAST_MLIST, r=r)
+ overrides = dict(m=LAST_MLIST, r=r,
+ commit=config.db.commit,
+ abort=config.db.abort,
+ config=config)
interact.interact(upframe=False, banner=banner, overrides=overrides)
=== modified file 'mailman/pipeline/smtp_direct.py'
--- a/mailman/pipeline/smtp_direct.py 2008-03-27 09:14:14 +0000
+++ b/mailman/pipeline/smtp_direct.py 2008-03-31 02:55:22 +0000
@@ -34,6 +34,7 @@
import time
import email
import socket
+import string
import logging
import smtplib
@@ -44,7 +45,6 @@
from mailman import Errors
from mailman import Utils
-from mailman.SafeDict import MsgSafeDict
from mailman.configuration import config
from mailman.i18n import _
from mailman.interfaces import IHandler, Personalization
@@ -101,6 +101,20 @@
+class MessageDict(dict):
+ def __init__(self, message, extras):
+ super(MessageDict, self).__init__()
+ for key, value in message.items():
+ self[key.lower()] = value
+ self.update(extras)
+
+
+class Template(string.Template):
+ # Allow dashes and # signs, in addition to the standard pattern.
+ idpattern = '[_a-z#-][_a-z0-9#-]*'
+
+
+
def process(mlist, msg, msgdata):
recips = msgdata.get('recips')
if not recips:
@@ -178,23 +192,27 @@
msgdata['recips'] = origrecips
# Log the successful post
t1 = time.time()
- d = MsgSafeDict(msg, {'time' : t1-t0,
- # BAW: Urg. This seems inefficient.
- 'size' : len(msg.as_string()),
- '#recips' : len(recips),
- '#refused': len(refused),
- 'listname': mlist.fqdn_listname,
- 'sender' : origsender,
- })
+ substitutions = MessageDict(msg, {
+ 'time' : t1-t0,
+ 'size' : msg.original_size,
+ '#recips' : len(recips),
+ '#refused' : len(refused),
+ 'listname' : mlist.fqdn_listname,
+ 'sender' : origsender,
+ })
+ if 'message-id' not in substitutions:
+ substitutions['message-id'] = 'n/a'
# We have to use the copy() method because extended call syntax requires a
# concrete dictionary object; it does not allow a generic mapping (XXX is
# this still true in Python 2.3?).
if config.SMTP_LOG_EVERY_MESSAGE:
- every_log.info('%s', config.SMTP_LOG_EVERY_MESSAGE[1] % d)
+ template = Template(config.SMTP_LOG_EVERY_MESSAGE[1])
+ every_log.info('%s', template.safe_substitute(substitutions))
if refused:
if config.SMTP_LOG_REFUSED:
- refused_log.info('%s', config.SMTP_LOG_REFUSED[1] % d)
+ template = Template(config.SMTP_LOG_REFUSED[1])
+ refused_log.info('%s', template.safe_substitute(substitutions))
elif msgdata.get('tolist'):
# Log the successful post, but only if it really was a post to the
@@ -203,7 +221,8 @@
# the other messages, but in that case, we should probably have a
# separate configuration variable to control that.
if config.SMTP_LOG_SUCCESS:
- success_log.info('%s', config.SMTP_LOG_SUCCESS[1] % d)
+ template = Template(config.SMTP_LOG_SUCCESS[1])
+ success_log.info('%s', template.safe_substitute(substitutions))
# Process any failed deliveries.
tempfailures = []
@@ -226,10 +245,13 @@
# future delivery. TBD: this could generate lots of log entries!
tempfailures.append(recip)
if config.SMTP_LOG_EACH_FAILURE:
- d.update({'recipient': recip,
- 'failcode' : code,
- 'failmsg' : smtpmsg})
- failure_log.info('%s', config.SMTP_LOG_EACH_FAILURE[1] % d)
+ substitutions.update({
+ 'recipient' : recip,
+ 'failcode' : code,
+ 'failmsg' : smtpmsg,
+ })
+ template = Template(config.SMTP_LOG_EACH_FAILURE[1])
+ failure_log.info('%s', template.safe_substitute(substitutions))
# Return the results
if tempfailures or permfailures:
raise Errors.SomeRecipientsFailed(tempfailures, permfailures)
@@ -371,11 +393,11 @@
# Send the message
refused = conn.sendmail(envsender, recips, msgtext)
except smtplib.SMTPRecipientsRefused, e:
- flog.error('All recipients refused: %s, msgid: %s', e, msgid)
+ flog.error('%s recipients refused: %s', msgid, e)
refused = e.recipients
except smtplib.SMTPResponseException, e:
- flog.error('SMTP session failure: %s, %s, msgid: %s',
- e.smtp_code, e.smtp_error, msgid)
+ flog.error('%s SMTP session failure: %s, %s',
+ msgid, e.smtp_code, e.smtp_error)
# If this was a permanent failure, don't add the recipients to the
# refused, because we don't want them to be added to failures.
# Otherwise, if the MTA rejects the message because of the message
@@ -390,7 +412,7 @@
# MTA not responding, or other socket problems, or any other kind of
# SMTPException. In that case, nothing got delivered, so treat this
# as a temporary failure.
- flog.error('Low level smtp error: %s, msgid: %s', e, msgid)
+ flog.error('%s low level smtp error: %s', msgid, e)
error = str(e)
for r in recips:
refused[r] = (-1, error)
=== modified file 'mailman/queue/__init__.py'
--- a/mailman/queue/__init__.py 2008-03-30 23:38:18 +0000
+++ b/mailman/queue/__init__.py 2008-03-31 02:55:22 +0000
@@ -60,7 +60,8 @@
# prevents skipping one of two entries with the same time until the next pass.
DELTA = .0001
-log = logging.getLogger('mailman.error')
+elog = logging.getLogger('mailman.error')
+dlog = logging.getLogger('mailman.debug')
@@ -168,7 +169,8 @@
else:
os.unlink(bakfile)
except EnvironmentError, e:
- log.exception('Failed to unlink/preserve backup file: %s', bakfile)
+ elog.exception(
+ 'Failed to unlink/preserve backup file: %s', bakfile)
@property
def files(self):
@@ -257,12 +259,15 @@
self._cleanup()
def _oneloop(self):
+ me = self.__class__.__name__
+ dlog.debug('[%s] starting oneloop', me)
# First, list all the files in our queue directory.
# Switchboard.files() is guaranteed to hand us the files in FIFO
# order. Return an integer count of the number of files that were
# available for this qrunner to process.
files = self._switchboard.files
for filebase in files:
+ dlog.debug('[%s] processing filebase: %s', me, filebase)
try:
# Ask the switchboard for the message and metadata objects
# associated with this filebase.
@@ -274,13 +279,15 @@
# We don't want the runner to die, so we just log and skip
# this entry, but preserve it for analysis.
self._log(e)
- log.error('Skipping and preserving unparseable message: %s',
- filebase)
+ elog.error('Skipping and preserving unparseable message: %s',
+ filebase)
self._switchboard.finish(filebase, preserve=True)
config.db.abort()
continue
try:
+ dlog.debug('[%s] processing onefile', me)
self._onefile(msg, msgdata)
+ dlog.debug('[%s] finishing filebase: %s', me, filebase)
self._switchboard.finish(filebase)
except Exception, e:
# All runners that implement _dispose() must guarantee that
@@ -297,23 +304,30 @@
# message. Try to be graceful.
try:
new_filebase = self._shunt.enqueue(msg, msgdata)
- log.error('SHUNTING: %s', new_filebase)
+ elog.error('SHUNTING: %s', new_filebase)
self._switchboard.finish(filebase)
except Exception, e:
# The message wasn't successfully shunted. Log the
# exception and try to preserve the original queue entry
# for possible analysis.
self._log(e)
- log.error('SHUNTING FAILED, preserving original entry: %s',
- filebase)
+ elog.error(
+ 'SHUNTING FAILED, preserving original entry: %s',
+ filebase)
self._switchboard.finish(filebase, preserve=True)
- config.db.abort()
+ config.db.abort()
# Other work we want to do each time through the loop.
+ dlog.debug('[%s] reaping', me)
Utils.reap(self._kids, once=True)
+ dlog.debug('[%s] doing periodic', me)
self._doperiodic()
+ dlog.debug('[%s] checking short circuit', me)
if self._shortcircuit():
+ dlog.debug('[%s] short circuiting', me)
break
+ dlog.debug('[%s] commiting', me)
config.db.commit()
+ dlog.debug('[%s] ending oneloop: %s', me, len(files))
return len(files)
def _onefile(self, msg, msgdata):
@@ -327,8 +341,8 @@
listname = msgdata.get('listname')
mlist = config.db.list_manager.get(listname)
if not mlist:
- log.error('Dequeuing message destined for missing list: %s',
- listname)
+ elog.error('Dequeuing message destined for missing list: %s',
+ listname)
self._shunt.enqueue(msg, msgdata)
return
# Now process this message, keeping track of any subprocesses that may
@@ -357,10 +371,10 @@
self._switchboard.enqueue(msg, msgdata)
def _log(self, exc):
- log.error('Uncaught runner exception: %s', exc)
+ elog.error('Uncaught runner exception: %s', exc)
s = StringIO()
traceback.print_exc(file=s)
- log.error('%s', s.getvalue())
+ elog.error('%s', s.getvalue())
#
# Subclasses can override these methods.
=== modified file 'mailman/tests/test_safedict.py'
--- a/mailman/tests/test_safedict.py 2008-02-27 06:26:18 +0000
+++ b/mailman/tests/test_safedict.py 2008-03-31 02:55:22 +0000
@@ -42,57 +42,7 @@
-class TestMsgSafeDict(unittest.TestCase):
- def setUp(self):
- self._msg = email.message_from_string("""To: foo
-From: bar
-Subject: baz
-Cc: [EMAIL PROTECTED]
-Cc: [EMAIL PROTECTED]
-
-""")
-
- def test_normal_key(self):
- sd = SafeDict.MsgSafeDict(self._msg, {'key': 'value'})
- si = '%(key)s' % sd
- self.assertEqual(si, 'value')
-
- def test_msg_key(self):
- sd = SafeDict.MsgSafeDict(self._msg, {'to': 'value'})
- si = '%(msg_to)s' % sd
- self.assertEqual(si, 'foo')
-
- def test_allmsg_key(self):
- sd = SafeDict.MsgSafeDict(self._msg, {'cc': 'value'})
- si = '%(allmsg_cc)s' % sd
- self.assertEqual(si, '[EMAIL PROTECTED], [EMAIL PROTECTED]')
-
- def test_msg_no_key(self):
- sd = SafeDict.MsgSafeDict(self._msg)
- si = '%(msg_date)s' % sd
- self.assertEqual(si, 'n/a')
-
- def test_allmsg_no_key(self):
- sd = SafeDict.MsgSafeDict(self._msg)
- si = '%(allmsg_date)s' % sd
- self.assertEqual(si, 'n/a')
-
- def test_copy(self):
- sd = SafeDict.MsgSafeDict(self._msg, {'foo': 'bar'})
- copy = sd.copy()
- items = copy.items()
- items.sort()
- self.assertEqual(items, [
- ('allmsg_cc', '[EMAIL PROTECTED], [EMAIL PROTECTED]'),
- ('foo', 'bar'),
- ('msg_from', 'bar'),
- ('msg_subject', 'baz'),
- ('msg_to', 'foo'),
- ])
-
-
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestSafeDict))
- suite.addTest(unittest.makeSuite(TestMsgSafeDict))
return suite
--
Primary development focus
https://code.launchpad.net/~mailman-coders/mailman/3.0
You are receiving this branch notification because you are subscribed to it.
_______________________________________________
Mailman-checkins mailing list
[email protected]
Unsubscribe:
http://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
