------------------------------------------------------------
revno: 6619
committer: Barry Warsaw <[EMAIL PROTECTED]>
branch nick: endtoend
timestamp: Sun 2008-03-30 19:38:18 -0400
message:
More fixes to get end-to-end delivery basically working.
- Add a VirginPipeline for the virgin queue, which is mostly the same as the
BuiltInPipeline, so refactor the commonality into a BasePipeline.
- Simplify and update bin/dumpdb.
- Rename inject() to inject_text() and add inject_message(). Both interfaces
will be useful.
- When enqueuing and not using _plaintext, use the highest pickle protocol
supported.
- Fix the archive runner's calculation of whether to clobber the message's
Date: header.
- Fix the outgoing queue's deliver_after calculation.
- Update the virgin queue runner. It doesn't need to inherit from
IncomingRunner any more; it can just execute the 'virgin' pipeline.
modified:
mailman/Message.py
mailman/app/chains.py
mailman/app/pipelines.py
mailman/bin/dumpdb.py
mailman/bin/inject.py
mailman/inject.py
mailman/queue/__init__.py
mailman/queue/archive.py
mailman/queue/outgoing.py
mailman/queue/virgin.py
=== modified file 'mailman/Message.py'
--- a/mailman/Message.py 2008-02-27 06:26:18 +0000
+++ b/mailman/Message.py 2008-03-30 23:38:18 +0000
@@ -248,11 +248,10 @@
This is used for all internally crafted messages.
"""
# Since we're crafting the message from whole cloth, let's make sure
- # this message has a Message-ID. Yes, the MTA would give us one, but
- # this is useful for logging to logs/smtp.
+ # this message has a Message-ID.
if 'message-id' not in self:
self['Message-ID'] = email.utils.make_msgid()
- # Ditto for Date: which is required by RFC 2822
+ # Ditto for Date: as required by RFC 2822.
if 'date' not in self:
self['Date'] = email.utils.formatdate(localtime=True)
# UserNotifications are typically for admin messages, and for messages
@@ -271,6 +270,7 @@
recips=self.recips,
nodecorate=True,
reduced_list_headers=True,
+ pipeline='virgin',
)
if mlist is not None:
enqueue_kws['listname'] = mlist.fqdn_listname
@@ -307,4 +307,5 @@
nodecorate=True,
reduced_list_headers=True,
envsender=self._sender,
+ pipeline='virgin',
**_kws)
=== modified file 'mailman/app/chains.py'
--- a/mailman/app/chains.py 2008-02-27 06:26:18 +0000
+++ b/mailman/app/chains.py 2008-03-30 23:38:18 +0000
@@ -25,11 +25,11 @@
from mailman.chains.accept import AcceptChain
+from mailman.chains.builtin import BuiltInChain
from mailman.chains.discard import DiscardChain
from mailman.chains.headers import HeaderMatchChain
from mailman.chains.hold import HoldChain
from mailman.chains.reject import RejectChain
-from mailman.chains.builtin import BuiltInChain
from mailman.configuration import config
from mailman.interfaces import LinkAction
=== modified file 'mailman/app/pipelines.py'
--- a/mailman/app/pipelines.py 2008-02-27 06:26:18 +0000
+++ b/mailman/app/pipelines.py 2008-03-30 23:38:18 +0000
@@ -48,11 +48,27 @@
-class BuiltInPipeline:
+class BasePipeline:
+ """Base pipeline implementation."""
+
+ implements(IPipeline)
+
+ _default_handlers = ()
+
+ def __init__(self):
+ self._handlers = []
+ for handler_name in self._default_handlers:
+ self._handlers.append(config.handlers[handler_name])
+
+ def __iter__(self):
+ """See `IPipeline`."""
+ for handler in self._handlers:
+ yield handler
+
+
+class BuiltInPipeline(BasePipeline):
"""The built-in pipeline."""
- implements(IPipeline)
-
name = 'built-in'
description = _('The built-in pipeline.')
@@ -73,15 +89,19 @@
'to-outgoing',
)
- def __init__(self):
- self._handlers = []
- for handler_name in self._default_handlers:
- self._handlers.append(config.handlers[handler_name])
-
- def __iter__(self):
- """See `IPipeline`."""
- for handler in self._handlers:
- yield handler
+
+class VirginPipeline(BasePipeline):
+ """The processing pipeline for virgin messages.
+
+ Virgin messages are those that are crafted internally by Mailman.
+ """
+ name = 'virgin'
+ description = _('The virgin queue pipeline.')
+
+ _default_handlers = (
+ 'cook-headers',
+ 'to-outgoing',
+ )
@@ -97,5 +117,6 @@
(handler.name, handler_finder))
config.handlers[handler.name] = handler
# Set up some pipelines.
- pipeline = BuiltInPipeline()
- config.pipelines[pipeline.name] = pipeline
+ for pipeline_class in (BuiltInPipeline, VirginPipeline):
+ pipeline = pipeline_class()
+ config.pipelines[pipeline.name] = pipeline
=== modified file 'mailman/bin/dumpdb.py'
--- a/mailman/bin/dumpdb.py 2008-02-27 06:26:18 +0000
+++ b/mailman/bin/dumpdb.py 2008-03-30 23:38:18 +0000
@@ -1,5 +1,3 @@
-#! @PYTHON@
-#
# Copyright (C) 1998-2008 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
@@ -17,110 +15,76 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
-import sys
+from __future__ import with_statement
+
import pprint
import cPickle
-import marshal
-import optparse
-from mailman import Version
from mailman.configuration import config
from mailman.i18n import _
+from mailman.interact import interact
+from mailman.options import Options
COMMASPACE = ', '
+m = []
-def parseargs():
- parser = optparse.OptionParser(version=Version.MAILMAN_VERSION,
- usage=_("""\
-%%prog [options] filename
-
-Dump the contents of any Mailman `database' file.
-
-If the filename ends with `.db', then it is assumed that the file contains a
-Python marshal. If the file ends with `.pck' then it is assumed to contain a
-Python pickle. In either case, if you want to override the default assumption
--- or if the file ends in neither suffix -- use the -p or -m flags."""))
- parser.add_option('-m', '--marshal',
- default=False, action='store_true',
- help=_("""\
-Assume the file contains a Python marshal,
-overridding any automatic guessing."""))
- parser.add_option('-p', '--pickle',
- default=False, action='store_true',
- help=_("""\
-Assume the file contains a Python pickle,
-overridding any automatic guessing."""))
- parser.add_option('-n', '--noprint',
- default=False, action='store_true',
- help=_("""\
-Don't attempt to pretty print the object. This is useful if there's
-some problem with the object and you just want to get an unpickled
-representation. Useful with `python -i bin/dumpdb <file>'. In that
-case, the root of the tree will be left in a global called "msg"."""))
- parser.add_option('-C', '--config',
- help=_('Alternative configuration file to use'))
- opts, args = parser.parse_args()
- # Options.
- # None == guess, 0 == pickle, 1 == marshal
- opts.filetype = None
- if opts.pickle:
- opts.filetype = 0
- if opts.marshal:
- opts.filetype = 1
- opts.doprint = not opts.noprint
- if len(args) < 1:
- parser.error(_('No filename given.'))
- elif len(args) > 1:
- pargs = COMMASPACE.join(args)
- parser.error(_('Bad arguments: $pargs'))
- else:
- opts.filename = args[0]
- if opts.filetype is None:
- if opts.filename.endswith('.db'):
- opts.filetype = 1
- elif opts.filename.endswith('.pck'):
- opts.filetype = 0
+class ScriptOptions(Options):
+ usage=_("""\
+%prog [options] filename
+
+Dump the contents of any Mailman queue file. The queue file is a data file
+with multiple Python pickles in it.""")
+
+ def add_options(self):
+ super(ScriptOptions, self).add_options()
+ self.parser.add_option(
+ '-n', '--noprint',
+ dest='doprint', default=True, action='store_false',
+ help=_("""\
+Don't attempt to pretty print the object. This is useful if there is some
+problem with the object and you just want to get an unpickled representation.
+Useful with 'bin/dumpdb -i <file>'. In that case, the list of
+unpickled objects will be left in a variable called 'm'."""))
+ self.parser.add_option(
+ '-i', '--interact',
+ default=False, action='store_true',
+ help=_("""\
+Start an interactive Python session, with a variable called 'm' containing the
+list of unpickled objects."""))
+
+ def sanity_check(self):
+ if len(self.arguments) < 1:
+ self.parser.error(_('No filename given.'))
+ elif len(self.arguments) > 1:
+ self.parser.error(_('Unexpected arguments'))
else:
- parser.error(_('Please specify either -p or -m.'))
- return parser, opts, args
+ self.filename = self.arguments[0]
def main():
- parser, opts, args = parseargs()
- config.load(opts.config)
+ options = ScriptOptions()
+ options.initialize()
- # Handle dbs
pp = pprint.PrettyPrinter(indent=4)
- if opts.filetype == 1:
- load = marshal.load
- typename = 'marshal'
- else:
- load = cPickle.load
- typename = 'pickle'
- fp = open(opts.filename)
- m = []
- try:
- cnt = 1
- if opts.doprint:
- print _('[----- start $typename file -----]')
+ with open(options.filename) as fp:
while True:
try:
- obj = load(fp)
+ m.append(cPickle.load(fp))
except EOFError:
- if opts.doprint:
- print _('[----- end $typename file -----]')
break
- if opts.doprint:
- print _('<----- start object $cnt ----->')
- if isinstance(obj, str):
- print obj
- else:
- pp.pprint(obj)
- cnt += 1
- m.append(obj)
- finally:
- fp.close()
+ if options.options.doprint:
+ print _('[----- start pickle -----]')
+ for i, obj in enumerate(m):
+ count = i + 1
+ print _('<----- start object $count ----->')
+ if isinstance(obj, basestring):
+ print obj
+ else:
+ pp.pprint(obj)
+ print _('[----- end pickle -----]')
+ if options.options.interact:
+ interact()
=== modified file 'mailman/bin/inject.py'
--- a/mailman/bin/inject.py 2008-03-26 02:28:57 +0000
+++ b/mailman/bin/inject.py 2008-03-30 23:38:18 +0000
@@ -20,11 +20,14 @@
import os
import sys
+from email import message_from_string
+
from mailman import Utils
from mailman import Version
+from mailman.Message import Message
from mailman.configuration import config
from mailman.i18n import _
-from mailman.inject import inject
+from mailman.inject import inject_text
from mailman.options import SingleMailingListOptions
@@ -81,7 +84,7 @@
with open(options.filename) as fp:
message_text = fp.read()
- inject(fqdn_listname, message_text, qdir=qdir)
+ inject_text(mlist, message_text, qdir=qdir)
=== modified file 'mailman/inject.py'
--- a/mailman/inject.py 2008-02-27 06:26:18 +0000
+++ b/mailman/inject.py 2008-03-30 23:38:18 +0000
@@ -15,20 +15,64 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
+__metaclass__ = type
+__all__ = [
+ 'inject_message',
+ 'inject_text',
+ ]
+
+
+from email import message_from_string
+from email.utils import formatdate, make_msgid
+
+from mailman.Message import Message
from mailman.configuration import config
from mailman.queue import Switchboard
-def inject(listname, msg, recips=None, qdir=None):
+def inject_message(mlist, msg, recips=None, qdir=None):
+ """Inject a message into a queue.
+
+ :param mlist: The mailing list this message is destined for.
+ :param msg: The Message object to inject.
+ :param recips: Optional set of recipients to put into the message's
+ metadata.
+ :param qdir: Optional queue directory to inject this message into. If not
+ given, the incoming queue is used.
+ """
if qdir is None:
qdir = config.INQUEUE_DIR
+ # Since we're crafting the message from whole cloth, let's make sure this
+ # message has a Message-ID.
+ if 'message-id' not in msg:
+ msg['Message-ID'] = make_msgid()
+ # Ditto for Date: as required by RFC 2822.
+ if 'date' not in msg:
+ msg['Date'] = formatdate(localtime=True)
queue = Switchboard(qdir)
kws = dict(
- listname=listname,
+ listname=mlist.fqdn_listname,
tolist=True,
- _plaintext=True,
+ original_size=getattr(msg, 'original_size', len(msg.as_string())),
)
if recips is not None:
kws['recips'] = recips
queue.enqueue(msg, **kws)
+
+
+
+def inject_text(mlist, text, recips=None, qdir=None):
+ """Inject a message into a queue.
+
+ :param mlist: The mailing list this message is destined for.
+ :param text: The text of the message to inject. This will be parsed into
+ a Message object.
+ :param recips: Optional set of recipients to put into the message's
+ metadata.
+ :param qdir: Optional queue directory to inject this message into. If not
+ given, the incoming queue is used.
+ """
+ message = message_from_string(text, Message)
+ message.original_size = len(text)
+ inject_message(mlist, message, recips, qdir)
=== modified file 'mailman/queue/__init__.py'
--- a/mailman/queue/__init__.py 2008-03-26 02:28:57 +0000
+++ b/mailman/queue/__init__.py 2008-03-30 23:38:18 +0000
@@ -38,6 +38,7 @@
import time
import email
import errno
+import pickle
import cPickle
import logging
import marshal
@@ -95,12 +96,12 @@
listname = data.get('listname', '--nolist--')
# Get some data for the input to the sha hash.
now = time.time()
- if not data.get('_plaintext'):
- protocol = 1
- msgsave = cPickle.dumps(_msg, protocol)
- else:
+ if data.get('_plaintext'):
protocol = 0
msgsave = cPickle.dumps(str(_msg), protocol)
+ else:
+ protocol = pickle.HIGHEST_PROTOCOL
+ msgsave = cPickle.dumps(_msg, protocol)
# listname is unicode but the input to the hash function must be an
# 8-bit string (eventually, a bytes object).
hashfood = msgsave + listname.encode('utf-8') + repr(now)
=== modified file 'mailman/queue/archive.py'
--- a/mailman/queue/archive.py 2008-03-30 04:06:07 +0000
+++ b/mailman/queue/archive.py 2008-03-30 23:38:18 +0000
@@ -28,6 +28,7 @@
import os
import time
+from datetime import datetime
from email.Utils import parsedate_tz, mktime_tz, formatdate
from locknix.lockfile import Lock
@@ -44,37 +45,38 @@
# Support clobber_date, i.e. setting the date in the archive to the
# received date, not the (potentially bogus) Date: header of the
# original message.
- clobber = 0
- originaldate = msg.get('date')
- receivedtime = formatdate(msgdata['received_time'])
- if not originaldate:
- clobber = 1
+ clobber = False
+ original_date = msg.get('date')
+ received_time = formatdate(msgdata['received_time'])
+ if not original_date:
+ clobber = True
elif config.ARCHIVER_CLOBBER_DATE_POLICY == 1:
- clobber = 1
+ clobber = True
elif config.ARCHIVER_CLOBBER_DATE_POLICY == 2:
- # what's the timestamp on the original message?
- tup = parsedate_tz(originaldate)
- now = time.time()
+ # What's the timestamp on the original message?
+ timetup = parsedate_tz(original_date)
+ now = datetime.now()
try:
- if not tup:
- clobber = 1
- elif abs(now - mktime_tz(tup)) > \
- config.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW:
- clobber = 1
+ if not timetup:
+ clobber = True
+ else:
+ utc_timestamp = datetime.fromtimestamp(mktime_tz(timetup))
+ clobber = (abs(now - utc_timestamp) >
+ config.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW)
except (ValueError, OverflowError):
# The likely cause of this is that the year in the Date: field
# is horribly incorrect, e.g. (from SF bug # 571634):
# Date: Tue, 18 Jun 0102 05:12:09 +0500
# Obviously clobber such dates.
- clobber = 1
+ clobber = True
if clobber:
del msg['date']
del msg['x-original-date']
- msg['Date'] = receivedtime
+ msg['Date'] = received_time
if originaldate:
- msg['X-Original-Date'] = originaldate
+ msg['X-Original-Date'] = original_date
# Always put an indication of when we received the message.
- msg['X-List-Received-Date'] = receivedtime
+ msg['X-List-Received-Date'] = received_time
# While a list archiving lock is acquired, archive the message.
with Lock(os.path.join(mlist.data_path, 'archive.lck')):
for archive_factory in get_plugins('mailman.archiver'):
=== modified file 'mailman/queue/outgoing.py'
--- a/mailman/queue/outgoing.py 2008-03-30 04:06:07 +0000
+++ b/mailman/queue/outgoing.py 2008-03-30 23:38:18 +0000
@@ -23,7 +23,8 @@
import email
import socket
import logging
-import datetime
+
+from datetime import datetime
from mailman import Errors
from mailman import Message
@@ -56,8 +57,8 @@
def _dispose(self, mlist, msg, msgdata):
# See if we should retry delivery of this message again.
- deliver_after = msgdata.get('deliver_after', 0)
- if datetime.datetime.now() < deliver_after:
+ deliver_after = msgdata.get('deliver_after', datetime.fromtimestamp(0))
+ if datetime.now() < deliver_after:
return True
# Make sure we have the most up-to-date state
try:
@@ -102,7 +103,7 @@
# occasionally move them back here for another shot at
# delivery.
if e.tempfailures:
- now = datetime.datetime.now()
+ now = datetime.now()
recips = e.tempfailures
last_recip_count = msgdata.get('last_recip_count', 0)
deliver_until = msgdata.get('deliver_until', now)
=== modified file 'mailman/queue/virgin.py'
--- a/mailman/queue/virgin.py 2008-02-27 06:26:18 +0000
+++ b/mailman/queue/virgin.py 2008-03-30 23:38:18 +0000
@@ -23,22 +23,20 @@
recipient.
"""
+from mailman.app.pipelines import process
from mailman.configuration import config
from mailman.queue import Runner
-from mailman.queue.incoming import IncomingRunner
-class VirginRunner(IncomingRunner):
+class VirginRunner(Runner):
QDIR = config.VIRGINQUEUE_DIR
def _dispose(self, mlist, msg, msgdata):
- # We need to fasttrack this message through any handlers that touch
- # it. E.g. especially CookHeaders.
- msgdata['_fasttrack'] = 1
- return IncomingRunner._dispose(self, mlist, msg, msgdata)
-
- def _get_pipeline(self, mlist, msg, msgdata):
- # It's okay to hardcode this, since it'll be the same for all
- # internally crafted messages.
- return ['CookHeaders', 'ToOutgoing']
+ # We need to fast track this message through any pipeline handlers
+ # that touch it, e.g. especially cook-headers.
+ msgdata['_fasttrack'] = True
+ # Use the 'virgin' pipeline.
+ process(mlist, msg, msgdata, 'virgin')
+ # Do not keep this message queued.
+ return False
--
Primary development focus
https://code.launchpad.net/~mailman-coders/mailman/3.0
You are receiving this branch notification because you are subscribed to it.
_______________________________________________
Mailman-checkins mailing list
[email protected]
Unsubscribe:
http://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
