------------------------------------------------------------
revno: 6616
committer: Barry Warsaw <[EMAIL PROTECTED]>
branch nick: endtoend
timestamp: Thu 2008-03-27 05:14:14 -0400
message:
Added a test of the OutgoingRunner, and subsequent changes to make it pass,
including:
- MailingList.full_path -> IMailingList.data_path and implement this as a
property on the MailingList object.
- Fix the 'decorate' handler to work with the new member/user data model,
instead of the old MemberAdaptor interface.
- Fix a few problems with the smtp-direct handler, though this needs more work
and tests.
- Add some debug logging to both the test smtplistener and the SMTPServer
proxy. Fix the proxy's consumption of messages from the thread queue.
- Fix the smtplistener's calculation of the X-Peer header.
added:
mailman/queue/docs/outgoing.txt
modified:
mailman/Utils.py
mailman/app/membership.py
mailman/bin/arch.py
mailman/bin/senddigests.py
mailman/database/mailinglist.py
mailman/interfaces/mailinglist.py
mailman/pipeline/decorate.py
mailman/pipeline/docs/file-recips.txt
mailman/pipeline/file_recipients.py
mailman/pipeline/smtp_direct.py
mailman/pipeline/to_digest.py
mailman/queue/archive.py
mailman/queue/outgoing.py
mailman/tests/helpers.py
mailman/tests/smtplistener.py
=== modified file 'mailman/Utils.py'
--- a/mailman/Utils.py 2008-03-23 21:38:41 +0000
+++ b/mailman/Utils.py 2008-03-27 09:14:14 +0000
@@ -481,7 +481,7 @@
# Calculate the locations to scan
searchdirs = []
if mlist is not None:
- searchdirs.append(mlist.full_path)
+ searchdirs.append(mlist.data_path)
searchdirs.append(os.path.join(TEMPLATE_DIR, mlist.host_name))
searchdirs.append(os.path.join(TEMPLATE_DIR, 'site'))
searchdirs.append(TEMPLATE_DIR)
=== modified file 'mailman/app/membership.py'
--- a/mailman/app/membership.py 2008-03-23 21:38:41 +0000
+++ b/mailman/app/membership.py 2008-03-27 09:14:14 +0000
@@ -45,8 +45,8 @@
ack=None, admin_notif=None, text=''):
"""Add a member right now.
- The member's subscription must be approved by what ever policy the
- list enforces.
+ The member's subscription must be approved by whatever policy the list
+ enforces.
ack is a flag that specifies whether the user should get an
acknowledgement of their being subscribed. Default is to use the
=== modified file 'mailman/bin/arch.py'
--- a/mailman/bin/arch.py 2008-02-27 06:26:18 +0000
+++ b/mailman/bin/arch.py 2008-03-27 09:14:14 +0000
@@ -115,7 +115,7 @@
# really don't know how long it will take.
#
# XXX processUnixMailbox() should refresh the lock.
- lock_path = os.path.join(mlist.full_path, '.archiver.lck')
+ lock_path = os.path.join(mlist.data_path, '.archiver.lck')
with Lock(lock_path, lifetime=int(hours(3))):
# Maybe wipe the old archives
if opts.wipe:
=== modified file 'mailman/bin/senddigests.py'
--- a/mailman/bin/senddigests.py 2008-02-27 06:26:18 +0000
+++ b/mailman/bin/senddigests.py 2008-03-27 09:14:14 +0000
@@ -72,7 +72,7 @@
print >> sys.stderr, \
'List: %s: problem processing %s:\n%s' % \
(listname,
- os.path.join(mlist.full_path(), 'digest.mbox'),
+ os.path.join(mlist.data_path, 'digest.mbox'),
errmsg)
finally:
mlist.Unlock()
=== modified file 'mailman/database/mailinglist.py'
--- a/mailman/database/mailinglist.py 2008-03-26 02:28:57 +0000
+++ b/mailman/database/mailinglist.py 2008-03-27 09:14:14 +0000
@@ -174,11 +174,10 @@
# 2-tuple of the date of the last autoresponse and the number of
# autoresponses sent on that date.
self.hold_and_cmd_autoresponses = {}
- self.full_path = os.path.join(config.LIST_DATA_DIR, fqdn_listname)
self.personalization = Personalization.none
self.real_name = string.capwords(
SPACE.join(listname.split(UNDERSCORE)))
- makedirs(self.full_path)
+ makedirs(self.data_path)
# XXX FIXME
def _restore(self):
@@ -192,20 +191,25 @@
@property
def fqdn_listname(self):
- """See IMailingListIdentity."""
+ """See `IMailingList`."""
return fqdn_listname(self.list_name, self.host_name)
@property
def web_host(self):
- """See IMailingListWeb."""
+ """See `IMailingList`."""
return config.domains[self.host_name]
def script_url(self, target, context=None):
- """See IMailingListWeb."""
+ """See `IMailingList`."""
# XXX Handle the case for when context is not None; those would be
# relative URLs.
return self.web_page_url + target + '/' + self.fqdn_listname
+ @property
+ def data_path(self):
+ """See `IMailingList`."""
+ return os.path.join(config.LIST_DATA_DIR, self.fqdn_listname)
+
# IMailingListAddresses
@property
=== modified file 'mailman/interfaces/mailinglist.py'
--- a/mailman/interfaces/mailinglist.py 2008-03-26 02:28:57 +0000
+++ b/mailman/interfaces/mailinglist.py 2008-03-27 09:14:14 +0000
@@ -260,3 +260,10 @@
Every mailing list has a processing pipeline that messages flow
through once they've been accepted.
""")
+
+ data_path = Attribute(
+ """The file system path to list-specific data.
+
+ An example of list-specific data is the temporary digest mbox file
+ that gets created to accumlate messages for the digest.
+ """)
=== modified file 'mailman/pipeline/decorate.py'
--- a/mailman/pipeline/decorate.py 2008-02-27 06:26:18 +0000
+++ b/mailman/pipeline/decorate.py 2008-03-27 09:14:14 +0000
@@ -48,21 +48,21 @@
if msgdata.get('personalize'):
# Calculate the extra personalization dictionary. Note that the
# length of the recips list better be exactly 1.
- recips = msgdata.get('recips')
- assert isinstance(recips, list) and len(recips) == 1, (
+ recips = msgdata.get('recips', [])
+ assert len(recips) == 1, (
'The number of intended recipients must be exactly 1')
- member = recips[0].lower()
- d['user_address'] = member
- try:
- d['user_delivered_to'] = mlist.getMemberCPAddress(member)
+ recipient = recips[0].lower()
+ user = config.db.user_manager.get_user(recipient)
+ member = mlist.members.get_member(recipient)
+ d['user_address'] = recipient
+ if user is not None and member is not None:
+ d['user_delivered_to'] = member.address.original_address
# BAW: Hmm, should we allow this?
- d['user_password'] = mlist.getMemberPassword(member)
- d['user_language'] = mlist.getMemberLanguage(member)
- username = mlist.getMemberName(member) or None
- d['user_name'] = username or d['user_delivered_to']
- d['user_optionsurl'] = mlist.GetOptionsURL(member)
- except Errors.NotAMemberError:
- pass
+ d['user_password'] = user.password
+ d['user_language'] = member.preferred_language
+ d['user_name'] = (user.real_name if user.real_name
+ else member.address.original_address)
+ d['user_optionsurl'] = member.options_url
# These strings are descriptive for the log file and shouldn't be i18n'd
d.update(msgdata.get('decoration-data', {}))
header = decorate(mlist, mlist.msg_header, d)
=== modified file 'mailman/pipeline/docs/file-recips.txt'
--- a/mailman/pipeline/docs/file-recips.txt 2008-02-27 06:26:18 +0000
+++ b/mailman/pipeline/docs/file-recips.txt 2008-03-27 09:14:14 +0000
@@ -40,7 +40,7 @@
empty.
>>> import os
- >>> file_path = os.path.join(mlist.full_path, 'members.txt')
+ >>> file_path = os.path.join(mlist.data_path, 'members.txt')
>>> open(file_path)
Traceback (most recent call last):
...
=== modified file 'mailman/pipeline/file_recipients.py'
--- a/mailman/pipeline/file_recipients.py 2008-02-27 06:26:18 +0000
+++ b/mailman/pipeline/file_recipients.py 2008-03-27 09:14:14 +0000
@@ -46,7 +46,7 @@
"""See `IHandler`."""
if 'recips' in msgdata:
return
- filename = os.path.join(mlist.full_path, 'members.txt')
+ filename = os.path.join(mlist.data_path, 'members.txt')
try:
with open(filename) as fp:
addrs = set(line.strip() for line in fp)
=== modified file 'mailman/pipeline/smtp_direct.py'
--- a/mailman/pipeline/smtp_direct.py 2008-02-27 06:26:18 +0000
+++ b/mailman/pipeline/smtp_direct.py 2008-03-27 09:14:14 +0000
@@ -110,7 +110,7 @@
envsender = msgdata.get('envsender')
if envsender is None:
if mlist:
- envsender = mlist.GetBouncesEmail()
+ envsender = mlist.bounces_address
else:
envsender = Utils.get_site_noreply()
# Time to split up the recipient list. If we're personalizing or VERPing
@@ -183,7 +183,7 @@
'size' : len(msg.as_string()),
'#recips' : len(recips),
'#refused': len(refused),
- 'listname': mlist.internal_name(),
+ 'listname': mlist.fqdn_listname,
'sender' : origsender,
})
# We have to use the copy() method because extended call syntax requires a
=== modified file 'mailman/pipeline/to_digest.py'
--- a/mailman/pipeline/to_digest.py 2008-02-27 06:26:18 +0000
+++ b/mailman/pipeline/to_digest.py 2008-03-27 09:14:14 +0000
@@ -74,7 +74,7 @@
# Short circuit non-digestable lists.
if not mlist.digestable or msgdata.get('isdigest'):
return
- mboxfile = os.path.join(mlist.full_path, 'digest.mbox')
+ mboxfile = os.path.join(mlist.data_path, 'digest.mbox')
mboxfp = open(mboxfile, 'a+')
mbox = Mailbox(mboxfp)
mbox.AppendMessage(msg)
=== modified file 'mailman/queue/archive.py'
--- a/mailman/queue/archive.py 2008-02-27 06:26:18 +0000
+++ b/mailman/queue/archive.py 2008-03-27 09:14:14 +0000
@@ -19,6 +19,7 @@
from __future__ import with_statement
+import os
import time
from email.Utils import parsedate_tz, mktime_tz, formatdate
@@ -68,5 +69,5 @@
# Always put an indication of when we received the message.
msg['X-List-Received-Date'] = receivedtime
# While a list archiving lock is acquired, archive the message.
- with Lock(os.path.join(mlist.full_path, 'archive.lck')):
+ with Lock(os.path.join(mlist.data_path, 'archive.lck')):
mlist.ArchiveMail(msg)
=== added file 'mailman/queue/docs/outgoing.txt'
--- a/mailman/queue/docs/outgoing.txt 1970-01-01 00:00:00 +0000
+++ b/mailman/queue/docs/outgoing.txt 2008-03-27 09:14:14 +0000
@@ -0,0 +1,96 @@
+Outgoing queue runner
+=====================
+
+The outgoing queue runner is the process that delivers messages to the
+directly upstream SMTP server. It is this external SMTP server that performs
+final delivery to the intended recipients.
+
+Messages that appear in the outgoing queue are processed individually through
+a 'delivery module', essentially a pluggable interface for determining how the
+recipient set will be batched, whether messages will be personalized and
+VERP'd, etc. The outgoing runner doesn't itself support retrying but it can
+move messages to the 'retry queue' for handling delivery failures.
+
+ >>> from mailman.app.lifecycle import create_list
+ >>> mlist = create_list(u'[EMAIL PROTECTED]')
+
+ >>> from mailman.app.membership import add_member
+ >>> from mailman.interfaces import DeliveryMode
+ >>> add_member(mlist, u'[EMAIL PROTECTED]', u'Anne Person',
+ ... u'password', DeliveryMode.regular, u'en',
+ ... ack=False, admin_notif=False)
+ >>> add_member(mlist, u'[EMAIL PROTECTED]', u'Bart Person',
+ ... u'password', DeliveryMode.regular, u'en',
+ ... ack=False, admin_notif=False)
+ >>> add_member(mlist, u'[EMAIL PROTECTED]', u'Cris Person',
+ ... u'password', DeliveryMode.regular, u'en',
+ ... ack=False, admin_notif=False)
+
+ >>> from mailman.tests.helpers import SMTPServer
+ >>> smtpd = SMTPServer()
+ >>> smtpd.start()
+ >>> from mailman.configuration import config
+ >>> old_host = config.SMTPHOST
+ >>> old_port = config.SMTPPORT
+ >>> config.SMTPHOST = smtpd.host
+ >>> config.SMTPPORT = smtpd.port
+
+By setting the mailing list to personalize messages, each recipient will get a
+unique copy of the message, with certain headers tailored for that recipient.
+
+ >>> from mailman.interfaces import Personalization
+ >>> mlist.personalize = Personalization.individual
+ >>> config.db.commit()
+
+ >>> msg = message_from_string("""\
+ ... From: [EMAIL PROTECTED]
+ ... To: [EMAIL PROTECTED]
+ ... Subject: My first post
+ ... Message-ID: <first>
+ ...
+ ... First post!
+ ... """)
+
+Normally, messages would show up in the outgoing queue after the message has
+been processed by the rule set and pipeline. But we can simulate that here by
+injecting a message directly into the outgoing queue.
+
+ >>> msgdata = {}
+ >>> handler = config.handlers['calculate-recipients']
+ >>> handler.process(mlist, msg, msgdata)
+
+ >>> from mailman.queue import Switchboard
+ >>> outgoing_queue = Switchboard(config.OUTQUEUE_DIR)
+ >>> ignore = outgoing_queue.enqueue(
+ ... msg, msgdata,
+ ... verp=True, listname=mlist.fqdn_listname, tolist=True,
+ ... _plaintext=True)
+
+Running the outgoing queue runner processes the message, delivering it to the
+upstream SMTP, which happens to be our test server.
+
+ >>> from mailman.queue.outgoing import OutgoingRunner
+ >>> from mailman.tests.helpers import make_testable_runner
+ >>> outgoing = make_testable_runner(OutgoingRunner)
+ >>> outgoing.run()
+
+Three messages have been delivered to our SMTP server, one for each recipient.
+
+ >>> from operator import itemgetter
+ >>> messages = sorted(smtpd.messages, key=itemgetter('sender'))
+ >>> len(messages)
+ 3
+
+ >>> for message in messages:
+ ... print message['sender']
+ [EMAIL PROTECTED]
+ [EMAIL PROTECTED]
+ [EMAIL PROTECTED]
+
+
+Clean up
+--------
+
+ >>> smtpd.stop()
+ >>> config.SMTPHOST = old_host
+ >>> config.SMTPPORT = old_port
=== modified file 'mailman/queue/outgoing.py'
--- a/mailman/queue/outgoing.py 2008-02-27 06:26:18 +0000
+++ b/mailman/queue/outgoing.py 2008-03-27 09:14:14 +0000
@@ -60,7 +60,6 @@
if time.time() < deliver_after:
return True
# Make sure we have the most up-to-date state
- mlist.Load()
try:
pid = os.getpid()
self._func(mlist, msg, msgdata)
=== modified file 'mailman/tests/helpers.py'
--- a/mailman/tests/helpers.py 2008-03-21 16:43:45 +0000
+++ b/mailman/tests/helpers.py 2008-03-27 09:14:14 +0000
@@ -34,6 +34,7 @@
import errno
import signal
import socket
+import logging
import mailbox
import smtplib
import tempfile
@@ -48,6 +49,9 @@
from mailman.tests.smtplistener import Server
+log = logging.getLogger('mailman.debug')
+
+
def make_testable_runner(runner_class):
"""Create a queue runner that runs until its queue is empty.
@@ -97,7 +101,7 @@
:param mlist: The mailing list.
:return: The mailing list's pending digest as a mailbox.
"""
- path = os.path.join(mlist.full_path, 'digest.mbox')
+ path = os.path.join(mlist.data_path, 'digest.mbox')
return mailbox.mbox(path)
@@ -168,6 +172,7 @@
def start(self):
"""Start the smtp server in a thread."""
+ log.info('test SMTP server starting')
self._thread.start()
def stop(self):
@@ -178,18 +183,22 @@
self.clear()
# Wait for the thread to exit.
self._thread.join()
+ log.info('test SMTP server stopped')
@property
def messages(self):
"""Return all the messages received by the smtp server."""
- for message in self._messages:
- # See if there's anything waiting in the queue.
+ # Look at the thread queue and append any messages from there to our
+ # internal list of messages.
+ while True:
try:
message = self._queue.get_nowait()
except Empty:
- pass
+ break
else:
self._messages.append(message)
+ # Now return all the messages we know about.
+ for message in self._messages:
yield message
def clear(self):
=== modified file 'mailman/tests/smtplistener.py'
--- a/mailman/tests/smtplistener.py 2008-03-06 05:29:11 +0000
+++ b/mailman/tests/smtplistener.py 2008-03-27 09:14:14 +0000
@@ -18,11 +18,14 @@
"""A test SMTP listener."""
import smtpd
+import logging
import asyncore
from email import message_from_string
+
COMMASPACE = ', '
+log = logging.getLogger('mailman.debug')
@@ -60,14 +63,16 @@
def handle_accept(self):
"""Handle connections by creating our own Channel object."""
conn, addr = self.accept()
+ log.info('accepted: %s', addr)
Channel(self, conn, addr)
def process_message(self, peer, mailfrom, rcpttos, data):
"""Process a message by adding it to the mailbox."""
message = message_from_string(data)
- message['X-Peer'] = peer
+ message['X-Peer'] = '%s:%s' % peer
message['X-MailFrom'] = mailfrom
message['X-RcptTo'] = COMMASPACE.join(rcpttos)
+ log.info('processed message: %s', message.get('message-id', 'n/a'))
self._queue.put(message)
def start(self):
--
Primary development focus
https://code.launchpad.net/~mailman-coders/mailman/3.0
You are receiving this branch notification because you are subscribed to it.
_______________________________________________
Mailman-checkins mailing list
[email protected]
Unsubscribe:
http://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
