------------------------------------------------------------
revno: 6618
committer: Barry Warsaw <[EMAIL PROTECTED]>
branch nick: endtoend
timestamp: Sun 2008-03-30 00:06:07 -0400
message:
More fixes to get the basic end-to-end delivery mechanisms working.
- Lots of Pipermail work just to get it to play nice with the new apis.
- Fix ARCHIVE_SCRUBBER
- Lots of changes to mailman.app.archiving, especially to provide an
adapter from the new MailingList api to the one expected by
Pipermail (e.g. archive_dir()).
- Add a test of the basic pipermail archiver.
added:
mailman/queue/docs/archiver.txt
modified:
mailman/Archiver/Archiver.py
mailman/Archiver/HyperArch.py
mailman/Archiver/pipermail.py
mailman/Defaults.py
mailman/app/archiving.py
mailman/bin/master.py
mailman/pipeline/cook_headers.py
mailman/pipeline/scrubber.py
mailman/pipeline/to_digest.py
mailman/queue/archive.py
mailman/queue/outgoing.py
mailman/rules/administrivia.py
setup.py
=== modified file 'mailman/Archiver/Archiver.py'
--- a/mailman/Archiver/Archiver.py 2008-02-08 04:01:48 +0000
+++ b/mailman/Archiver/Archiver.py 2008-03-30 04:06:07 +0000
@@ -31,12 +31,11 @@
from cStringIO import StringIO
from string import Template
-from Mailman import Mailbox
-from Mailman import Utils
-from Mailman.SafeDict import SafeDict
-from Mailman.configuration import config
-from Mailman.configuration import config
-from Mailman.i18n import _
+from mailman import Mailbox
+from mailman import Utils
+from mailman.SafeDict import SafeDict
+from mailman.configuration import config
+from mailman.i18n import _
log = logging.getLogger('mailman.error')
@@ -82,10 +81,11 @@
# the private directory, pointing directly to the private/listname
# which has o+rx permissions. Private archives do not have the
# symbolic links.
+ archdir = archive_dir(self.fqdn_listname)
omask = os.umask(0)
try:
try:
- os.mkdir(self.archive_dir()+'.mbox', 02775)
+ os.mkdir(archdir+'.mbox', 02775)
except OSError, e:
if e.errno <> errno.EEXIST: raise
# We also create an empty pipermail archive directory into
@@ -93,12 +93,12 @@
# that lists that have not yet received a posting have
# /something/ as their index.html, and don't just get a 404.
try:
- os.mkdir(self.archive_dir(), 02775)
+ os.mkdir(archdir, 02775)
except OSError, e:
if e.errno <> errno.EEXIST: raise
# See if there's an index.html file there already and if not,
# write in the empty archive notice.
- indexfile = os.path.join(self.archive_dir(), 'index.html')
+ indexfile = os.path.join(archdir, 'index.html')
fp = None
try:
fp = open(indexfile)
@@ -119,11 +119,6 @@
finally:
os.umask(omask)
- def archive_dir(self):
- # Return the private archive directory
- return os.path.join(config.PRIVATE_ARCHIVE_FILE_DIR,
- self.fqdn_listname)
-
def ArchiveFileName(self):
"""The mbox name where messages are left for archive construction."""
return os.path.join(self.archive_dir() + '.mbox',
=== modified file 'mailman/Archiver/HyperArch.py'
--- a/mailman/Archiver/HyperArch.py 2008-02-08 04:01:48 +0000
+++ b/mailman/Archiver/HyperArch.py 2008-03-30 04:06:07 +0000
@@ -43,15 +43,14 @@
from email.Header import decode_header, make_header
from locknix.lockfile import Lock
-from Mailman import Errors
-from Mailman import MailList
-from Mailman import Utils
-from Mailman import i18n
-from Mailman.Archiver import HyperDatabase
-from Mailman.Archiver import pipermail
-from Mailman.Mailbox import ArchiverMailbox
-from Mailman.SafeDict import SafeDict
-from Mailman.configuration import config
+from mailman import Errors
+from mailman import Utils
+from mailman import i18n
+from mailman.Archiver import HyperDatabase
+from mailman.Archiver import pipermail
+from mailman.Mailbox import ArchiverMailbox
+from mailman.SafeDict import SafeDict
+from mailman.configuration import config
log = logging.getLogger('mailman.error')
@@ -303,30 +302,6 @@
self.decode_headers()
- # Mapping of listnames to MailList instances as a weak value dictionary.
- # This code is copied from Runner.py but there's one important operational
- # difference. In Runner.py, we always .Load() the MailList object for
- # each _dispose() run, otherwise the object retrieved from the cache won't
- # be up-to-date. Since we're creating a new HyperArchive instance for
- # each message being archived, we don't need to worry about that -- but it
- # does mean there are additional opportunities for optimization.
- _listcache = weakref.WeakValueDictionary()
-
- def _open_list(self, listname):
- # Cache the open list so that any use of the list within this process
- # uses the same object. We use a WeakValueDictionary so that when the
- # list is no longer necessary, its memory is freed.
- mlist = self._listcache.get(listname)
- if not mlist:
- try:
- mlist = MailList.MailList(listname, lock=0)
- except Errors.MMListError, e:
- log.error('error opening list: %s\n%s', listname, e)
- return None
- else:
- self._listcache[listname] = mlist
- return mlist
-
def __getstate__(self):
d = self.__dict__.copy()
# We definitely don't want to pickle the MailList instance, so just
@@ -355,7 +330,7 @@
listname = d.get('__listname')
if listname:
del d['__listname']
- d['_mlist'] = self._open_list(listname)
+ d['_mlist'] = config.db.list_manager.get(listname)
if not d.has_key('_lang'):
if hasattr(self, '_mlist'):
self._lang = self._mlist.preferred_language
@@ -394,7 +369,7 @@
if subject:
if config.ARCHIVER_OBSCURES_EMAILADDRS:
with i18n.using_language(self._lang):
- atmark = unicode(_(' at '), Utils.GetCharSet(self._lang))
+ atmark = _(' at ')
subject = re.sub(r'([-+,.\w]+)@([-+.\w]+)',
'\g<1>' + atmark + '\g<2>', subject)
self.decoded['subject'] = subject
@@ -443,7 +418,7 @@
if config.ARCHIVER_OBSCURES_EMAILADDRS:
# Point the mailto url back to the list
author = re.sub('@', _(' at '), self.author)
- emailurl = self._mlist.GetListEmail()
+ emailurl = self._mlist.posting_address
else:
author = self.author
emailurl = self.email
@@ -451,7 +426,7 @@
d["email_url"] = url_quote(emailurl)
d["datestr_html"] = self.quote(i18n.ctime(int(self.date)))
d["body"] = self._get_body()
- d['listurl'] = self._mlist.GetScriptURL('listinfo', absolute=1)
+ d['listurl'] = self._mlist.script_url('listinfo')
d['listname'] = self._mlist.real_name
d['encoding'] = ''
charset = Utils.GetCharSet(self._lang)
@@ -546,7 +521,7 @@
body = unicode(body, cset, 'replace')
if config.ARCHIVER_OBSCURES_EMAILADDRS:
with i18n.using_language(self._lang):
- atmark = unicode(_(' at '), cset)
+ atmark = _(' at ')
body = re.sub(r'([-+,.\w]+)@([-+.\w]+)',
'\g<1>' + atmark + '\g<2>', body)
# Return body to character set of article.
@@ -650,7 +625,7 @@
with i18n.using_language(mlist.preferred_language):
d = {"lastdate": quotetime(self.lastdate),
"archivedate": quotetime(self.archivedate),
- "listinfo": mlist.GetScriptURL('listinfo', absolute=1),
+ "listinfo": mlist.script_url('listinfo'),
"version": self.version,
}
i = {"thread": _("thread"),
@@ -679,7 +654,7 @@
d = {"listname": html_quote(mlist.real_name, self.lang),
"archtype": self.type,
"archive": self.volNameToDesc(self.archive),
- "listinfo": mlist.GetScriptURL('listinfo', absolute=1),
+ "listinfo": mlist.script_url('listinfo'),
"firstdate": quotetime(self.firstdate),
"lastdate": quotetime(self.lastdate),
"size": self.size,
@@ -710,7 +685,7 @@
listname = mlist.fqdn_listname
mbox = os.path.join(mlist.archive_dir()+'.mbox', listname+'.mbox')
d = {"listname": mlist.real_name,
- "listinfo": mlist.GetScriptURL('listinfo', absolute=1),
+ "listinfo": mlist.script_url('listinfo'),
"fullarch": '../%s.mbox/%s.mbox' % (listname, listname),
"size": sizeof(mbox, mlist.preferred_language),
'meta': '',
=== modified file 'mailman/Archiver/pipermail.py'
--- a/mailman/Archiver/pipermail.py 2006-04-17 04:08:17 +0000
+++ b/mailman/Archiver/pipermail.py 2008-03-30 04:06:07 +0000
@@ -13,13 +13,13 @@
from email.Utils import parseaddr, parsedate_tz, mktime_tz, formatdate
from string import lowercase
-__version__ = '0.10 (Mailman edition)'
+__version__ = '0.11 (Mailman edition)'
VERSION = __version__
CACHESIZE = 100 # Number of slots in the cache
-from Mailman import Errors
-from Mailman.Mailbox import ArchiverMailbox
-from Mailman.i18n import _
+from mailman import Errors
+from mailman.Mailbox import ArchiverMailbox
+from mailman.i18n import _
SPACE = ' '
=== modified file 'mailman/Defaults.py'
--- a/mailman/Defaults.py 2008-02-27 06:26:18 +0000
+++ b/mailman/Defaults.py 2008-03-30 04:06:07 +0000
@@ -273,7 +273,7 @@
# a MailList object and a Message object. It should raise
# Errors.DiscardMessage if it wants to throw the message away. Otherwise it
# should modify the Message object as necessary.
-ARCHIVE_SCRUBBER = 'mailman.Handlers.Scrubber'
+ARCHIVE_SCRUBBER = 'mailman.pipeline.scrubber'
# Control parameter whether mailman.Handlers.Scrubber should use message
# attachment's filename as is indicated by the filename parameter or use
=== modified file 'mailman/app/archiving.py'
--- a/mailman/app/archiving.py 2008-02-27 06:26:18 +0000
+++ b/mailman/app/archiving.py 2008-03-30 04:06:07 +0000
@@ -17,21 +17,46 @@
"""Application level archiving support."""
+__metaclass__ = type
__all__ = [
'Pipermail',
- 'get_archiver',
+ 'get_primary_archiver',
]
-__metaclass__ = type
-
+
+
+import os
+import pkg_resources
from string import Template
from zope.interface import implements
from zope.interface.verify import verifyObject
-from mailman.app.plugins import get_plugin
+from mailman.app.plugins import get_plugins
from mailman.configuration import config
from mailman.interfaces import IArchiver
+from mailman.Archiver.HyperArch import HyperArchive
+from cStringIO import StringIO
+
+
+
+class PipermailMailingListAdapter:
+ """An adapter for MailingList objects to work with Pipermail."""
+
+ def __init__(self, mlist):
+ self._mlist = mlist
+
+ def __getattr__(self, name):
+ return getattr(self._mlist, name)
+
+ def archive_dir(self):
+ """The directory for storing Pipermail artifacts."""
+ if self._mlist.archive_private:
+ basedir = config.PRIVATE_ARCHIVE_FILE_DIR
+ else:
+ basedir = config.PUBLIC_ARCHIVE_FILE_DIR
+ return os.path.join(basedir, self._mlist.fqdn_listname)
+
class Pipermail:
@@ -39,39 +64,47 @@
implements(IArchiver)
- def get_list_url(self, mlist):
+ def __init__(self, mlist):
+ self._mlist = mlist
+
+ def get_list_url(self):
"""See `IArchiver`."""
- if mlist.archive_private:
- url = mlist.script_url('private') + '/index.html'
+ if self._mlist.archive_private:
+ url = self._mlist.script_url('private') + '/index.html'
else:
- web_host = config.domains.get(mlist.host_name, mlist.host_name)
+ web_host = config.domains.get(
+ self._mlist.host_name, self._mlist.host_name)
url = Template(config.PUBLIC_ARCHIVE_URL).safe_substitute(
- listname=mlist.fqdn_listname,
+ listname=self._mlist.fqdn_listname,
hostname=web_host,
- fqdn_listname=mlist.fqdn_listname,
+ fqdn_listname=self._mlist.fqdn_listname,
)
return url
- def get_message_url(self, mlist, message):
+ def get_message_url(self, message):
"""See `IArchiver`."""
# Not currently implemented.
return None
- def archive_message(self, mlist, message):
+ def archive_message(self, message):
"""See `IArchiver`."""
+ text = str(message)
+ fileobj = StringIO(text)
+ h = HyperArchive(PipermailMailingListAdapter(self._mlist))
+ h.processUnixMailbox(fileobj)
+ h.close()
+ fileobj.close()
+ # There's no good way to know the url for the archived message.
return None
-_archiver = None
-
-def get_archiver():
- """Return the currently registered global archiver.
-
- Uses the plugin architecture to find the IArchiver instance.
- """
- global _archiver
- if _archiver is None:
- _archiver = get_plugin('mailman.archiver')()
- verifyObject(IArchiver, _archiver)
- return _archiver
+def get_primary_archiver(mlist):
+ """Return the primary archiver."""
+ entry_points = list(pkg_resources.iter_entry_points('mailman.archiver'))
+ if len(entry_points) == 0:
+ return None
+ for ep in entry_points:
+ if ep.name == 'default':
+ return ep.load()(mlist)
+ return None
=== modified file 'mailman/bin/master.py'
--- a/mailman/bin/master.py 2008-03-26 02:28:57 +0000
+++ b/mailman/bin/master.py 2008-03-30 04:06:07 +0000
@@ -213,7 +213,7 @@
Lock host: $hostname
Exiting.""")
- parser.error(message)
+ config.options.parser.error(message)
=== modified file 'mailman/pipeline/cook_headers.py'
--- a/mailman/pipeline/cook_headers.py 2008-02-27 06:26:18 +0000
+++ b/mailman/pipeline/cook_headers.py 2008-03-30 04:06:07 +0000
@@ -31,10 +31,11 @@
from mailman import Utils
from mailman import Version
-from mailman.app.archiving import get_archiver
from mailman.configuration import config
from mailman.i18n import _
from mailman.interfaces import IHandler, Personalization, ReplyToMunging
+from mailman.app.archiving import get_primary_archiver
+
CONTINUATION = ',\n\t'
COMMASPACE = ', '
@@ -205,7 +206,7 @@
'List-Unsubscribe': subfieldfmt % (listinfo, mlist.leave_address),
'List-Subscribe' : subfieldfmt % (listinfo, mlist.join_address),
})
- archiver = get_archiver()
+ archiver = get_primary_archiver(mlist)
if msgdata.get('reduced_list_headers'):
headers['X-List-Administrivia'] = 'yes'
else:
@@ -214,7 +215,7 @@
headers['List-Post'] = '<mailto:%s>' % mlist.posting_address
# Add this header if we're archiving
if mlist.archive:
- archiveurl = archiver.get_list_url(mlist)
+ archiveurl = archiver.get_list_url()
headers['List-Archive'] = '<%s>' % archiveurl
# XXX RFC 2369 also defines a List-Owner header which we are not currently
# supporting, but should.
@@ -222,7 +223,7 @@
# Draft RFC 5064 defines an Archived-At header which contains the pointer
# directly to the message in the archive. If the currently defined
# archiver can tell us the URL, go ahead and include this header.
- archived_at = archiver.get_message_url(mlist, msg)
+ archived_at = archiver.get_message_url(msg)
if archived_at is not None:
headers['Archived-At'] = archived_at
# First we delete any pre-existing headers because the RFC permits only
=== modified file 'mailman/pipeline/scrubber.py'
--- a/mailman/pipeline/scrubber.py 2008-02-27 06:26:18 +0000
+++ b/mailman/pipeline/scrubber.py 2008-03-30 04:06:07 +0000
@@ -40,7 +40,7 @@
from mailman import Utils
from mailman.Errors import DiscardMessage
-from mailman.app.archiving import get_archiver
+from mailman.app.archiving import get_primary_archiver
from mailman.configuration import config
from mailman.i18n import _
from mailman.interfaces import IHandler
@@ -497,7 +497,7 @@
fp.write(decodedpayload)
fp.close()
# Now calculate the url to the list's archive.
- baseurl = get_archiver().get_list_url(mlist)
+ baseurl = get_primary_archiver().get_list_url(mlist)
if not baseurl.endswith('/'):
baseurl += '/'
# Trailing space will definitely be a problem with format=flowed.
=== modified file 'mailman/pipeline/to_digest.py'
--- a/mailman/pipeline/to_digest.py 2008-03-27 09:14:14 +0000
+++ b/mailman/pipeline/to_digest.py 2008-03-30 04:06:07 +0000
@@ -347,7 +347,7 @@
print >> plainmsg
# Now add the footer
if mlist.digest_footer:
- footertxt = decorate(mlist, mlist.digest_footer, _('digest footer'))
+ footertxt = decorate(mlist, mlist.digest_footer)
# MIME
footer = MIMEText(footertxt.encode(lcset), _charset=lcset)
footer['Content-Description'] = _('Digest Footer')
=== modified file 'mailman/queue/archive.py'
--- a/mailman/queue/archive.py 2008-03-27 09:14:14 +0000
+++ b/mailman/queue/archive.py 2008-03-30 04:06:07 +0000
@@ -19,12 +19,19 @@
from __future__ import with_statement
+__metaclass__ = type
+__all__ = [
+ 'ArchiveRunner',
+ ]
+
+
import os
import time
from email.Utils import parsedate_tz, mktime_tz, formatdate
from locknix.lockfile import Lock
+from mailman.app.plugins import get_plugins
from mailman.configuration import config
from mailman.queue import Runner
@@ -70,4 +77,7 @@
msg['X-List-Received-Date'] = receivedtime
# While a list archiving lock is acquired, archive the message.
with Lock(os.path.join(mlist.data_path, 'archive.lck')):
- mlist.ArchiveMail(msg)
+ for archive_factory in get_plugins('mailman.archiver'):
+ archiver = archive_factory(mlist)
+ archiver.archive_message(msg)
+
=== added file 'mailman/queue/docs/archiver.txt'
--- a/mailman/queue/docs/archiver.txt 1970-01-01 00:00:00 +0000
+++ b/mailman/queue/docs/archiver.txt 2008-03-30 04:06:07 +0000
@@ -0,0 +1,35 @@
+Archiving
+=========
+
+Mailman can archive to any number of archivers that adhere to the IArchiver
+interface. By default, there's a Pipermail archiver.
+
+ >>> from mailman.app.lifecycle import create_list
+ >>> mlist = create_list(u'[EMAIL PROTECTED]')
+ >>> mlist.web_page_url = u'http://www.example.com/'
+
+ >>> msg = message_from_string("""\
+ ... From: [EMAIL PROTECTED]
+ ... To: [EMAIL PROTECTED]
+ ... Subject: My first post
+ ... Message-ID: <first>
+ ...
+ ... First post!
+ ... """)
+
+ >>> from mailman.configuration import config
+ >>> from mailman.queue import Switchboard
+ >>> archiver_queue = Switchboard(config.ARCHQUEUE_DIR)
+ >>> ignore = archiver_queue.enqueue(msg, {}, listname=mlist.fqdn_listname)
+
+ >>> from mailman.queue.archive import ArchiveRunner
+ >>> from mailman.tests.helpers import make_testable_runner
+ >>> runner = make_testable_runner(ArchiveRunner)
+ >>> runner.run()
+
+ # The best we can do is verify some landmark exists. Let's use the
+ # Pipermail pickle file exists.
+ >>> import os
+ >>> os.path.exists(os.path.join(config.PUBLIC_ARCHIVE_FILE_DIR,
+ ... mlist.fqdn_listname, 'pipermail.pck'))
+ True
=== modified file 'mailman/queue/outgoing.py'
--- a/mailman/queue/outgoing.py 2008-03-27 09:14:14 +0000
+++ b/mailman/queue/outgoing.py 2008-03-30 04:06:07 +0000
@@ -20,10 +20,10 @@
import os
import sys
import copy
-import time
import email
import socket
import logging
+import datetime
from mailman import Errors
from mailman import Message
@@ -57,7 +57,7 @@
def _dispose(self, mlist, msg, msgdata):
# See if we should retry delivery of this message again.
deliver_after = msgdata.get('deliver_after', 0)
- if time.time() < deliver_after:
+ if datetime.datetime.now() < deliver_after:
return True
# Make sure we have the most up-to-date state
try:
@@ -102,7 +102,7 @@
# occasionally move them back here for another shot at
# delivery.
if e.tempfailures:
- now = time.time()
+ now = datetime.datetime.now()
recips = e.tempfailures
last_recip_count = msgdata.get('last_recip_count', 0)
deliver_until = msgdata.get('deliver_until', now)
=== modified file 'mailman/rules/administrivia.py'
--- a/mailman/rules/administrivia.py 2008-02-27 06:26:18 +0000
+++ b/mailman/rules/administrivia.py 2008-03-30 04:06:07 +0000
@@ -64,7 +64,7 @@
return False
# First check the Subject text.
lines_to_check = []
- subject = msg.get('subject', '')
+ subject = str(msg.get('subject', ''))
if subject <> '':
lines_to_check.append(subject)
# Search only the first text/plain subpart of the message. There's
=== modified file 'setup.py'
--- a/setup.py 2008-02-27 06:26:18 +0000
+++ b/setup.py 2008-03-30 04:06:07 +0000
@@ -83,7 +83,7 @@
entry_points = {
'console_scripts': list(scripts),
# Entry point for plugging in different database backends.
- 'mailman.archiver' : 'stock = mailman.app.archiving:Pipermail',
+ 'mailman.archiver' : 'default = mailman.app.archiving:Pipermail',
'mailman.database' : 'stock = mailman.database:StockDatabase',
'mailman.mta' : 'stock = mailman.MTA:Manual',
'mailman.styles' : 'default = mailman.app.styles:DefaultStyle',
--
Primary development focus
https://code.launchpad.net/~mailman-coders/mailman/3.0
You are receiving this branch notification because you are subscribed to it.
_______________________________________________
Mailman-checkins mailing list
[email protected]
Unsubscribe:
http://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
