On Fri, Nov 09, 2001 at 09:25:52AM +0100, Andre Oppermann allegedly wrote:
> Mark Delany wrote:
> >
> > Did you see my earlier patches that approach this from a different
> > angle? Namely constraining injections to the qmail-send service rate
> > and reading trigger (rather than opening/closing) so multiple trigger
> > writes result in a single todo scan.
>
> No, I did not see them. Could you repost?
The wait_for_unlink.patch is for systems that have a very real risk of
long-term insertion rates that constantly exceed qmail-sends
processing rate. This patch effectively flow-controls insertion so
that todo never gets too big. Not for poorly managed systems as
insertions will stall if qmail-send is not running.
The wait_for_unlink.patch is only known to work on FreeBSD but should
port to any system that supports kqueue().
Regards.
diff -cN qmail-1.03/Makefile qmail-wait-for-unlink/Makefile
*** qmail-1.03/Makefile Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/Makefile Wed Oct 3 14:41:16 2001
***************
*** 1421,1431 ****
qmail-queue: \
load qmail-queue.o triggerpull.o fmtqfn.o now.o date822fmt.o \
datetime.a seek.a ndelay.a open.a sig.a alloc.a substdio.a error.a \
! str.a fs.a auto_qmail.o auto_split.o auto_uids.o
./load qmail-queue triggerpull.o fmtqfn.o now.o \
date822fmt.o datetime.a seek.a ndelay.a open.a sig.a \
alloc.a substdio.a error.a str.a fs.a auto_qmail.o \
! auto_split.o auto_uids.o
qmail-queue.0: \
qmail-queue.8
--- 1421,1436 ----
qmail-queue: \
load qmail-queue.o triggerpull.o fmtqfn.o now.o date822fmt.o \
datetime.a seek.a ndelay.a open.a sig.a alloc.a substdio.a error.a \
! str.a fs.a auto_qmail.o auto_split.o auto_uids.o \
! wait_for_unlink.o
./load qmail-queue triggerpull.o fmtqfn.o now.o \
date822fmt.o datetime.a seek.a ndelay.a open.a sig.a \
alloc.a substdio.a error.a str.a fs.a auto_qmail.o \
! auto_split.o auto_uids.o wait_for_unlink.o
!
! wait_for_unlink.o: \
! compile wait_for_unlink.c
! ./compile wait_for_unlink.c
qmail-queue.0: \
qmail-queue.8
diff -cN qmail-1.03/TARGETS qmail-wait-for-unlink/TARGETS
*** qmail-1.03/TARGETS Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/TARGETS Wed Oct 3 14:41:16 2001
***************
*** 40,45 ****
--- 40,46 ----
sig_bug.o
sig_misc.o
sig.a
+ wait_for_unlink.o
open_append.o
open_excl.o
open_read.o
diff -cN qmail-1.03/condredirect.c qmail-wait-for-unlink/condredirect.c
*** qmail-1.03/condredirect.c Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/condredirect.c Wed Oct 3 14:41:16 2001
***************
*** 68,74 ****
dtline = env_get("DTLINE");
if (!dtline) strerr_die2x(100,FATAL,"DTLINE not set");
! if (qmail_open(&qqt) == -1)
strerr_die2sys(111,FATAL,"unable to fork: ");
qmail_puts(&qqt,dtline);
if (substdio_copy(&ssout,&ssin) != 0)
--- 68,74 ----
dtline = env_get("DTLINE");
if (!dtline) strerr_die2x(100,FATAL,"DTLINE not set");
! if (qmail_open(&qqt,1) == -1)
strerr_die2sys(111,FATAL,"unable to fork: ");
qmail_puts(&qqt,dtline);
if (substdio_copy(&ssout,&ssin) != 0)
diff -cN qmail-1.03/forward.c qmail-wait-for-unlink/forward.c
*** qmail-1.03/forward.c Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/forward.c Wed Oct 3 14:41:16 2001
***************
*** 43,49 ****
if (!dtline)
strerr_die2x(100,FATAL,"DTLINE not set");
! if (qmail_open(&qqt) == -1)
strerr_die2sys(111,FATAL,"unable to fork: ");
qmail_puts(&qqt,dtline);
if (substdio_copy(&ssout,&ssin) != 0)
--- 43,49 ----
if (!dtline)
strerr_die2x(100,FATAL,"DTLINE not set");
! if (qmail_open(&qqt,1) == -1)
strerr_die2sys(111,FATAL,"unable to fork: ");
qmail_puts(&qqt,dtline);
if (substdio_copy(&ssout,&ssin) != 0)
diff -cN qmail-1.03/qmail-inject.c qmail-wait-for-unlink/qmail-inject.c
*** qmail-1.03/qmail-inject.c Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/qmail-inject.c Wed Oct 3 14:41:16 2001
***************
*** 587,593 ****
/* could check at this point whether there are any recipients */
if (flagqueue)
! if (qmail_open(&qqt) == -1) die_qqt();
if (flagresent)
{
--- 587,593 ----
/* could check at this point whether there are any recipients */
if (flagqueue)
! if (qmail_open(&qqt,1) == -1) die_qqt();
if (flagresent)
{
diff -cN qmail-1.03/qmail-local.c qmail-wait-for-unlink/qmail-local.c
*** qmail-1.03/qmail-local.c Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/qmail-local.c Wed Oct 3 14:41:16 2001
***************
*** 273,279 ****
if (seek_begin(0) == -1) temp_rewind();
substdio_fdbuf(&ss,read,0,buf,sizeof(buf));
! if (qmail_open(&qqt) == -1) temp_fork();
mailforward_qp = qmail_qp(&qqt);
qmail_put(&qqt,dtline.s,dtline.len);
do
--- 273,279 ----
if (seek_begin(0) == -1) temp_rewind();
substdio_fdbuf(&ss,read,0,buf,sizeof(buf));
! if (qmail_open(&qqt,1) == -1) temp_fork();
mailforward_qp = qmail_qp(&qqt);
qmail_put(&qqt,dtline.s,dtline.len);
do
diff -cN qmail-1.03/qmail-qmqpd.c qmail-wait-for-unlink/qmail-qmqpd.c
*** qmail-1.03/qmail-qmqpd.c Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/qmail-qmqpd.c Wed Oct 3 14:41:17 2001
***************
*** 121,127 ****
len = getlen();
if (chdir(auto_qmail) == -1) resources();
! if (qmail_open(&qq) == -1) resources();
qp = qmail_qp(&qq);
identify();
--- 121,127 ----
len = getlen();
if (chdir(auto_qmail) == -1) resources();
! if (qmail_open(&qq,1) == -1) resources();
qp = qmail_qp(&qq);
identify();
diff -cN qmail-1.03/qmail-qmtpd.c qmail-wait-for-unlink/qmail-qmtpd.c
*** qmail-1.03/qmail-qmtpd.c Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/qmail-qmtpd.c Wed Oct 3 14:41:17 2001
***************
*** 121,127 ****
if (len == 0) badproto();
if (databytes) bytestooverflow = databytes + 1;
! if (qmail_open(&qq) == -1) resources();
qp = qmail_qp(&qq);
substdio_get(&ssin,&ch,1);
--- 121,127 ----
if (len == 0) badproto();
if (databytes) bytestooverflow = databytes + 1;
! if (qmail_open(&qq,1) == -1) resources();
qp = qmail_qp(&qq);
substdio_get(&ssin,&ch,1);
diff -cN qmail-1.03/qmail-queue.c qmail-wait-for-unlink/qmail-queue.c
*** qmail-1.03/qmail-queue.c Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/qmail-queue.c Wed Oct 3 14:41:17 2001
***************
*** 6,11 ****
--- 6,12 ----
#include "open.h"
#include "seek.h"
#include "fmt.h"
+ #include "scan.h"
#include "alloc.h"
#include "substdio.h"
#include "datetime.h"
***************
*** 151,160 ****
char tmp[FMT_ULONG];
! void main()
{
unsigned int len;
char ch;
sig_blocknone();
umask(033);
--- 152,164 ----
char tmp[FMT_ULONG];
! void main(argc,argv)
! int argc;
! char **argv;
{
unsigned int len;
char ch;
+ unsigned long maxwait;
sig_blocknone();
umask(033);
***************
*** 247,254 ****
if (substdio_flush(&ssout) == -1) die_write();
if (fsync(intdfd) == -1) die_write();
if (link(intdfn,todofn) == -1) die(66);
-
triggerpull();
die(0);
}
--- 251,261 ----
if (substdio_flush(&ssout) == -1) die_write();
if (fsync(intdfd) == -1) die_write();
+ maxwait = -1;
+ if (argc == 2) scan_ulong(argv[1],&maxwait);
+ if (maxwait != 0) wfu_register(intdfd);
if (link(intdfn,todofn) == -1) die(66);
triggerpull();
+ if (maxwait != 0) wfu_wait((int) maxwait);
die(0);
}
diff -cN qmail-1.03/qmail-send.c qmail-wait-for-unlink/qmail-send.c
*** qmail-1.03/qmail-send.c Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/qmail-send.c Wed Oct 3 14:41:17 2001
***************
*** 685,691 ****
log3("triple bounce: discarding ",fn2.s,"\n");
else
{
! if (qmail_open(&qqt) == -1)
{ log1("warning: unable to start qmail-queue, will try later\n"); return 0; }
qp = qmail_qp(&qqt);
--- 685,691 ----
log3("triple bounce: discarding ",fn2.s,"\n");
else
{
! if (qmail_open(&qqt,0) == -1)
{ log1("warning: unable to start qmail-queue, will try later\n"); return 0; }
qp = qmail_qp(&qqt);
diff -cN qmail-1.03/qmail-smtpd.c qmail-wait-for-unlink/qmail-smtpd.c
*** qmail-1.03/qmail-smtpd.c Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/qmail-smtpd.c Wed Oct 3 14:41:17 2001
***************
*** 374,380 ****
if (!rcptto.len) { err_wantrcpt(); return; }
seenmail = 0;
if (databytes) bytestooverflow = databytes + 1;
! if (qmail_open(&qqt) == -1) { err_qqt(); return; }
qp = qmail_qp(&qqt);
out("354 go ahead\r\n");
--- 374,380 ----
if (!rcptto.len) { err_wantrcpt(); return; }
seenmail = 0;
if (databytes) bytestooverflow = databytes + 1;
! if (qmail_open(&qqt,1) == -1) { err_qqt(); return; }
qp = qmail_qp(&qqt);
out("354 go ahead\r\n");
diff -cN qmail-1.03/qmail.c qmail-wait-for-unlink/qmail.c
*** qmail-1.03/qmail.c Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/qmail.c Wed Oct 3 14:42:11 2001
***************
*** 7,16 ****
#include "qmail.h"
#include "auto_qmail.h"
! static char *binqqargs[2] = { "bin/qmail-queue", 0 } ;
! int qmail_open(qq)
struct qmail *qq;
{
int pim[2];
int pie[2];
--- 7,18 ----
#include "qmail.h"
#include "auto_qmail.h"
! static char *binqqargswait[3] = { "bin/qmail-queue", "59", 0 } ;
! static char *binqqargsnowait[3] = { "bin/qmail-queue", "0", 0 } ;
! int qmail_open(qq,waitflag)
struct qmail *qq;
+ int waitflag;
{
int pim[2];
int pie[2];
***************
*** 29,35 ****
if (fd_move(0,pim[0]) == -1) _exit(120);
if (fd_move(1,pie[0]) == -1) _exit(120);
if (chdir(auto_qmail) == -1) _exit(61);
! execv(*binqqargs,binqqargs);
_exit(120);
}
--- 31,38 ----
if (fd_move(0,pim[0]) == -1) _exit(120);
if (fd_move(1,pie[0]) == -1) _exit(120);
if (chdir(auto_qmail) == -1) _exit(61);
! if (waitflag) { execv(*binqqargswait,binqqargswait); _exit(120); }
! execv(*binqqargsnowait,binqqargsnowait); _exit(120);
_exit(120);
}
diff -cN qmail-1.03/qreceipt.c qmail-wait-for-unlink/qreceipt.c
*** qmail-1.03/qreceipt.c Mon Jun 15 03:53:16 1998
--- qmail-wait-for-unlink/qreceipt.c Wed Oct 3 14:41:18 2001
***************
*** 68,74 ****
if (!quote2("ed,returnpath)) die_nomem();
! if (qmail_open(&qqt) == -1) die_fork();
qmail_puts(&qqt,"From: DELIVERY NOTICE SYSTEM <");
qmail_put(&qqt,quoted.s,quoted.len);
--- 68,74 ----
if (!quote2("ed,returnpath)) die_nomem();
! if (qmail_open(&qqt,1) == -1) die_fork();
qmail_puts(&qqt,"From: DELIVERY NOTICE SYSTEM <");
qmail_put(&qqt,quoted.s,quoted.len);
diff -cN qmail-1.03/wait_for_unlink.c qmail-wait-for-unlink/wait_for_unlink.c
*** qmail-1.03/wait_for_unlink.c Wed Dec 31 16:00:00 1969
--- qmail-wait-for-unlink/wait_for_unlink.c Wed Oct 3 14:41:18 2001
***************
*** 0 ****
--- 1,42 ----
+ #include <sys/time.h>
+ #include <sys/types.h>
+ #include <sys/event.h>
+
+ static int kfd = -1;
+
+ extern int
+ wfu_register(int fd)
+ {
+
+ struct kevent ev_in;
+
+ if ((kfd = kqueue()) == -1) return -1;
+
+ ev_in.ident = fd;
+ ev_in.flags = EV_ONESHOT | EV_ENABLE | EV_ADD | EV_CLEAR;
+ ev_in.filter = EVFILT_VNODE;
+ ev_in.fflags = NOTE_DELETE;
+ return kevent(kfd, &ev_in, 1,
+ NULL, 0,
+ NULL);
+ }
+
+
+ extern int
+ wfu_wait(int max_delay_seconds)
+ {
+
+ struct kevent ev_out;
+ struct timespec to;
+
+ if (kfd == -1) return -1;
+
+ if (max_delay_seconds > 0) {
+ to.tv_sec = max_delay_seconds;
+ to.tv_nsec = 0;
+ }
+
+ return kevent(kfd, NULL, 0,
+ &ev_out, 1,
+ (max_delay_seconds > 0) ? &to : NULL);
+ }
