Patch to implement hashed per-IP connection limiting in qmail-send and qmail-remote. See the included README.concurrencyperip file for detauls. This patch was created by Joshua Megerman (josh@coyotetechnical.com) and is released to the public domain. Latest version can be found at http://www.coyotetechnical.com/software/patches/qmail-send-concurrencyperip.patch diff -urN ../../clean/netqmail-1.05/FILES netqmail-1.05/FILES --- ../../clean/netqmail-1.05/FILES 2005-11-23 17:05:59.000000000 -0500 +++ netqmail-1.05/FILES 2007-01-12 09:55:59.000000000 -0500 @@ -432,3 +432,5 @@ tcp-environ.5 constmap.h constmap.c +sem.h +sem.c diff -urN ../../clean/netqmail-1.05/Makefile netqmail-1.05/Makefile --- ../../clean/netqmail-1.05/Makefile 2005-11-23 17:05:59.000000000 -0500 +++ netqmail-1.05/Makefile 2007-01-12 09:59:00.000000000 -0500 @@ -1441,12 +1441,12 @@ load qmail-remote.o control.o constmap.o timeoutread.o timeoutwrite.o \ timeoutconn.o tcpto.o now.o dns.o ip.o ipalloc.o ipme.o quote.o \ ndelay.a case.a sig.a open.a lock.a seek.a getln.a stralloc.a alloc.a \ -substdio.a error.a str.a fs.a auto_qmail.o dns.lib socket.lib +substdio.a error.a str.a fs.a auto_qmail.o dns.lib socket.lib sem.o ./load qmail-remote control.o constmap.o timeoutread.o \ timeoutwrite.o timeoutconn.o tcpto.o now.o dns.o ip.o \ ipalloc.o ipme.o quote.o ndelay.a case.a sig.a open.a \ lock.a seek.a getln.a stralloc.a alloc.a substdio.a error.a \ - str.a fs.a auto_qmail.o `cat dns.lib` `cat socket.lib` + str.a fs.a auto_qmail.o `cat dns.lib` `cat socket.lib` sem.o qmail-remote.0: \ qmail-remote.8 @@ -1457,7 +1457,7 @@ subfd.h substdio.h scan.h case.h error.h auto_qmail.h control.h dns.h \ alloc.h quote.h ip.h ipalloc.h ip.h gen_alloc.h ipme.h ip.h ipalloc.h \ gen_alloc.h gen_allocdefs.h str.h now.h datetime.h exit.h constmap.h \ -tcpto.h readwrite.h timeoutconn.h timeoutread.h timeoutwrite.h +tcpto.h readwrite.h timeoutconn.h timeoutread.h timeoutwrite.h sem.h ./compile qmail-remote.c qmail-rspawn: \ @@ -1483,12 +1483,12 @@ trigger.o fmtqfn.o quote.o now.o readsubdir.o qmail.o date822fmt.o \ datetime.a case.a ndelay.a getln.a wait.a seek.a fd.a sig.a open.a \ lock.a stralloc.a alloc.a substdio.a error.a str.a fs.a auto_qmail.o \ -auto_split.o env.a +auto_split.o env.a sem.o ./load qmail-send qsutil.o control.o constmap.o newfield.o \ prioq.o trigger.o fmtqfn.o quote.o now.o readsubdir.o \ qmail.o date822fmt.o datetime.a case.a ndelay.a getln.a \ wait.a seek.a fd.a sig.a open.a lock.a stralloc.a alloc.a \ - substdio.a error.a str.a fs.a auto_qmail.o auto_split.o env.a + substdio.a error.a str.a fs.a auto_qmail.o auto_split.o env.a sem.o qmail-send.0: \ qmail-send.8 @@ -1508,7 +1508,7 @@ substdio.h alloc.h error.h stralloc.h gen_alloc.h str.h byte.h fmt.h \ scan.h case.h auto_qmail.h trigger.h newfield.h stralloc.h quote.h \ qmail.h substdio.h qsutil.h prioq.h datetime.h gen_alloc.h constmap.h \ -fmtqfn.h readsubdir.h direntry.h +fmtqfn.h readsubdir.h direntry.h sem.h ./compile qmail-send.c qmail-showctl: \ @@ -1731,6 +1731,10 @@ && cat select.h2 || cat select.h1 ) > select.h rm -f trysysel.o trysysel +sem.o: \ +compile sem.c sem.h ip.h + ./compile sem.c + sendmail: \ load sendmail.o env.a getopt.a alloc.a substdio.a error.a str.a \ auto_qmail.o diff -urN ../../clean/netqmail-1.05/qmail-control.9 netqmail-1.05/qmail-control.9 --- ../../clean/netqmail-1.05/qmail-control.9 1998-06-15 06:53:16.000000000 -0400 +++ netqmail-1.05/qmail-control.9 2007-01-12 10:11:49.000000000 -0500 @@ -45,6 +45,7 @@ .I bouncehost \fIme \fRqmail-send .I concurrencylocal \fR10 \fRqmail-send .I concurrencyremote \fR20 \fRqmail-send +.I concurrencyperip \fR0 \fRqmail-send .I defaultdomain \fIme \fRqmail-inject .I defaulthost \fIme \fRqmail-inject .I databytes \fR0 \fRqmail-smtpd diff -urN ../../clean/netqmail-1.05/qmail-remote.c netqmail-1.05/qmail-remote.c --- ../../clean/netqmail-1.05/qmail-remote.c 1998-06-15 06:53:16.000000000 -0400 +++ netqmail-1.05/qmail-remote.c 2007-01-12 10:07:14.000000000 -0500 @@ -28,6 +28,7 @@ #include "timeoutconn.h" #include "timeoutread.h" #include "timeoutwrite.h" +#include "sem.h" #define HUGESMTPTEXT 5000 @@ -70,6 +71,10 @@ Unable to switch to home directory. (#4.3.0)\n"); zerodie(); } void temp_control() { out("Z\ Unable to read control files. (#4.3.0)\n"); zerodie(); } +void temp_nosem() { out("Z\ +Unable to setup semaphore for per-ip connection limiting. (#4.3.0)\n"); zerodie(); } +void temp_semnoconn() { out("Z\ +Sorry, there were too many simultaneous SMTP connections to the remote server. (#4.4.5)\n"); zerodie(); } void perm_partialline() { out("D\ SMTP cannot transfer messages with partial final lines. (#5.6.2)\n"); zerodie(); } void perm_usage() { out("D\ @@ -308,6 +313,8 @@ if (!stralloc_cat(saout,&canonhost)) temp_nomem(); } +int perip = 0; + void getcontrols() { if (control_init() == -1) temp_control(); @@ -324,6 +331,7 @@ case 1: if (!constmap_init(&maproutes,routes.s,routes.len,1)) temp_nomem(); break; } + if (control_readint(&perip,"control/concurrencyperip") == -1) temp_nosem(); } void main(argc,argv) @@ -331,18 +339,23 @@ char **argv; { static ipalloc ip = {0}; - int i; + int i, j; unsigned long random; char **recips; unsigned long prefme; int flagallaliases; int flagalias; char *relayhost; + int semdelay = 0; sig_pipeignore(); if (argc < 4) perm_usage(); if (chdir(auto_qmail) == -1) temp_chdir(); getcontrols(); + if (perip) { + i = setup_semaphore(0); + if (i == -1) temp_nosem(); + } if (!stralloc_copys(&host,argv[1])) temp_nomem(); @@ -411,6 +424,14 @@ for (i = 0;i < ip.len;++i) if (ip.ix[i].pref < prefme) { if (tcpto(&ip.ix[i].ip)) continue; + if (perip) { + j = dec_semaphore(&ip.ix[i].ip); + if (j == -1) { + semdelay = 1; + continue; + } + } + smtpfd = socket(AF_INET,SOCK_STREAM,0); if (smtpfd == -1) temp_oserr(); @@ -421,7 +442,14 @@ } tcpto_err(&ip.ix[i].ip,errno == error_timeout); close(smtpfd); + if (perip) { + j = inc_semaphore(&ip.ix[i].ip); + } } - temp_noconn(); + if (semdelay) { + temp_semnoconn(); + } else { + temp_noconn(); + } } diff -urN ../../clean/netqmail-1.05/qmail-send.9 netqmail-1.05/qmail-send.9 --- ../../clean/netqmail-1.05/qmail-send.9 1998-06-15 06:53:16.000000000 -0400 +++ netqmail-1.05/qmail-send.9 2007-01-12 10:10:48.000000000 -0500 @@ -93,6 +93,14 @@ is limited at compile time to SPAWN. .TP 5 +.I concurrencyperip +Maximum number of simultaneous remote delivery attempts per hashed IP address. +Default: 0 +If 0, per-address deliveries will not be limited. +.I concurrencyperip +is limited at compile time to +32767. +.TP 5 .I doublebouncehost Double-bounce host. Default: diff -urN ../../clean/netqmail-1.05/qmail-send.c netqmail-1.05/qmail-send.c --- ../../clean/netqmail-1.05/qmail-send.c 1998-06-15 06:53:16.000000000 -0400 +++ netqmail-1.05/qmail-send.c 2007-01-12 09:55:21.000000000 -0500 @@ -31,6 +31,7 @@ #include "constmap.h" #include "fmtqfn.h" #include "readsubdir.h" +#include "sem.h" /* critical timing feature #1: if not triggered, do not busy-loop */ /* critical timing feature #2: if triggered, respond within fixed time */ @@ -782,6 +783,7 @@ unsigned long masterdelid = 1; unsigned int concurrency[CHANNELS] = { 10, 20 }; unsigned int concurrencyused[CHANNELS] = { 0, 0 }; +unsigned int concurrencyperip = 0; struct del *d[CHANNELS]; stralloc dline[CHANNELS]; char delbuf[2048]; @@ -1445,6 +1447,7 @@ if (control_readint(&lifetime,"control/queuelifetime") == -1) return 0; if (control_readint(&concurrency[0],"control/concurrencylocal") == -1) return 0; if (control_readint(&concurrency[1],"control/concurrencyremote") == -1) return 0; + if (control_readint(&concurrencyperip,"control/concurrencyperip") == -1) return 0; if (control_rldef(&envnoathost,"control/envnoathost",1,"envnoathost") != 1) return 0; if (control_rldef(&bouncefrom,"control/bouncefrom",0,"MAILER-DAEMON") != 1) return 0; if (control_rldef(&bouncehost,"control/bouncehost",1,"bouncehost") != 1) return 0; @@ -1521,11 +1524,21 @@ int nfds; struct timeval tv; int c; + char cip_strnum[FMT_ULONG]; if (chdir(auto_qmail) == -1) { log1("alert: cannot start: unable to switch to home directory\n"); _exit(111); } if (!getcontrols()) { log1("alert: cannot start: unable to read controls\n"); _exit(111); } + if (concurrencyperip) + { + cip_strnum[fmt_ulong(cip_strnum,concurrencyperip)] = 0; + log3("per-IP concurrency is set to: ", cip_strnum, "\n"); + c = setup_semaphore(concurrencyperip); + if (c == -1) + { log1("alert: cannot setup semaphore for outbound per-ip rate limiting\n"); _exit(111); } + } + else semaphore = -1; if (chdir("queue") == -1) { log1("alert: cannot start: unable to switch to queue directory\n"); _exit(111); } sig_pipeignore(); @@ -1607,6 +1620,10 @@ } } pqfinish(); + if (semaphore != -1) + { c = destroy_semaphore(); } + if (c == -1) + { log1("warning: could not destroy semaphore\n"); } log1("status: exiting\n"); _exit(0); } diff -urN ../../clean/netqmail-1.05/README.concurrencyperip netqmail-1.05/README.concurrencyperip --- ../../clean/netqmail-1.05/README.concurrencyperip 1969-12-31 19:00:00.000000000 -0500 +++ netqmail-1.05/README.concurrencyperip 2007-01-12 10:16:34.000000000 -0500 @@ -0,0 +1,34 @@ +Qmail Per-IP Limit Patch by Joshua Megerman + +Because qmail attempts to deliver every message individually, and doesn't +have any way to limit the number of connections to a given system at one time, +some mail servers may respond slowly or not at all once a certain number of +connections have been made. This patch provides the ability to limit the +number of deliveries going to a single IP address at any given time by using +semaphores to allow or deny delivery. This patch uses semaphores, and may not +be portable to OS's other than Linux. In order to enable this patch, you need +to enter the per-IP concurrency limit into control/concurrencyperip - if you +enter 0 or don't create the control file, qmail will default to not using the +semaphores and deliver mail in the traditional manner. + +In order to conserve semaphores, a hash method is used to reduce the 4 octets +of the IP address to a single 8-bit integer. For an IP address a.b.c.d, the +has is ((a^b)^(c^d)), where ^ is the C XOR operator. THus multiple IP +addresses will map to the same hash, and can potentially be limited to a +combined total # of remote connections. If too many connections are open to a +particular IP address (hash), qmail-remote will try the next available MX, if +there is one. If there are no more MXs available, qmail-remote will fail with +a 4XX and the message will be requeued. This is the best solution to avoid +holding up qmail-remote processes waiting for delivery in high-volume settings, +even if it does delay mail delivery slightly for some messages. + +NOTE: Most Linux systems have a default maximum of 250 semaphores per semaphore +set, and this patch uses 256. This is just a kernel tuning parameter in /proc +and can be adjusted as follows: + +1) echo "256 32000 32 128" > /proc/sys/kernel/sem +2) place the following line in /etc/sysctl.conf (or its equivalent) to set it +on startup: + kernel.sem = 256 32000 32 128 + +Questions, comments and bug reports to josh@honorablemenschen.com diff -urN ../../clean/netqmail-1.05/sem.c netqmail-1.05/sem.c --- ../../clean/netqmail-1.05/sem.c 1969-12-31 19:00:00.000000000 -0500 +++ netqmail-1.05/sem.c 2007-01-12 09:55:21.000000000 -0500 @@ -0,0 +1,88 @@ +#include +#include +#include +#include "sem.h" +#include "ip.h" +#include "qsutil.h" + +int semaphore; + +int setup_semaphore(int max_out) { + + key_t key; + int i; + unsigned short array[256]; + union semun semopts; + + if ((max_out > 32767) || (max_out < 0)) { + return(-1); + } + + if ((key = ftok("control/me", 'q')) == -1) { + return(-1); + } + if ((semaphore = semget(key, 256, 0660 | IPC_CREAT)) == -1) { + return(-1); + } + + if (max_out) { + for (i = 0; i < 256; i++) { array[i] = max_out; } + semopts.array = array; + if ((i = semctl(semaphore, 0, SETALL, array)) == -1) { + semctl(semaphore, 0, IPC_RMID); + return(-1); + } + } + + return(0); +} + +int destroy_semaphore() { + + int i; + + if((i = semctl(semaphore, 0, IPC_RMID)) == -1) { + return(-1); + } + return(0); +} + +int dec_semaphore(struct ip_address *ip) { + + struct sembuf sop; + int i; + unsigned char a, b, c, d; + a = ip->d[0]; + b = ip->d[1]; + c = ip->d[2]; + d = ip->d[3]; + + sop.sem_num = ((a^b)^(c^d)); + sop.sem_op = -1; + sop.sem_flg = (IPC_NOWAIT | SEM_UNDO); + + if ((i = semop(semaphore, &sop, 1)) == -1) { + return(-1); + } + return(0); +} + +int inc_semaphore(struct ip_address *ip) { + + struct sembuf sop; + int i; + unsigned char a, b, c, d; + a = ip->d[0]; + b = ip->d[1]; + c = ip->d[2]; + d = ip->d[3]; + + sop.sem_num = ((a^b)^(c^d)); + sop.sem_op = 1; + sop.sem_flg = SEM_UNDO; + + if ((i = semop(semaphore, &sop, 1)) == -1) { + return(-1); + } + return(0); +} diff -urN ../../clean/netqmail-1.05/sem.h netqmail-1.05/sem.h --- ../../clean/netqmail-1.05/sem.h 1969-12-31 19:00:00.000000000 -0500 +++ netqmail-1.05/sem.h 2007-01-12 09:55:21.000000000 -0500 @@ -0,0 +1,24 @@ +#ifndef SEM_H +#define SEM_H + +#if defined(__GNU_LIBRARY__) && !defined(_SEM_SEMUN_UNDEFINED) +/* union semun is defined by including */ +#else +/* according to X/OPEN we have to define it ourselves */ +union semun { + int val; /* value for SETVAL */ + struct semid_ds *buf; /* buffer for IPC_STAT, IPC_SET */ + unsigned short *array; /* array for GETALL, SETALL */ + /* Linux specific part: */ + struct seminfo *__buf; /* buffer for IPC_INFO */ +}; +#endif + +extern int semaphore; + +int setup_semaphore(); +int destroy_semaphore(); +int dec_semaphore(); +int inc_semaphore(); + +#endif diff -urN ../../clean/netqmail-1.05/TARGETS netqmail-1.05/TARGETS --- ../../clean/netqmail-1.05/TARGETS 1998-06-15 06:53:16.000000000 -0400 +++ netqmail-1.05/TARGETS 2007-01-12 10:05:45.000000000 -0500 @@ -385,3 +385,4 @@ man setup check +sem.o