rbb 99/02/16 07:34:04
Modified: pthreads/src/main http_main.c
Log:
First pass at accept loop serialization. Right now, it is one mutex for the
whole server. This should be one mutex per socket, but I just wanted to get
the basic code in, I'll clean it up later.
Revision Changes Path
1.30 +512 -3 apache-apr/pthreads/src/main/http_main.c
Index: http_main.c
===================================================================
RCS file: /home/cvs/apache-apr/pthreads/src/main/http_main.c,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -r1.29 -r1.30
--- http_main.c 1999/02/16 07:10:27 1.29
+++ http_main.c 1999/02/16 15:34:04 1.30
@@ -247,8 +247,6 @@
}
#define tls() ((tls_main_t *) gettls(tls_main_key)) /* ZZZZZ */
-
-/* used by accept_loop(), which is serialized */
static listen_rec *head_listener;
/* *Non*-shared http_main globals... */
@@ -427,6 +425,514 @@
exit(code);
}
+
+/****** ZZZ this should probably be abstracted to it's own file. ****/
+
+#if defined(USE_FCNTL_SERIALIZED_ACCEPT) ||
defined(USE_FLOCK_SERIALIZED_ACCEPT)
+static void expand_lock_fname(pool *p)
+{
+ /* XXXX possibly bogus cast */
+ ap_lock_fname = ap_psprintf(p, "%s.%lu",
+ ap_server_root_relative(p, ap_lock_fname), (unsigned long)getpid());
+}
+#endif
+
+#if defined (USE_USLOCK_SERIALIZED_ACCEPT)
+
+#include <ulocks.h>
+
+static ulock_t uslock = NULL;
+
+#define accept_mutex_child_init(x)
+
+static void accept_mutex_init(pool *p)
+{
+ ptrdiff_t old;
+ usptr_t *us;
+
+
+ /* default is 8, allocate enough for all the children plus the parent */
+ if ((old = usconfig(CONF_INITUSERS, HARD_SERVER_LIMIT + 1)) == -1) {
+ perror("usconfig(CONF_INITUSERS)");
+ exit(-1);
+ }
+
+ if ((old = usconfig(CONF_LOCKTYPE, US_NODEBUG)) == -1) {
+ perror("usconfig(CONF_LOCKTYPE)");
+ exit(-1);
+ }
+ if ((old = usconfig(CONF_ARENATYPE, US_SHAREDONLY)) == -1) {
+ perror("usconfig(CONF_ARENATYPE)");
+ exit(-1);
+ }
+ if ((us = usinit("/dev/zero")) == NULL) {
+ perror("usinit");
+ exit(-1);
+ }
+
+ if ((uslock = usnewlock(us)) == NULL) {
+ perror("usnewlock");
+ exit(-1);
+ }
+}
+
+static void accept_mutex_on(void)
+{
+ switch (ussetlock(uslock)) {
+ case 1:
+ /* got lock */
+ break;
+ case 0:
+ fprintf(stderr, "didn't get lock\n");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ case -1:
+ perror("ussetlock");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+}
+
+static void accept_mutex_off(void)
+{
+ if (usunsetlock(uslock) == -1) {
+ perror("usunsetlock");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+}
+
+#elif defined (USE_PTHREAD_SERIALIZED_ACCEPT)
+
+/* This code probably only works on Solaris ... but it works really fast
+ * on Solaris. Note that pthread mutexes are *NOT* released when a task
+ * dies ... the task has to free it itself. So we block signals and
+ * try to be nice about releasing the mutex.
+ */
+
+#include <pthread.h>
+
+static pthread_mutex_t *accept_mutex = (void *)(caddr_t) -1;
+static int have_accept_mutex;
+static sigset_t accept_block_mask;
+static sigset_t accept_previous_mask;
+
+static void accept_mutex_child_cleanup(void *foo)
+{
+ if (accept_mutex != (void *)(caddr_t)-1
+ && have_accept_mutex) {
+ pthread_mutex_unlock(accept_mutex);
+ }
+}
+
+static void accept_mutex_child_init(pool *p)
+{
+ ap_register_cleanup(p, NULL, accept_mutex_child_cleanup,
ap_null_cleanup);
+}
+
+static void accept_mutex_cleanup(void *foo)
+{
+ if (accept_mutex != (void *)(caddr_t)-1
+ && munmap((caddr_t) accept_mutex, sizeof(*accept_mutex))) {
+ perror("munmap");
+ }
+ accept_mutex = (void *)(caddr_t)-1;
+}
+
+static void accept_mutex_init(pool *p)
+{
+ pthread_mutexattr_t mattr;
+ int fd;
+
+ fd = open("/dev/zero", O_RDWR);
+ if (fd == -1) {
+ perror("open(/dev/zero)");
+ exit(APEXIT_INIT);
+ }
+ accept_mutex = (pthread_mutex_t *) mmap((caddr_t) 0,
sizeof(*accept_mutex),
+ PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ if (accept_mutex == (void *) (caddr_t) - 1) {
+ perror("mmap");
+ exit(APEXIT_INIT);
+ }
+ close(fd);
+ if ((errno = pthread_mutexattr_init(&mattr))) {
+ perror("pthread_mutexattr_init");
+ exit(APEXIT_INIT);
+ }
+ if ((errno = pthread_mutexattr_setpshared(&mattr,
+ PTHREAD_PROCESS_SHARED))) {
+ perror("pthread_mutexattr_setpshared");
+ exit(APEXIT_INIT);
+ }
+ if ((errno = pthread_mutex_init(accept_mutex, &mattr))) {
+ perror("pthread_mutex_init");
+ exit(APEXIT_INIT);
+ }
+ sigfillset(&accept_block_mask);
+ sigdelset(&accept_block_mask, SIGHUP);
+ sigdelset(&accept_block_mask, SIGTERM);
+ sigdelset(&accept_block_mask, SIGUSR1);
+ ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup);
+}
+
+static void accept_mutex_on(void)
+{
+ int err;
+
+ if (sigprocmask(SIG_BLOCK, &accept_block_mask, &accept_previous_mask)) {
+ perror("sigprocmask(SIG_BLOCK)");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+ if ((err = pthread_mutex_lock(accept_mutex))) {
+ errno = err;
+ perror("pthread_mutex_lock");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+ have_accept_mutex = 1;
+}
+
+static void accept_mutex_off(void)
+{
+ int err;
+
+ if ((err = pthread_mutex_unlock(accept_mutex))) {
+ errno = err;
+ perror("pthread_mutex_unlock");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+ /* There is a slight race condition right here... if we were to die right
+ * now, we'd do another pthread_mutex_unlock. Now, doing that would let
+ * another process into the mutex. pthread mutexes are designed to be
+ * fast, as such they don't have protection for things like testing if
the
+ * thread owning a mutex is actually unlocking it (or even any way of
+ * testing who owns the mutex).
+ *
+ * If we were to unset have_accept_mutex prior to releasing the mutex
+ * then the race could result in the server unable to serve hits. Doing
+ * it this way means that the server can continue, but an additional
+ * child might be in the critical section ... at least it's still serving
+ * hits.
+ */
+ have_accept_mutex = 0;
+ if (sigprocmask(SIG_SETMASK, &accept_previous_mask, NULL)) {
+ perror("sigprocmask(SIG_SETMASK)");
+ clean_child_exit(1);
+ }
+}
+
+#elif defined (USE_SYSVSEM_SERIALIZED_ACCEPT)
+
+#include <sys/types.h>
+#include <sys/ipc.h>
+#include <sys/sem.h>
+
+#ifdef NEED_UNION_SEMUN
+/* it makes no sense, but this isn't defined on solaris */
+union semun {
+ long val;
+ struct semid_ds *buf;
+ ushort *array;
+};
+
+#endif
+
+static int sem_id = -1;
+static struct sembuf op_on;
+static struct sembuf op_off;
+
+/* We get a random semaphore ... the lame sysv semaphore interface
+ * means we have to be sure to clean this up or else we'll leak
+ * semaphores.
+ */
+static void accept_mutex_cleanup(void *foo)
+{
+ union semun ick;
+
+ if (sem_id < 0)
+ return;
+ /* this is ignored anyhow */
+ ick.val = 0;
+ semctl(sem_id, 0, IPC_RMID, ick);
+}
+
+#define accept_mutex_child_init(x)
+
+static void accept_mutex_init(pool *p)
+{
+ union semun ick;
+ struct semid_ds buf;
+
+ /* acquire the semaphore */
+ sem_id = semget(IPC_PRIVATE, 1, IPC_CREAT | 0600);
+ if (sem_id < 0) {
+ perror("semget");
+ exit(APEXIT_INIT);
+ }
+ ick.val = 1;
+ if (semctl(sem_id, 0, SETVAL, ick) < 0) {
+ perror("semctl(SETVAL)");
+ exit(APEXIT_INIT);
+ }
+ if (!getuid()) {
+ /* restrict it to use only by the appropriate user_id ... not that this
+ * stops CGIs from acquiring it and dinking around with it.
+ */
+ buf.sem_perm.uid = ap_user_id;
+ buf.sem_perm.gid = ap_group_id;
+ buf.sem_perm.mode = 0600;
+ ick.buf = &buf;
+ if (semctl(sem_id, 0, IPC_SET, ick) < 0) {
+ perror("semctl(IPC_SET)");
+ exit(APEXIT_INIT);
+ }
+ }
+ ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup);
+
+ /* pre-initialize these */
+ op_on.sem_num = 0;
+ op_on.sem_op = -1;
+ op_on.sem_flg = SEM_UNDO;
+ op_off.sem_num = 0;
+ op_off.sem_op = 1;
+ op_off.sem_flg = SEM_UNDO;
+}
+
+static void accept_mutex_on(void)
+{
+ if (semop(sem_id, &op_on, 1) < 0) {
+ perror("accept_mutex_on");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+}
+
+static void accept_mutex_off(void)
+{
+ if (semop(sem_id, &op_off, 1) < 0) {
+ perror("accept_mutex_off");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+}
+
+#elif defined(USE_FCNTL_SERIALIZED_ACCEPT)
+static struct flock lock_it;
+static struct flock unlock_it;
+
+static int lock_fd = -1;
+
+#define accept_mutex_child_init(x)
+
+/*
+ * Initialize mutex lock.
+ * Must be safe to call this on a restart.
+ */
+static void accept_mutex_init(pool *p)
+{
+
+ lock_it.l_whence = SEEK_SET; /* from current point */
+ lock_it.l_start = 0; /* -"- */
+ lock_it.l_len = 0; /* until end of file */
+ lock_it.l_type = F_WRLCK; /* set exclusive/write lock */
+ lock_it.l_pid = 0; /* pid not actually interesting
*/
+ unlock_it.l_whence = SEEK_SET; /* from current point */
+ unlock_it.l_start = 0; /* -"- */
+ unlock_it.l_len = 0; /* until end of file */
+ unlock_it.l_type = F_UNLCK; /* set exclusive/write lock */
+ unlock_it.l_pid = 0; /* pid not actually interesting */
+
+ expand_lock_fname(p);
+ lock_fd = ap_popenf(p, ap_lock_fname, O_CREAT | O_WRONLY | O_EXCL, 0644);
+ if (lock_fd == -1) {
+ perror("open");
+ fprintf(stderr, "Cannot open lock file: %s\n", ap_lock_fname);
+ exit(APEXIT_INIT);
+ }
+ unlink(ap_lock_fname);
+}
+
+static void accept_mutex_on(void)
+{
+ int ret;
+
+ while ((ret = fcntl(lock_fd, F_SETLKW, &lock_it)) < 0 && errno == EINTR)
{
+ /* nop */
+ }
+
+ if (ret < 0) {
+ ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf,
+ "fcntl: F_SETLKW: Error getting accept lock, exiting! "
+ "Perhaps you need to use the LockFile directive to place "
+ "your lock file on a local disk!");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+}
+
+static void accept_mutex_off(void)
+{
+ int ret;
+
+ while ((ret = fcntl(lock_fd, F_SETLKW, &unlock_it)) < 0 && errno ==
EINTR) {
+ /* nop */
+ }
+ if (ret < 0) {
+ ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf,
+ "fcntl: F_SETLKW: Error freeing accept lock, exiting! "
+ "Perhaps you need to use the LockFile directive to place "
+ "your lock file on a local disk!");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+}
+
+#elif defined(USE_FLOCK_SERIALIZED_ACCEPT)
+
+static int lock_fd = -1;
+
+static void accept_mutex_cleanup(void *foo)
+{
+ unlink(ap_lock_fname);
+}
+
+/*
+ * Initialize mutex lock.
+ * Done by each child at it's birth
+ */
+static void accept_mutex_child_init(pool *p)
+{
+
+ lock_fd = ap_popenf(p, ap_lock_fname, O_WRONLY, 0600);
+ if (lock_fd == -1) {
+ ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf,
+ "Child cannot open lock file: %s", ap_lock_fname);
+ clean_child_exit(APEXIT_CHILDINIT);
+ }
+}
+
+/*
+ * Initialize mutex lock.
+ * Must be safe to call this on a restart.
+ */
+static void accept_mutex_init(pool *p)
+{
+ expand_lock_fname(p);
+ unlink(ap_lock_fname);
+ lock_fd = ap_popenf(p, ap_lock_fname, O_CREAT | O_WRONLY | O_EXCL, 0600);
+ if (lock_fd == -1) {
+ ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf,
+ "Parent cannot open lock file: %s", ap_lock_fname);
+ exit(APEXIT_INIT);
+ }
+ ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup);
+}
+
+static void accept_mutex_on(void)
+{
+ int ret;
+
+ while ((ret = flock(lock_fd, LOCK_EX)) < 0 && errno == EINTR)
+ continue;
+
+ if (ret < 0) {
+ ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf,
+ "flock: LOCK_EX: Error getting accept lock. Exiting!");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+}
+
+static void accept_mutex_off(void)
+{
+ if (flock(lock_fd, LOCK_UN) < 0) {
+ ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf,
+ "flock: LOCK_UN: Error freeing accept lock. Exiting!");
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+}
+
+#elif defined(USE_OS2SEM_SERIALIZED_ACCEPT)
+
+static HMTX lock_sem = -1;
+
+static void accept_mutex_cleanup(void *foo)
+{
+ DosReleaseMutexSem(lock_sem);
+ DosCloseMutexSem(lock_sem);
+}
+
+/*
+ * Initialize mutex lock.
+ * Done by each child at it's birth
+ */
+static void accept_mutex_child_init(pool *p)
+{
+ int rc = DosOpenMutexSem(NULL, &lock_sem);
+
+ if (rc != 0) {
+ ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, server_conf,
+ "Child cannot open lock semaphore, rc=%d", rc);
+ clean_child_exit(APEXIT_CHILDINIT);
+ }
+}
+
+/*
+ * Initialize mutex lock.
+ * Must be safe to call this on a restart.
+ */
+static void accept_mutex_init(pool *p)
+{
+ int rc = DosCreateMutexSem(NULL, &lock_sem, DC_SEM_SHARED, FALSE);
+
+ if (rc != 0) {
+ ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, server_conf,
+ "Parent cannot create lock semaphore, rc=%d", rc);
+ exit(APEXIT_INIT);
+ }
+
+ ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup);
+}
+
+static void accept_mutex_on(void)
+{
+ int rc = DosRequestMutexSem(lock_sem, SEM_INDEFINITE_WAIT);
+
+ if (rc != 0) {
+ ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, server_conf,
+ "OS2SEM: Error %d getting accept lock. Exiting!", rc);
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+}
+
+static void accept_mutex_off(void)
+{
+ int rc = DosReleaseMutexSem(lock_sem);
+
+ if (rc != 0) {
+ ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, server_conf,
+ "OS2SEM: Error %d freeing accept lock. Exiting!", rc);
+ clean_child_exit(APEXIT_CHILDFATAL);
+ }
+}
+
+#else
+/* Default --- no serialization. Other methods *could* go here,
+ * as #elifs...
+ */
+#if !defined(MULTITHREAD)
+/* Multithreaded systems don't complete between processes for
+ * the sockets. */
+#define NO_SERIALIZED_ACCEPT
+#define accept_mutex_child_init(x)
+#define accept_mutex_init(x)
+#define accept_mutex_on()
+#define accept_mutex_off()
+#endif
+#endif
+
+/*** End of accept serialization code. */
+
+/* On some architectures it's safe to do unserialized accept()s in the single
+ * Listen case. But it's never safe to do it in the case where there's
+ * multiple Listen statements. Define SINGLE_LISTEN_UNSERIALIZED_ACCEPT
+ * when it's safe in the single Listen case. We haven't defined this yet
+ * for the hybrid server. ZZZ
+ */
+#define SAFE_ACCEPT(stmt) do {stmt;} while(0)
+
static void usage(char *bin)
{
char pad[MAX_STRING_LEN];
@@ -1768,7 +2274,10 @@
while (0 < requests_this_child) {
(void) ap_update_child_status(my_pid, my_tid, SERVER_ACCEPTING,
(request_rec *) NULL);
+ /* lock around the accept if necessary */
+ SAFE_ACCEPT(accept_mutex_on());
csd = accept(sd, &sa_client, &len);
+ SAFE_ACCEPT(accept_mutex_off());
(void) ap_update_child_status(my_pid, my_tid, SERVER_QUEUEING,
(request_rec *) NULL);
if (csd >= 0) {
@@ -1955,7 +2464,7 @@
/*stuff to do before we switch id's, so we have permissions.*/
reopen_scoreboard(pchild);
- /* SAFE_ACCEPT(accept_mutex_child_init(pchild));*/
+ SAFE_ACCEPT(accept_mutex_child_init(pchild));
set_group_privs();
