rbb 99/02/16 07:34:04
Modified: pthreads/src/main http_main.c Log: First pass at accept loop serialization. Right now, it is one mutex for the whole server. This should be one mutex per socket, but I just wanted to get the basic code in, I'll clean it up later. Revision Changes Path 1.30 +512 -3 apache-apr/pthreads/src/main/http_main.c Index: http_main.c =================================================================== RCS file: /home/cvs/apache-apr/pthreads/src/main/http_main.c,v retrieving revision 1.29 retrieving revision 1.30 diff -u -r1.29 -r1.30 --- http_main.c 1999/02/16 07:10:27 1.29 +++ http_main.c 1999/02/16 15:34:04 1.30 @@ -247,8 +247,6 @@ } #define tls() ((tls_main_t *) gettls(tls_main_key)) /* ZZZZZ */ - -/* used by accept_loop(), which is serialized */ static listen_rec *head_listener; /* *Non*-shared http_main globals... */ @@ -427,6 +425,514 @@ exit(code); } + +/****** ZZZ this should probably be abstracted to it's own file. ****/ + +#if defined(USE_FCNTL_SERIALIZED_ACCEPT) || defined(USE_FLOCK_SERIALIZED_ACCEPT) +static void expand_lock_fname(pool *p) +{ + /* XXXX possibly bogus cast */ + ap_lock_fname = ap_psprintf(p, "%s.%lu", + ap_server_root_relative(p, ap_lock_fname), (unsigned long)getpid()); +} +#endif + +#if defined (USE_USLOCK_SERIALIZED_ACCEPT) + +#include <ulocks.h> + +static ulock_t uslock = NULL; + +#define accept_mutex_child_init(x) + +static void accept_mutex_init(pool *p) +{ + ptrdiff_t old; + usptr_t *us; + + + /* default is 8, allocate enough for all the children plus the parent */ + if ((old = usconfig(CONF_INITUSERS, HARD_SERVER_LIMIT + 1)) == -1) { + perror("usconfig(CONF_INITUSERS)"); + exit(-1); + } + + if ((old = usconfig(CONF_LOCKTYPE, US_NODEBUG)) == -1) { + perror("usconfig(CONF_LOCKTYPE)"); + exit(-1); + } + if ((old = usconfig(CONF_ARENATYPE, US_SHAREDONLY)) == -1) { + perror("usconfig(CONF_ARENATYPE)"); + exit(-1); + } + if ((us = usinit("/dev/zero")) == NULL) { + perror("usinit"); + exit(-1); + } + + if ((uslock = usnewlock(us)) == NULL) { + perror("usnewlock"); + exit(-1); + } +} + +static void accept_mutex_on(void) +{ + switch (ussetlock(uslock)) { + case 1: + /* got lock */ + break; + case 0: + fprintf(stderr, "didn't get lock\n"); + clean_child_exit(APEXIT_CHILDFATAL); + case -1: + perror("ussetlock"); + clean_child_exit(APEXIT_CHILDFATAL); + } +} + +static void accept_mutex_off(void) +{ + if (usunsetlock(uslock) == -1) { + perror("usunsetlock"); + clean_child_exit(APEXIT_CHILDFATAL); + } +} + +#elif defined (USE_PTHREAD_SERIALIZED_ACCEPT) + +/* This code probably only works on Solaris ... but it works really fast + * on Solaris. Note that pthread mutexes are *NOT* released when a task + * dies ... the task has to free it itself. So we block signals and + * try to be nice about releasing the mutex. + */ + +#include <pthread.h> + +static pthread_mutex_t *accept_mutex = (void *)(caddr_t) -1; +static int have_accept_mutex; +static sigset_t accept_block_mask; +static sigset_t accept_previous_mask; + +static void accept_mutex_child_cleanup(void *foo) +{ + if (accept_mutex != (void *)(caddr_t)-1 + && have_accept_mutex) { + pthread_mutex_unlock(accept_mutex); + } +} + +static void accept_mutex_child_init(pool *p) +{ + ap_register_cleanup(p, NULL, accept_mutex_child_cleanup, ap_null_cleanup); +} + +static void accept_mutex_cleanup(void *foo) +{ + if (accept_mutex != (void *)(caddr_t)-1 + && munmap((caddr_t) accept_mutex, sizeof(*accept_mutex))) { + perror("munmap"); + } + accept_mutex = (void *)(caddr_t)-1; +} + +static void accept_mutex_init(pool *p) +{ + pthread_mutexattr_t mattr; + int fd; + + fd = open("/dev/zero", O_RDWR); + if (fd == -1) { + perror("open(/dev/zero)"); + exit(APEXIT_INIT); + } + accept_mutex = (pthread_mutex_t *) mmap((caddr_t) 0, sizeof(*accept_mutex), + PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (accept_mutex == (void *) (caddr_t) - 1) { + perror("mmap"); + exit(APEXIT_INIT); + } + close(fd); + if ((errno = pthread_mutexattr_init(&mattr))) { + perror("pthread_mutexattr_init"); + exit(APEXIT_INIT); + } + if ((errno = pthread_mutexattr_setpshared(&mattr, + PTHREAD_PROCESS_SHARED))) { + perror("pthread_mutexattr_setpshared"); + exit(APEXIT_INIT); + } + if ((errno = pthread_mutex_init(accept_mutex, &mattr))) { + perror("pthread_mutex_init"); + exit(APEXIT_INIT); + } + sigfillset(&accept_block_mask); + sigdelset(&accept_block_mask, SIGHUP); + sigdelset(&accept_block_mask, SIGTERM); + sigdelset(&accept_block_mask, SIGUSR1); + ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup); +} + +static void accept_mutex_on(void) +{ + int err; + + if (sigprocmask(SIG_BLOCK, &accept_block_mask, &accept_previous_mask)) { + perror("sigprocmask(SIG_BLOCK)"); + clean_child_exit(APEXIT_CHILDFATAL); + } + if ((err = pthread_mutex_lock(accept_mutex))) { + errno = err; + perror("pthread_mutex_lock"); + clean_child_exit(APEXIT_CHILDFATAL); + } + have_accept_mutex = 1; +} + +static void accept_mutex_off(void) +{ + int err; + + if ((err = pthread_mutex_unlock(accept_mutex))) { + errno = err; + perror("pthread_mutex_unlock"); + clean_child_exit(APEXIT_CHILDFATAL); + } + /* There is a slight race condition right here... if we were to die right + * now, we'd do another pthread_mutex_unlock. Now, doing that would let + * another process into the mutex. pthread mutexes are designed to be + * fast, as such they don't have protection for things like testing if the + * thread owning a mutex is actually unlocking it (or even any way of + * testing who owns the mutex). + * + * If we were to unset have_accept_mutex prior to releasing the mutex + * then the race could result in the server unable to serve hits. Doing + * it this way means that the server can continue, but an additional + * child might be in the critical section ... at least it's still serving + * hits. + */ + have_accept_mutex = 0; + if (sigprocmask(SIG_SETMASK, &accept_previous_mask, NULL)) { + perror("sigprocmask(SIG_SETMASK)"); + clean_child_exit(1); + } +} + +#elif defined (USE_SYSVSEM_SERIALIZED_ACCEPT) + +#include <sys/types.h> +#include <sys/ipc.h> +#include <sys/sem.h> + +#ifdef NEED_UNION_SEMUN +/* it makes no sense, but this isn't defined on solaris */ +union semun { + long val; + struct semid_ds *buf; + ushort *array; +}; + +#endif + +static int sem_id = -1; +static struct sembuf op_on; +static struct sembuf op_off; + +/* We get a random semaphore ... the lame sysv semaphore interface + * means we have to be sure to clean this up or else we'll leak + * semaphores. + */ +static void accept_mutex_cleanup(void *foo) +{ + union semun ick; + + if (sem_id < 0) + return; + /* this is ignored anyhow */ + ick.val = 0; + semctl(sem_id, 0, IPC_RMID, ick); +} + +#define accept_mutex_child_init(x) + +static void accept_mutex_init(pool *p) +{ + union semun ick; + struct semid_ds buf; + + /* acquire the semaphore */ + sem_id = semget(IPC_PRIVATE, 1, IPC_CREAT | 0600); + if (sem_id < 0) { + perror("semget"); + exit(APEXIT_INIT); + } + ick.val = 1; + if (semctl(sem_id, 0, SETVAL, ick) < 0) { + perror("semctl(SETVAL)"); + exit(APEXIT_INIT); + } + if (!getuid()) { + /* restrict it to use only by the appropriate user_id ... not that this + * stops CGIs from acquiring it and dinking around with it. + */ + buf.sem_perm.uid = ap_user_id; + buf.sem_perm.gid = ap_group_id; + buf.sem_perm.mode = 0600; + ick.buf = &buf; + if (semctl(sem_id, 0, IPC_SET, ick) < 0) { + perror("semctl(IPC_SET)"); + exit(APEXIT_INIT); + } + } + ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup); + + /* pre-initialize these */ + op_on.sem_num = 0; + op_on.sem_op = -1; + op_on.sem_flg = SEM_UNDO; + op_off.sem_num = 0; + op_off.sem_op = 1; + op_off.sem_flg = SEM_UNDO; +} + +static void accept_mutex_on(void) +{ + if (semop(sem_id, &op_on, 1) < 0) { + perror("accept_mutex_on"); + clean_child_exit(APEXIT_CHILDFATAL); + } +} + +static void accept_mutex_off(void) +{ + if (semop(sem_id, &op_off, 1) < 0) { + perror("accept_mutex_off"); + clean_child_exit(APEXIT_CHILDFATAL); + } +} + +#elif defined(USE_FCNTL_SERIALIZED_ACCEPT) +static struct flock lock_it; +static struct flock unlock_it; + +static int lock_fd = -1; + +#define accept_mutex_child_init(x) + +/* + * Initialize mutex lock. + * Must be safe to call this on a restart. + */ +static void accept_mutex_init(pool *p) +{ + + lock_it.l_whence = SEEK_SET; /* from current point */ + lock_it.l_start = 0; /* -"- */ + lock_it.l_len = 0; /* until end of file */ + lock_it.l_type = F_WRLCK; /* set exclusive/write lock */ + lock_it.l_pid = 0; /* pid not actually interesting */ + unlock_it.l_whence = SEEK_SET; /* from current point */ + unlock_it.l_start = 0; /* -"- */ + unlock_it.l_len = 0; /* until end of file */ + unlock_it.l_type = F_UNLCK; /* set exclusive/write lock */ + unlock_it.l_pid = 0; /* pid not actually interesting */ + + expand_lock_fname(p); + lock_fd = ap_popenf(p, ap_lock_fname, O_CREAT | O_WRONLY | O_EXCL, 0644); + if (lock_fd == -1) { + perror("open"); + fprintf(stderr, "Cannot open lock file: %s\n", ap_lock_fname); + exit(APEXIT_INIT); + } + unlink(ap_lock_fname); +} + +static void accept_mutex_on(void) +{ + int ret; + + while ((ret = fcntl(lock_fd, F_SETLKW, &lock_it)) < 0 && errno == EINTR) { + /* nop */ + } + + if (ret < 0) { + ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf, + "fcntl: F_SETLKW: Error getting accept lock, exiting! " + "Perhaps you need to use the LockFile directive to place " + "your lock file on a local disk!"); + clean_child_exit(APEXIT_CHILDFATAL); + } +} + +static void accept_mutex_off(void) +{ + int ret; + + while ((ret = fcntl(lock_fd, F_SETLKW, &unlock_it)) < 0 && errno == EINTR) { + /* nop */ + } + if (ret < 0) { + ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf, + "fcntl: F_SETLKW: Error freeing accept lock, exiting! " + "Perhaps you need to use the LockFile directive to place " + "your lock file on a local disk!"); + clean_child_exit(APEXIT_CHILDFATAL); + } +} + +#elif defined(USE_FLOCK_SERIALIZED_ACCEPT) + +static int lock_fd = -1; + +static void accept_mutex_cleanup(void *foo) +{ + unlink(ap_lock_fname); +} + +/* + * Initialize mutex lock. + * Done by each child at it's birth + */ +static void accept_mutex_child_init(pool *p) +{ + + lock_fd = ap_popenf(p, ap_lock_fname, O_WRONLY, 0600); + if (lock_fd == -1) { + ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf, + "Child cannot open lock file: %s", ap_lock_fname); + clean_child_exit(APEXIT_CHILDINIT); + } +} + +/* + * Initialize mutex lock. + * Must be safe to call this on a restart. + */ +static void accept_mutex_init(pool *p) +{ + expand_lock_fname(p); + unlink(ap_lock_fname); + lock_fd = ap_popenf(p, ap_lock_fname, O_CREAT | O_WRONLY | O_EXCL, 0600); + if (lock_fd == -1) { + ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf, + "Parent cannot open lock file: %s", ap_lock_fname); + exit(APEXIT_INIT); + } + ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup); +} + +static void accept_mutex_on(void) +{ + int ret; + + while ((ret = flock(lock_fd, LOCK_EX)) < 0 && errno == EINTR) + continue; + + if (ret < 0) { + ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf, + "flock: LOCK_EX: Error getting accept lock. Exiting!"); + clean_child_exit(APEXIT_CHILDFATAL); + } +} + +static void accept_mutex_off(void) +{ + if (flock(lock_fd, LOCK_UN) < 0) { + ap_log_error(APLOG_MARK, APLOG_EMERG, server_conf, + "flock: LOCK_UN: Error freeing accept lock. Exiting!"); + clean_child_exit(APEXIT_CHILDFATAL); + } +} + +#elif defined(USE_OS2SEM_SERIALIZED_ACCEPT) + +static HMTX lock_sem = -1; + +static void accept_mutex_cleanup(void *foo) +{ + DosReleaseMutexSem(lock_sem); + DosCloseMutexSem(lock_sem); +} + +/* + * Initialize mutex lock. + * Done by each child at it's birth + */ +static void accept_mutex_child_init(pool *p) +{ + int rc = DosOpenMutexSem(NULL, &lock_sem); + + if (rc != 0) { + ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, server_conf, + "Child cannot open lock semaphore, rc=%d", rc); + clean_child_exit(APEXIT_CHILDINIT); + } +} + +/* + * Initialize mutex lock. + * Must be safe to call this on a restart. + */ +static void accept_mutex_init(pool *p) +{ + int rc = DosCreateMutexSem(NULL, &lock_sem, DC_SEM_SHARED, FALSE); + + if (rc != 0) { + ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, server_conf, + "Parent cannot create lock semaphore, rc=%d", rc); + exit(APEXIT_INIT); + } + + ap_register_cleanup(p, NULL, accept_mutex_cleanup, ap_null_cleanup); +} + +static void accept_mutex_on(void) +{ + int rc = DosRequestMutexSem(lock_sem, SEM_INDEFINITE_WAIT); + + if (rc != 0) { + ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, server_conf, + "OS2SEM: Error %d getting accept lock. Exiting!", rc); + clean_child_exit(APEXIT_CHILDFATAL); + } +} + +static void accept_mutex_off(void) +{ + int rc = DosReleaseMutexSem(lock_sem); + + if (rc != 0) { + ap_log_error(APLOG_MARK, APLOG_NOERRNO|APLOG_EMERG, server_conf, + "OS2SEM: Error %d freeing accept lock. Exiting!", rc); + clean_child_exit(APEXIT_CHILDFATAL); + } +} + +#else +/* Default --- no serialization. Other methods *could* go here, + * as #elifs... + */ +#if !defined(MULTITHREAD) +/* Multithreaded systems don't complete between processes for + * the sockets. */ +#define NO_SERIALIZED_ACCEPT +#define accept_mutex_child_init(x) +#define accept_mutex_init(x) +#define accept_mutex_on() +#define accept_mutex_off() +#endif +#endif + +/*** End of accept serialization code. */ + +/* On some architectures it's safe to do unserialized accept()s in the single + * Listen case. But it's never safe to do it in the case where there's + * multiple Listen statements. Define SINGLE_LISTEN_UNSERIALIZED_ACCEPT + * when it's safe in the single Listen case. We haven't defined this yet + * for the hybrid server. ZZZ + */ +#define SAFE_ACCEPT(stmt) do {stmt;} while(0) + static void usage(char *bin) { char pad[MAX_STRING_LEN]; @@ -1768,7 +2274,10 @@ while (0 < requests_this_child) { (void) ap_update_child_status(my_pid, my_tid, SERVER_ACCEPTING, (request_rec *) NULL); + /* lock around the accept if necessary */ + SAFE_ACCEPT(accept_mutex_on()); csd = accept(sd, &sa_client, &len); + SAFE_ACCEPT(accept_mutex_off()); (void) ap_update_child_status(my_pid, my_tid, SERVER_QUEUEING, (request_rec *) NULL); if (csd >= 0) { @@ -1955,7 +2464,7 @@ /*stuff to do before we switch id's, so we have permissions.*/ reopen_scoreboard(pchild); - /* SAFE_ACCEPT(accept_mutex_child_init(pchild));*/ + SAFE_ACCEPT(accept_mutex_child_init(pchild)); set_group_privs();