Only in httpd-trunk.new: httpd_trunk_so_reuseport_v2.patch diff -ru httpd-trunk/include/ap_listen.h httpd-trunk.new/include/ap_listen.h --- httpd-trunk/include/ap_listen.h 2014-10-10 10:39:28.381416846 -0400 +++ httpd-trunk.new/include/ap_listen.h 2014-10-10 10:53:55.299402988 -0400 @@ -78,13 +78,7 @@ */ AP_DECLARE_DATA extern ap_listen_rec *ap_listeners; -AP_DECLARE_DATA extern ap_listen_rec **mpm_listen; - -AP_DECLARE_DATA extern int enable_default_listener; - -AP_DECLARE_DATA extern int num_buckets; - -AP_DECLARE_DATA extern int have_so_reuseport; +AP_DECLARE_DATA extern int ap_have_so_reuseport; /** * Setup all of the defaults for the listener list @@ -102,10 +96,17 @@ /**This function duplicates ap_listeners. * @param s The global server_rec * @param p The config pool - * @param num_buckets The total number of listener buckets. -**/ -AP_DECLARE(apr_status_t) ap_duplicate_listeners(server_rec *s, apr_pool_t *p, int num_buckets); - + * @param mpm_listener The array that contains listeners + * @param num_buckets The total number of listener buckets + * @param retain_num_buckets The flag to keep the previous num_buckets + * (for graceful restart only) + * @param one_process The flag to set ap_num_buckets = 1 (for one_process case). + */ +AP_DECLARE(apr_status_t) ap_duplicate_listeners(server_rec *s, apr_pool_t *p, + ap_listen_rec ***mpm_listener, + int *num_buckets, + int retain_num_buckets, + int one_process); /** * Loop through the global ap_listen_rec list and close each of the sockets. @@ -113,6 +114,19 @@ AP_DECLARE_NONSTD(void) ap_close_listeners(void); /** + * Close duplicated listeners if ap_have_reuseport is enabled + * @param mpm_listener The array of listeners + * @param num_listeners The number of listeners in the array. + */ +AP_DECLARE_NONSTD(void) ap_close_duplicated_listeners(ap_listen_rec **mpm_listener, + int num_listeners); +/** + * Close the listener in child process + * @param listener The listener to close. + */ +AP_DECLARE_NONSTD(void) ap_close_child_listener(ap_listen_rec *listener); + +/** * FIXMEDOC */ AP_DECLARE_NONSTD(int) ap_close_selected_listeners(ap_slave_t *); diff -ru httpd-trunk/server/listen.c httpd-trunk.new/server/listen.c --- httpd-trunk/server/listen.c 2014-10-10 10:39:26.806416871 -0400 +++ httpd-trunk.new/server/listen.c 2014-10-10 12:01:15.616338405 -0400 @@ -28,6 +28,11 @@ #include "http_log.h" #include "mpm_common.h" +#include +#if APR_HAVE_UNISTD_H +#include +#endif + #ifdef HAVE_SYSTEMD #include #endif @@ -38,10 +43,7 @@ AP_DECLARE_DATA ap_listen_rec *ap_listeners = NULL; -AP_DECLARE_DATA ap_listen_rec **mpm_listen = NULL; -AP_DECLARE_DATA int enable_default_listener = 1; -AP_DECLARE_DATA int num_buckets = 1; -AP_DECLARE_DATA int have_so_reuseport = 0; +AP_DECLARE_DATA int ap_have_so_reuseport = 0; static ap_listen_rec *old_listeners; static int ap_listenbacklog; @@ -136,7 +138,7 @@ if (setsockopt(thesock, SOL_SOCKET, SO_REUSEPORT, (void *)&one, sizeof(int)) < 0) { /* defined by not valid? */ if (errno == ENOPROTOOPT) { - have_so_reuseport = 0; + ap_have_so_reuseport = 0; } /* Check if SO_REUSEPORT is supported by the running Linux Kernel.*/ else { ap_log_perror(APLOG_MARK, APLOG_CRIT, stat, p, APLOGNO(02638) @@ -147,7 +149,7 @@ } } else { - have_so_reuseport = 1; + ap_have_so_reuseport = 1; } } #endif @@ -207,7 +209,7 @@ #endif server->sd = s; - server->active = enable_default_listener; + server->active = 1; server->accept_func = NULL; @@ -596,7 +598,7 @@ } } #endif - if (make_sock(pool, lr, enable_default_listener) == APR_SUCCESS) { + if (make_sock(pool, lr, 1) == APR_SUCCESS) { ++num_open; } else { @@ -759,14 +761,33 @@ } AP_DECLARE(apr_status_t) ap_duplicate_listeners(server_rec *s, apr_pool_t *p, - int num_buckets) { + ap_listen_rec ***mpm_listener, + int *num_buckets, + int retain_num_buckets, + int one_process) { int i; apr_status_t stat; int use_nonblock = 0; ap_listen_rec *lr; - mpm_listen = apr_palloc(p, sizeof(ap_listen_rec*) * num_buckets); - for (i = 0; i < num_buckets; i++) { + if (one_process) { + *num_buckets = 1; + } + else if (!retain_num_buckets) { + *num_buckets = 1; +#ifdef _SC_NPROCESSORS_ONLN + if (ap_have_so_reuseport) { + int num_online_cores = sysconf(_SC_NPROCESSORS_ONLN); + if (num_online_cores > 8) { + *num_buckets = num_online_cores / 8; + } + } +#endif + } + + *mpm_listener = apr_palloc(p, sizeof(ap_listen_rec*) * (*num_buckets)); + (*mpm_listener)[0] = ap_listeners; + for (i = 1; i < *num_buckets; i++) { ap_listen_rec *last = NULL; lr = ap_listeners; while (lr) { @@ -816,7 +837,7 @@ ap_apply_accept_filter(p, duplr, s); if (last == NULL) { - mpm_listen[i] = last = duplr; + (*mpm_listener)[i] = last = duplr; } else { last->next = duplr; @@ -831,15 +852,38 @@ AP_DECLARE_NONSTD(void) ap_close_listeners(void) { ap_listen_rec *lr; + for (lr = ap_listeners; lr; lr = lr->next) { + apr_socket_close(lr->sd); + lr->active = 0; + } +} + +AP_DECLARE_NONSTD(void) ap_close_duplicated_listeners(ap_listen_rec** mpm_listener, + int num_listeners) +{ + ap_listen_rec *lr; int i; - for (i = 0; i < num_buckets; i++) { - for (lr = mpm_listen[i]; lr; lr = lr->next) { + + for (i = 1; i < num_listeners; i++) { + for (lr = mpm_listener[i]; lr; lr = lr->next) { apr_socket_close(lr->sd); lr->active = 0; } } } +AP_DECLARE_NONSTD(void) ap_close_child_listener(ap_listen_rec* listener) +{ + ap_listen_rec *lr; + int i; + + for (lr = listener; lr; lr = lr->next) { + apr_socket_close(lr->sd); + lr->active = 0; + } +} + + AP_DECLARE_NONSTD(int) ap_close_selected_listeners(ap_slave_t *slave) { ap_listen_rec *lr; diff -ru httpd-trunk/server/log.c httpd-trunk.new/server/log.c --- httpd-trunk/server/log.c 2014-10-10 10:39:26.811416871 -0400 +++ httpd-trunk.new/server/log.c 2014-10-10 12:05:13.316334606 -0400 @@ -1492,9 +1492,8 @@ AP_DECLARE(void) ap_log_common(server_rec *s) { ap_log_error(APLOG_MARK, APLOG_DEBUG , 0, s, APLOGNO(02639) - "Using SO_REUSEPORT: %s (%d)", - have_so_reuseport ? "yes" : "no", - num_buckets); + "Using SO_REUSEPORT: %s ", + ap_have_so_reuseport ? "yes" : "no"); } AP_DECLARE(void) ap_remove_pid(apr_pool_t *p, const char *rel_fname) diff -ru httpd-trunk/server/mpm/event/event.c httpd-trunk.new/server/mpm/event/event.c --- httpd-trunk/server/mpm/event/event.c 2014-10-10 10:39:26.597416874 -0400 +++ httpd-trunk.new/server/mpm/event/event.c 2014-10-10 12:10:01.367330001 -0400 @@ -192,6 +192,8 @@ static fd_queue_t *worker_queue; static fd_queue_info_t *worker_queue_info; static int mpm_state = AP_MPMQ_STARTING; +static ap_listen_rec **event_listener; +static int ap_num_buckets; static apr_thread_mutex_t *timeout_mutex; @@ -361,12 +363,16 @@ } event_retained_data; static event_retained_data *retained; +typedef struct event_child_data { + ap_pod_t *pod; + ap_listen_rec *listener; + int bucket; +} event_child_data; +static event_child_data *child; + #define ID_FROM_CHILD_THREAD(c, t) ((c * thread_limit) + t) static ap_pod_t **pod; -static ap_pod_t *child_pod; -static ap_listen_rec *child_listen; - /* The event MPM respects a couple of runtime flags that can aid * in debugging. Setting the -DNO_DETACH flag will prevent the root process * from detaching from its controlling terminal. Additionally, setting @@ -1215,7 +1221,7 @@ if (!*closed) { int i; disable_listensocks(process_slot); - ap_close_listeners(); + ap_close_child_listener(child->listener); *closed = 1; dying = 1; ap_scoreboard_image->parent[process_slot].quiescing = 1; @@ -1289,7 +1295,7 @@ TO_QUEUE_INIT(short_linger_q); listener_pollfd = apr_palloc(p, sizeof(apr_pollfd_t) * num_listensocks); - for (lr = child_listen; lr != NULL; lr = lr->next, i++) { + for (lr = child->listener; lr != NULL; lr = lr->next, i++) { apr_pollfd_t *pfd; AP_DEBUG_ASSERT(i < num_listensocks); pfd = &listener_pollfd[i]; @@ -2409,7 +2415,7 @@ } } -static void child_main(int child_num_arg, int child_bucket) +static void child_main(int child_num_arg) { apr_thread_t **threads; apr_status_t rv; @@ -2427,13 +2433,13 @@ ap_fatal_signal_child_setup(ap_server_conf); apr_pool_create(&pchild, pconf); - child_listen = mpm_listen[child_bucket]; - child_pod = pod[child_bucket]; + child->listener = event_listener[child->bucket]; + child->pod = pod[child->bucket]; /* close unused listeners and pods */ - for (i = 0; i < num_buckets; i++) { - if (i != child_bucket) { - lr = mpm_listen[i]; + for (i = 0; i < ap_num_buckets; i++) { + if (i != child->bucket) { + lr = event_listener[i]; while(lr) { apr_socket_close(lr->sd); lr->active = 0; @@ -2553,7 +2559,7 @@ apr_signal(SIGTERM, dummy_signal_handler); /* Watch for any messages from the parent over the POD */ while (1) { - rv = ap_mpm_podx_check(child_pod); + rv = ap_mpm_podx_check(child->pod); if (rv == AP_MPM_PODX_NORESTART) { /* see if termination was triggered while we slept */ switch (terminate_mode) { @@ -2599,10 +2605,12 @@ retained->max_daemons_limit = slot + 1; } + child->bucket = bucket; + if (one_process) { set_signals(); event_note_child_started(slot, getpid()); - child_main(0, 0); + child_main(0); /* NOTREACHED */ ap_assert(0); return -1; @@ -2642,7 +2650,7 @@ RAISE_SIGSTOP(MAKE_CHILD); apr_signal(SIGTERM, just_die); - child_main(slot, bucket); + child_main(slot); /* NOTREACHED */ ap_assert(0); return -1; @@ -2671,7 +2679,7 @@ if (ap_scoreboard_image->parent[i].pid != 0) { continue; } - if (make_child(ap_server_conf, i, i % num_buckets) < 0) { + if (make_child(ap_server_conf, i, i % ap_num_buckets) < 0) { break; } --number_to_start; @@ -2748,7 +2756,7 @@ active_thread_count += child_threads_active; if (any_dead_threads && totally_free_length < retained->idle_spawn_rate[child_bucket] - && free_length < MAX_SPAWN_RATE / num_buckets + && free_length < MAX_SPAWN_RATE / ap_num_buckets && (!ps->pid /* no process in the slot */ || ps->quiescing)) { /* or at least one is going away */ if (all_dead_threads) { @@ -2804,12 +2812,12 @@ retained->max_daemons_limit = last_non_dead + 1; - if (idle_thread_count > max_spare_threads / num_buckets) { + if (idle_thread_count > max_spare_threads / ap_num_buckets) { /* Kill off one child */ ap_mpm_podx_signal(pod[child_bucket], AP_MPM_PODX_GRACEFUL); retained->idle_spawn_rate[child_bucket] = 1; } - else if (idle_thread_count < min_spare_threads / num_buckets) { + else if (idle_thread_count < min_spare_threads / ap_num_buckets) { /* terminate the free list */ if (free_length == 0) { /* scoreboard is full, can't fork */ @@ -2852,7 +2860,7 @@ --retained->hold_off_on_exponential_spawning; } else if (retained->idle_spawn_rate[child_bucket] < - MAX_SPAWN_RATE / num_buckets) { + MAX_SPAWN_RATE / ap_num_buckets) { retained->idle_spawn_rate[child_bucket] *= 2; } } @@ -2936,7 +2944,7 @@ if (processed_status == APEXIT_CHILDSICK && old_gen == retained->my_generation) { /* resource shortage, minimize the fork rate */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { retained->idle_spawn_rate[i] = 1; } } @@ -2977,7 +2985,7 @@ continue; } - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { perform_idle_server_maintenance(i); } } @@ -2991,6 +2999,8 @@ ap_log_pid(pconf, ap_pid_fname); + child = apr_palloc(_pconf, sizeof(*child)); + if (!retained->is_graceful) { if (ap_run_pre_mpm(s->process->pool, SB_SHARED) != OK) { mpm_state = AP_MPMQ_STOPPING; @@ -3006,10 +3016,10 @@ set_signals(); /* Don't thrash... */ - if (min_spare_threads < threads_per_child * num_buckets) - min_spare_threads = threads_per_child * num_buckets; - if (max_spare_threads < min_spare_threads + threads_per_child * num_buckets) - max_spare_threads = min_spare_threads + threads_per_child * num_buckets; + if (min_spare_threads < threads_per_child * ap_num_buckets) + min_spare_threads = threads_per_child * ap_num_buckets; + if (max_spare_threads < min_spare_threads + threads_per_child * ap_num_buckets) + max_spare_threads = min_spare_threads + threads_per_child * ap_num_buckets; /* If we're doing a graceful_restart then we're going to see a lot * of children exiting immediately when we get into the main loop @@ -3051,7 +3061,7 @@ /* Time to shut down: * Kill child processes, tell them to call child_exit, etc... */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); } ap_reclaim_child_processes(1, /* Start with SIGTERM */ @@ -3074,7 +3084,10 @@ /* Close our listeners, and then ask our children to do same */ ap_close_listeners(); - for (i = 0; i < num_buckets; i++) { + if (ap_have_so_reuseport) { + ap_close_duplicated_listeners(event_listener, ap_num_buckets); + } + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_GRACEFUL); } ap_relieve_child_processes(event_note_child_killed); @@ -3116,7 +3129,7 @@ * way, try and make sure that all of our processes are * really dead. */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); } ap_reclaim_child_processes(1, event_note_child_killed); @@ -3144,7 +3157,7 @@ AP_SIG_GRACEFUL_STRING " received. Doing graceful restart"); /* wake up the children...time to die. But we'll have more soon */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_GRACEFUL); } @@ -3158,7 +3171,7 @@ * and a SIGHUP, we may as well use the same signal, because some user * pthreads are stealing signals from us left and right. */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); } @@ -3190,7 +3203,6 @@ level_flags |= APLOG_STARTUP; } - enable_default_listener = 0; if ((num_listensocks = ap_setup_listeners(ap_server_conf)) < 1) { ap_log_error(APLOG_MARK, APLOG_ALERT | level_flags, 0, (startup ? NULL : s), @@ -3198,13 +3210,13 @@ return DONE; } - enable_default_listener = 1; - ap_duplicate_listeners(ap_server_conf, pconf, num_buckets); + ap_duplicate_listeners(ap_server_conf, pconf, &event_listener, + &ap_num_buckets, retained->is_graceful, one_process); - pod = apr_palloc(pconf, sizeof(ap_pod_t *) * num_buckets); + pod = apr_palloc(pconf, sizeof(ap_pod_t *) * ap_num_buckets); if (!one_process) { - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { if ((rv = ap_mpm_podx_open(pconf, &pod[i]))) { ap_log_error(APLOG_MARK, APLOG_CRIT | level_flags, rv, (startup ? NULL : s), @@ -3246,17 +3258,6 @@ retained = ap_retained_data_create(userdata_key, sizeof(*retained)); retained->max_daemons_limit = -1; } - if (!retained->is_graceful) { - num_buckets = 1; -#ifdef _SC_NPROCESSORS_ONLN - if (have_so_reuseport) { - int num_online_cores = sysconf(_SC_NPROCESSORS_ONLN); - if (num_online_cores > 8) { - num_buckets = num_online_cores / 8; - } - } -#endif - } ++retained->module_loads; if (retained->module_loads == 2) { /* test for correct operation of fdqueue */ @@ -3269,8 +3270,8 @@ "atomics not working as expected - add32 of negative number"); return HTTP_INTERNAL_SERVER_ERROR; } - retained->idle_spawn_rate = apr_palloc(pconf, sizeof(int) * num_buckets); - for (i = 0; i< num_buckets; i++) { + retained->idle_spawn_rate = apr_palloc(pconf, sizeof(int) * ap_num_buckets); + for (i = 0; i< ap_num_buckets; i++) { retained->idle_spawn_rate[i] = 1; } rv = apr_pollset_create(&event_pollset, 1, plog, @@ -3510,11 +3511,11 @@ } ap_daemons_limit = server_limit; } - else if (ap_daemons_limit < num_buckets) { - /* Don't thrash since num_buckets depends on + else if (ap_daemons_limit < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - ap_daemons_limit = num_buckets; + ap_daemons_limit = ap_num_buckets; } /* ap_daemons_to_start > ap_daemons_limit checked in ap_mpm_run() */ @@ -3530,11 +3531,11 @@ } ap_daemons_to_start = 1; } - if (ap_daemons_to_start < num_buckets) { - /* Don't thrash since num_buckets depends on + if (ap_daemons_to_start < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - ap_daemons_to_start = num_buckets; + ap_daemons_to_start = ap_num_buckets; } if (min_spare_threads < 1) { @@ -3553,11 +3554,11 @@ } min_spare_threads = 1; } - if (min_spare_threads < num_buckets) { - /* Don't thrash since num_buckets depends on + if (min_spare_threads < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - min_spare_threads = num_buckets; + min_spare_threads = ap_num_buckets; } /* max_spare_threads < min_spare_threads + threads_per_child diff -ru httpd-trunk/server/mpm/eventopt/eventopt.c httpd-trunk.new/server/mpm/eventopt/eventopt.c --- httpd-trunk/server/mpm/eventopt/eventopt.c 2014-10-10 10:39:26.477416876 -0400 +++ httpd-trunk.new/server/mpm/eventopt/eventopt.c 2014-10-10 12:10:35.493329456 -0400 @@ -346,11 +346,18 @@ } event_retained_data; static event_retained_data *retained; +typedef struct event_child_data { + ap_pod_t *pod; + ap_listen_rec *listener; + int bucket; +} event_child_data; +static event_child_data *child; + #define ID_FROM_CHILD_THREAD(c, t) ((c * thread_limit) + t) static ap_pod_t **pod; -static ap_pod_t *child_pod; -static ap_listen_rec *child_listen; +static int ap_num_buckets; +static ap_listen_rec **event_listener; /* The eventopt MPM respects a couple of runtime flags that can aid * in debugging. Setting the -DNO_DETACH flag will prevent the root process @@ -1176,7 +1183,7 @@ if (!*closed) { int i; disable_listensocks(process_slot); - ap_close_listeners(); + ap_close_child_listener(child->listener); *closed = 1; dying = 1; ap_scoreboard_image->parent[process_slot].quiescing = 1; @@ -1250,7 +1257,7 @@ TO_QUEUE_INIT(short_linger_q); listener_pollfd = apr_palloc(p, sizeof(apr_pollfd_t) * num_listensocks); - for (lr = child_listen; lr != NULL; lr = lr->next, i++) { + for (lr = child->listener; lr != NULL; lr = lr->next, i++) { apr_pollfd_t *pfd; AP_DEBUG_ASSERT(i < num_listensocks); pfd = &listener_pollfd[i]; @@ -2231,7 +2238,7 @@ } } -static void child_main(int child_num_arg, int child_bucket) +static void child_main(int child_num_arg) { apr_thread_t **threads; apr_status_t rv; @@ -2249,13 +2256,13 @@ ap_fatal_signal_child_setup(ap_server_conf); apr_pool_create(&pchild, pconf); - child_listen = mpm_listen[child_bucket]; - child_pod = pod[child_bucket]; + child->listener = event_listener[child->bucket]; + child->pod = pod[child->bucket]; /* close unused listeners and pods */ - for (i = 0; i < num_buckets; i++) { - if (i != child_bucket) { - lr = mpm_listen[i]; + for (i = 0; i < ap_num_buckets; i++) { + if (i != child->bucket) { + lr = event_listener[i]; while(lr) { apr_socket_close(lr->sd); lr->active = 0; @@ -2375,7 +2382,7 @@ apr_signal(SIGTERM, dummy_signal_handler); /* Watch for any messages from the parent over the POD */ while (1) { - rv = ap_mpm_podx_check(child_pod); + rv = ap_mpm_podx_check(child->pod); if (rv == AP_MPM_PODX_NORESTART) { /* see if termination was triggered while we slept */ switch (terminate_mode) { @@ -2420,11 +2427,12 @@ if (slot + 1 > retained->max_daemons_limit) { retained->max_daemons_limit = slot + 1; } + child->bucket = bucket; if (one_process) { set_signals(); event_note_child_started(slot, getpid()); - child_main(0, 0); + child_main(0); /* NOTREACHED */ ap_assert(0); return -1; @@ -2464,7 +2472,7 @@ RAISE_SIGSTOP(MAKE_CHILD); apr_signal(SIGTERM, just_die); - child_main(slot, bucket); + child_main(slot); /* NOTREACHED */ ap_assert(0); return -1; @@ -2493,7 +2501,7 @@ if (ap_scoreboard_image->parent[i].pid != 0) { continue; } - if (make_child(ap_server_conf, i, i % num_buckets) < 0) { + if (make_child(ap_server_conf, i, i % ap_num_buckets) < 0) { break; } --number_to_start; @@ -2570,7 +2578,7 @@ active_thread_count += child_threads_active; if (any_dead_threads && totally_free_length < retained->idle_spawn_rate[child_bucket] - && free_length < MAX_SPAWN_RATE / num_buckets + && free_length < MAX_SPAWN_RATE / ap_num_buckets && (!ps->pid /* no process in the slot */ || ps->quiescing)) { /* or at least one is going away */ if (all_dead_threads) { @@ -2625,12 +2633,12 @@ retained->max_daemons_limit = last_non_dead + 1; - if (idle_thread_count > max_spare_threads / num_buckets) { + if (idle_thread_count > max_spare_threads / ap_num_buckets) { /* Kill off one child */ ap_mpm_podx_signal(pod[child_bucket], AP_MPM_PODX_GRACEFUL); retained->idle_spawn_rate[child_bucket] = 1; } - else if (idle_thread_count < min_spare_threads / num_buckets) { + else if (idle_thread_count < min_spare_threads / ap_num_buckets) { /* terminate the free list */ if (free_length == 0) { /* scoreboard is full, can't fork */ @@ -2673,7 +2681,7 @@ --retained->hold_off_on_exponential_spawning; } else if (retained->idle_spawn_rate[child_bucket] - < MAX_SPAWN_RATE / num_buckets) { + < MAX_SPAWN_RATE / ap_num_buckets) { retained->idle_spawn_rate[child_bucket] *= 2; } } @@ -2757,7 +2765,7 @@ if (processed_status == APEXIT_CHILDSICK && old_gen == retained->my_generation) { /* resource shortage, minimize the fork rate */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { retained->idle_spawn_rate[i] = 1; } } @@ -2798,7 +2806,7 @@ continue; } - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { perform_idle_server_maintenance(i); } } @@ -2812,6 +2820,8 @@ ap_log_pid(pconf, ap_pid_fname); + child = apr_palloc(_pconf, sizeof(*child)); + if (!retained->is_graceful) { if (ap_run_pre_mpm(s->process->pool, SB_SHARED) != OK) { mpm_state = AP_MPMQ_STOPPING; @@ -2827,10 +2837,10 @@ set_signals(); /* Don't thrash... */ - if (min_spare_threads < threads_per_child * num_buckets) - min_spare_threads = threads_per_child * num_buckets; - if (max_spare_threads < min_spare_threads + threads_per_child * num_buckets) - max_spare_threads = min_spare_threads + threads_per_child * num_buckets; + if (min_spare_threads < threads_per_child * ap_num_buckets) + min_spare_threads = threads_per_child * ap_num_buckets; + if (max_spare_threads < min_spare_threads + threads_per_child * ap_num_buckets) + max_spare_threads = min_spare_threads + threads_per_child * ap_num_buckets; /* If we're doing a graceful_restart then we're going to see a lot * of children exiting immediately when we get into the main loop @@ -2872,7 +2882,7 @@ /* Time to shut down: * Kill child processes, tell them to call child_exit, etc... */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); } ap_reclaim_child_processes(1, /* Start with SIGTERM */ @@ -2895,7 +2905,10 @@ /* Close our listeners, and then ask our children to do same */ ap_close_listeners(); - for (i = 0; i < num_buckets; i++) { + if (ap_have_so_reuseport) { + ap_close_duplicated_listeners(event_listener, ap_num_buckets); + } + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_GRACEFUL); } ap_relieve_child_processes(event_note_child_killed); @@ -2937,7 +2950,7 @@ * way, try and make sure that all of our processes are * really dead. */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); } ap_reclaim_child_processes(1, event_note_child_killed); @@ -2965,7 +2978,7 @@ AP_SIG_GRACEFUL_STRING " received. Doing graceful restart"); /* wake up the children...time to die. But we'll have more soon */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_GRACEFUL); } @@ -2979,7 +2992,7 @@ * and a SIGHUP, we may as well use the same signal, because some user * pthreads are stealing signals from us left and right. */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); } @@ -3011,7 +3024,6 @@ level_flags |= APLOG_STARTUP; } - enable_default_listener = 0; if ((num_listensocks = ap_setup_listeners(ap_server_conf)) < 1) { ap_log_error(APLOG_MARK, APLOG_ALERT | level_flags, 0, (startup ? NULL : s), @@ -3019,13 +3031,13 @@ return DONE; } - enable_default_listener = 1; - ap_duplicate_listeners(ap_server_conf, pconf, num_buckets); + ap_duplicate_listeners(ap_server_conf, pconf, &event_listener, + &ap_num_buckets, retained->is_graceful, one_process); - pod = apr_palloc(pconf, sizeof(ap_pod_t *) * num_buckets); + pod = apr_palloc(pconf, sizeof(ap_pod_t *) * ap_num_buckets); if (!one_process) { - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { if ((rv = ap_mpm_podx_open(pconf, &pod[i]))) { ap_log_error(APLOG_MARK, APLOG_CRIT | level_flags, rv, (startup ? NULL : s), @@ -3067,17 +3079,6 @@ retained = ap_retained_data_create(userdata_key, sizeof(*retained)); retained->max_daemons_limit = -1; } - if (!retained->is_graceful) { - num_buckets = 1; -#ifdef _SC_NPROCESSORS_ONLN - if (have_so_reuseport) { - int num_online_cores = sysconf(_SC_NPROCESSORS_ONLN); - if (num_online_cores > 8) { - num_buckets = num_online_cores / 8; - } - } -#endif - } ++retained->module_loads; if (retained->module_loads == 2) { /* test for correct operation of fdqueue */ @@ -3090,8 +3091,8 @@ "atomics not working as expected - add32 of negative number"); return HTTP_INTERNAL_SERVER_ERROR; } - retained->idle_spawn_rate = apr_palloc(pconf, sizeof(int) * num_buckets); - for (i = 0; i< num_buckets; i++) { + retained->idle_spawn_rate = apr_palloc(pconf, sizeof(int) * ap_num_buckets); + for (i = 0; i< ap_num_buckets; i++) { retained->idle_spawn_rate[i] = 1; } rv = apr_pollset_create(&event_pollset, 1, plog, @@ -3329,11 +3330,11 @@ } ap_daemons_limit = server_limit; } - else if (ap_daemons_limit < num_buckets) { - /* Don't thrash since num_buckets depends on + else if (ap_daemons_limit < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - ap_daemons_limit = num_buckets; + ap_daemons_limit = ap_num_buckets; } /* ap_daemons_to_start > ap_daemons_limit checked in ap_mpm_run() */ @@ -3349,11 +3350,11 @@ } ap_daemons_to_start = 1; } - if (ap_daemons_to_start < num_buckets) { - /* Don't thrash since num_buckets depends on + if (ap_daemons_to_start < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - ap_daemons_to_start = num_buckets; + ap_daemons_to_start = ap_num_buckets; } if (min_spare_threads < 1) { @@ -3372,11 +3373,11 @@ } min_spare_threads = 1; } - if (min_spare_threads < num_buckets) { - /* Don't thrash since num_buckets depends on + if (min_spare_threads < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - min_spare_threads = num_buckets; + min_spare_threads = ap_num_buckets; } /* max_spare_threads < min_spare_threads + threads_per_child diff -ru httpd-trunk/server/mpm/prefork/prefork.c httpd-trunk.new/server/mpm/prefork/prefork.c --- httpd-trunk/server/mpm/prefork/prefork.c 2014-10-10 10:39:26.504416876 -0400 +++ httpd-trunk.new/server/mpm/prefork/prefork.c 2014-10-10 12:53:54.649287909 -0400 @@ -96,11 +96,9 @@ static int ap_daemons_limit=0; /* MaxRequestWorkers */ static int server_limit = 0; static int mpm_state = AP_MPMQ_STARTING; +static ap_listen_rec **prefork_listener; static ap_pod_t **pod; -static ap_pod_t *child_pod; -static apr_proc_mutex_t *child_mutex; -static ap_listen_rec *child_listen; - +static int ap_num_buckets; /* data retained by prefork across load/unload of the module * allocated on first call to pre-config hook; located on @@ -132,6 +130,14 @@ } prefork_retained_data; static prefork_retained_data *retained; +typedef struct prefork_child_data { + ap_pod_t *pod; + apr_proc_mutex_t *mutex; + ap_listen_rec *listener; + int bucket; +} prefork_child_data; +static prefork_child_data *child; + #define MPM_CHILD_PID(i) (ap_scoreboard_image->parent[i].pid) /* one_process --- debugging mode variable; can be set from the command line @@ -232,14 +238,14 @@ prefork_note_child_killed(/* slot */ 0, 0, 0); } - ap_mpm_pod_close(child_pod); + ap_mpm_pod_close(child->pod); chdir_for_gprof(); exit(code); } static void accept_mutex_on(void) { - apr_status_t rv = apr_proc_mutex_lock(child_mutex); + apr_status_t rv = apr_proc_mutex_lock(child->mutex); if (rv != APR_SUCCESS) { const char *msg = "couldn't grab the accept mutex"; @@ -257,7 +263,7 @@ static void accept_mutex_off(void) { - apr_status_t rv = apr_proc_mutex_unlock(child_mutex); + apr_status_t rv = apr_proc_mutex_unlock(child->mutex); if (rv != APR_SUCCESS) { const char *msg = "couldn't release the accept mutex"; @@ -282,7 +288,7 @@ * when it's safe in the single Listen case. */ #ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT -#define SAFE_ACCEPT(stmt) do {if (child_listen->next) {stmt;}} while(0) +#define SAFE_ACCEPT(stmt) do {if (ap_listeners->next) {stmt;}} while(0) #else #define SAFE_ACCEPT(stmt) do {stmt;} while(0) #endif @@ -363,7 +369,7 @@ { mpm_state = AP_MPMQ_STOPPING; ap_close_listeners(); - + ap_close_child_listener(child->listener); /* For a graceful stop, we want the child to exit when done */ die_now = 1; } @@ -487,7 +493,7 @@ static int requests_this_child; static int num_listensocks = 0; -static void child_main(int child_num_arg, int child_bucket) +static void child_main(int child_num_arg) { #if APR_HAS_THREADS apr_thread_t *thd = NULL; @@ -512,9 +518,9 @@ ap_my_pid = getpid(); requests_this_child = 0; - child_listen = mpm_listen[child_bucket]; - child_mutex = accept_mutex[child_bucket]; - child_pod = pod[child_bucket]; + child->listener = prefork_listener[child->bucket]; + child->mutex = accept_mutex[child->bucket]; + child->pod = pod[child->bucket]; ap_fatal_signal_child_setup(ap_server_conf); @@ -536,9 +542,9 @@ apr_pool_tag(ptrans, "transaction"); /* close unused listeners and pods */ - for (i = 0; i < num_buckets; i++) { - if (i != child_bucket) { - lr = mpm_listen[i]; + for (i = 0; i < ap_num_buckets; i++) { + if (i != child->bucket) { + lr = prefork_listener[i]; while(lr) { apr_socket_close(lr->sd); lr->active = 0; @@ -550,8 +556,8 @@ /* needs to be done before we switch UIDs so we have permissions */ ap_reopen_scoreboard(pchild, NULL, 0); - lockfile = apr_proc_mutex_lockfile(child_mutex); - status = apr_proc_mutex_child_init(&child_mutex, + lockfile = apr_proc_mutex_lockfile(child->mutex); + status = apr_proc_mutex_child_init(&child->mutex, lockfile, pchild); if (status != APR_SUCCESS) { @@ -559,7 +565,7 @@ "Couldn't initialize cross-process lock in child " "(%s) (%s)", lockfile ? lockfile : "none", - apr_proc_mutex_name(child_mutex)); + apr_proc_mutex_name(child->mutex)); clean_child_exit(APEXIT_CHILDFATAL); } @@ -581,7 +587,7 @@ clean_child_exit(APEXIT_CHILDSICK); /* assume temporary resource issue */ } - for (lr = child_listen, i = num_listensocks; i--; lr = lr->next) { + for (lr = child->listener, i = num_listensocks; i--; lr = lr->next) { apr_pollfd_t pfd = { 0 }; pfd.desc_type = APR_POLL_SOCKET; @@ -639,7 +645,7 @@ if (num_listensocks == 1) { /* There is only one listener record, so refer to that one. */ - lr = child_listen; + lr = child->listener; } else { /* multiple listening sockets - need to poll */ @@ -737,7 +743,7 @@ * while we were processing the connection or we are the lucky * idle server process that gets to die. */ - if (ap_mpm_pod_check(child_pod) == APR_SUCCESS) { /* selected as idle? */ + if (ap_mpm_pod_check(child->pod) == APR_SUCCESS) { /* selected as idle? */ die_now = 1; } else if (retained->my_generation != @@ -761,6 +767,8 @@ retained->max_daemons_limit = slot + 1; } + child->bucket = bucket; + if (one_process) { apr_signal(SIGHUP, sig_term); /* Don't catch AP_SIG_GRACEFUL in ONE_PROCESS mode :) */ @@ -770,7 +778,7 @@ #endif apr_signal(SIGTERM, sig_term); prefork_note_child_started(slot, getpid()); - child_main(slot, bucket); + child_main(slot); /* NOTREACHED */ ap_assert(0); return -1; @@ -832,7 +840,7 @@ * The pod is used for signalling the graceful restart. */ apr_signal(AP_SIG_GRACEFUL, stop_listening); - child_main(slot, bucket); + child_main(slot); } ap_scoreboard_image->parent[slot].bucket = bucket; @@ -851,7 +859,7 @@ if (ap_scoreboard_image->servers[i][0].status != SERVER_DEAD) { continue; } - if (make_child(ap_server_conf, i, i % num_buckets) < 0) { + if (make_child(ap_server_conf, i, i % ap_num_buckets) < 0) { break; } --number_to_start; @@ -912,7 +920,7 @@ * shut down gracefully, in case it happened to pick up a request * while we were counting */ - bucket_kill_child_record = (bucket_kill_child_record + 1) % num_buckets; + bucket_kill_child_record = (bucket_kill_child_record + 1) % ap_num_buckets; ap_mpm_pod_signal(pod[bucket_kill_child_record]); retained->idle_spawn_rate = 1; } @@ -940,7 +948,7 @@ for (i = 0; i < free_length; ++i) { bucket_make_child_record++; make_child(ap_server_conf, free_slots[i], - bucket_make_child_record % num_buckets); + bucket_make_child_record % ap_num_buckets); } /* the next time around we want to spawn twice as many if this * wasn't good enough, but not if we've just done a graceful @@ -971,9 +979,11 @@ ap_log_pid(pconf, ap_pid_fname); + child = apr_palloc(_pconf, sizeof(*child)); + /* Initialize cross-process accept lock for each bucket*/ - accept_mutex = apr_palloc(_pconf, sizeof(apr_proc_mutex_t *) * num_buckets); - for (i = 0; i < num_buckets; i++) { + accept_mutex = apr_palloc(_pconf, sizeof(apr_proc_mutex_t *) * ap_num_buckets); + for (i = 0; i < ap_num_buckets; i++) { rv = ap_proc_mutex_create(&accept_mutex[i], NULL, AP_ACCEPT_MUTEX_TYPE, NULL, s, _pconf, 0); if (rv != APR_SUCCESS) { @@ -1005,8 +1015,8 @@ } /* Don't thrash... */ - if (ap_daemons_max_free < ap_daemons_min_free + num_buckets) - ap_daemons_max_free = ap_daemons_min_free + num_buckets; + if (ap_daemons_max_free < ap_daemons_min_free + ap_num_buckets) + ap_daemons_max_free = ap_daemons_min_free + ap_num_buckets; /* If we're doing a graceful_restart then we're going to see a lot * of children exiting immediately when we get into the main loop @@ -1169,9 +1179,12 @@ /* Stop listening */ ap_close_listeners(); + if (ap_have_so_reuseport) { + ap_close_duplicated_listeners(prefork_listener, ap_num_buckets); + } /* kill off the idle ones */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_pod_killpg(pod[i], retained->max_daemons_limit); } @@ -1247,7 +1260,7 @@ "Graceful restart requested, doing restart"); /* kill off the idle ones */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_pod_killpg(pod[i], retained->max_daemons_limit); } @@ -1302,7 +1315,6 @@ level_flags |= APLOG_STARTUP; } - enable_default_listener = 0; if ((num_listensocks = ap_setup_listeners(ap_server_conf)) < 1) { ap_log_error(APLOG_MARK, APLOG_ALERT | level_flags, 0, (startup ? NULL : s), @@ -1310,11 +1322,11 @@ return DONE; } - enable_default_listener = 1; - ap_duplicate_listeners(ap_server_conf, pconf, num_buckets); + ap_duplicate_listeners(ap_server_conf, pconf, &prefork_listener, + &ap_num_buckets, retained->is_graceful, one_process); - pod = apr_palloc(pconf, sizeof(ap_pod_t *) * num_buckets); - for (i = 0; i < num_buckets; i++) { + pod = apr_palloc(pconf, sizeof(ap_pod_t *) * ap_num_buckets); + for (i = 0; i < ap_num_buckets; i++) { if ((rv = ap_mpm_pod_open(pconf, &pod[i]))) { ap_log_error(APLOG_MARK, APLOG_CRIT | level_flags, rv, (startup ? NULL : s), @@ -1356,17 +1368,6 @@ retained->max_daemons_limit = -1; retained->idle_spawn_rate = 1; } - if (!retained->is_graceful) { - num_buckets = 1; -#ifdef _SC_NPROCESSORS_ONLN - if (have_so_reuseport) { - int num_online_cores = sysconf(_SC_NPROCESSORS_ONLN); - if (num_online_cores > 8) { - num_buckets = num_online_cores / 8; - } - } -#endif - } ++retained->module_loads; if (retained->module_loads == 2) { if (!one_process && !foreground) { @@ -1480,11 +1481,11 @@ } ap_daemons_limit = 1; } - if (ap_daemons_limit < num_buckets) { - /* Don't thrash since num_buckets depends on + if (ap_daemons_limit < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - ap_daemons_limit = num_buckets; + ap_daemons_limit = ap_num_buckets; } /* ap_daemons_to_start > ap_daemons_limit checked in prefork_run() */ @@ -1500,11 +1501,11 @@ } ap_daemons_to_start = 1; } - if (ap_daemons_to_start < num_buckets) { - /* Don't thrash since num_buckets depends on + if (ap_daemons_to_start < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - ap_daemons_to_start = num_buckets; + ap_daemons_to_start = ap_num_buckets; } if (ap_daemons_min_free < 1) { @@ -1523,11 +1524,11 @@ } ap_daemons_min_free = 1; } - if (ap_daemons_min_free < num_buckets) { - /* Don't thrash since num_buckets depends on + if (ap_daemons_min_free < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - ap_daemons_min_free = num_buckets; + ap_daemons_min_free = ap_num_buckets; } /* ap_daemons_max_free < ap_daemons_min_free + 1 checked in prefork_run() */ diff -ru httpd-trunk/server/mpm/worker/worker.c httpd-trunk.new/server/mpm/worker/worker.c --- httpd-trunk/server/mpm/worker/worker.c 2014-10-10 10:39:26.533416875 -0400 +++ httpd-trunk.new/server/mpm/worker/worker.c 2014-10-10 11:36:04.755362556 -0400 @@ -136,6 +136,8 @@ static fd_queue_t *worker_queue; static fd_queue_info_t *worker_queue_info; static int mpm_state = AP_MPMQ_STARTING; +static int ap_num_buckets; +static ap_listen_rec **worker_listener; /* data retained by worker across load/unload of the module * allocated on first call to pre-config hook; located on @@ -171,6 +173,14 @@ } worker_retained_data; static worker_retained_data *retained; +typedef struct worker_child_data { + ap_pod_t *pod; + apr_proc_mutex_t *mutex; + ap_listen_rec *listener; + int bucket; +} worker_child_data; +static worker_child_data *child; + #define MPM_CHILD_PID(i) (ap_scoreboard_image->parent[i].pid) /* The structure used to pass unique initialization info to each thread */ @@ -193,7 +203,6 @@ #define ID_FROM_CHILD_THREAD(c, t) ((c * thread_limit) + t) static ap_pod_t **pod; -static ap_pod_t *child_pod; /* The worker MPM respects a couple of runtime flags that can aid * in debugging. Setting the -DNO_DETACH flag will prevent the root process @@ -224,11 +233,9 @@ /* Locks for accept serialization */ static apr_proc_mutex_t **accept_mutex; -static apr_proc_mutex_t *child_mutex; -static ap_listen_rec *child_listen; #ifdef SINGLE_LISTEN_UNSERIALIZED_ACCEPT -#define SAFE_ACCEPT(stmt) (child_listen->next ? (stmt) : APR_SUCCESS) +#define SAFE_ACCEPT(stmt) (ap_listeners->next ? (stmt) : APR_SUCCESS) #else #define SAFE_ACCEPT(stmt) (stmt) #endif @@ -696,6 +703,7 @@ ap_listen_rec *lr; int have_idle_worker = 0; int last_poll_idx = 0; + int i; free(ti); @@ -708,7 +716,7 @@ clean_child_exit(APEXIT_CHILDSICK); } - for (lr = child_listen; lr != NULL; lr = lr->next) { + for (lr = child->listener; lr != NULL; lr = lr->next) { apr_pollfd_t pfd = { 0 }; pfd.desc_type = APR_POLL_SOCKET; @@ -765,7 +773,7 @@ /* We've already decremented the idle worker count inside * ap_queue_info_wait_for_idler. */ - if ((rv = SAFE_ACCEPT(apr_proc_mutex_lock(child_mutex))) + if ((rv = SAFE_ACCEPT(apr_proc_mutex_lock(child->mutex))) != APR_SUCCESS) { if (!listener_may_exit) { @@ -774,9 +782,9 @@ break; /* skip the lock release */ } - if (!child_listen->next) { + if (!(child->listener)->next) { /* Only one listener, so skip the poll */ - lr = child_listen; + lr = child->listener; } else { while (!listener_may_exit) { @@ -846,7 +854,7 @@ resource_shortage = 1; signal_threads(ST_GRACEFUL); } - if ((rv = SAFE_ACCEPT(apr_proc_mutex_unlock(child_mutex))) + if ((rv = SAFE_ACCEPT(apr_proc_mutex_unlock(child->mutex))) != APR_SUCCESS) { if (listener_may_exit) { @@ -870,7 +878,7 @@ } } else { - if ((rv = SAFE_ACCEPT(apr_proc_mutex_unlock(child_mutex))) + if ((rv = SAFE_ACCEPT(apr_proc_mutex_unlock(child->mutex))) != APR_SUCCESS) { int level = APLOG_EMERG; @@ -887,7 +895,7 @@ } } - ap_close_listeners(); + ap_close_child_listener(child->listener); ap_queue_term(worker_queue); dying = 1; ap_scoreboard_image->parent[process_slot].quiescing = 1; @@ -1217,7 +1225,7 @@ } } -static void child_main(int child_num_arg, int child_bucket) +static void child_main(int child_num_arg) { apr_thread_t **threads; apr_status_t rv; @@ -1234,19 +1242,20 @@ ap_fatal_signal_child_setup(ap_server_conf); apr_pool_create(&pchild, pconf); - child_listen = mpm_listen[child_bucket]; - child_mutex = accept_mutex[child_bucket]; - child_pod = pod[child_bucket]; + child->listener = worker_listener[child->bucket]; + child->mutex = accept_mutex[child->bucket]; + child->pod = pod[child->bucket]; /* close unused listeners and pods */ - for (i = 0; i < num_buckets; i++) { - if (i != child_bucket) { - lr = mpm_listen[i]; + for (i = 0; i < ap_num_buckets; i++) { + if (i != child->bucket) { + lr = worker_listener[i]; while(lr) { apr_socket_close(lr->sd); lr->active = 0; lr = lr->next; } + worker_listener[i]->active = 0; ap_mpm_podx_close(pod[i]); } } @@ -1254,8 +1263,8 @@ /*stuff to do before we switch id's, so we have permissions.*/ ap_reopen_scoreboard(pchild, NULL, 0); - rv = SAFE_ACCEPT(apr_proc_mutex_child_init(&child_mutex, - apr_proc_mutex_lockfile(child_mutex), + rv = SAFE_ACCEPT(apr_proc_mutex_child_init(&child->mutex, + apr_proc_mutex_lockfile(child->mutex), pchild)); if (rv != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(00280) @@ -1364,7 +1373,7 @@ apr_signal(SIGTERM, dummy_signal_handler); /* Watch for any messages from the parent over the POD */ while (1) { - rv = ap_mpm_podx_check(child_pod); + rv = ap_mpm_podx_check(child->pod); if (rv == AP_MPM_PODX_NORESTART) { /* see if termination was triggered while we slept */ switch(terminate_mode) { @@ -1410,10 +1419,12 @@ retained->max_daemons_limit = slot + 1; } + child->bucket = bucket; + if (one_process) { set_signals(); worker_note_child_started(slot, getpid()); - child_main(0, 0); + child_main(0); /* NOTREACHED */ ap_assert(0); return -1; @@ -1452,7 +1463,7 @@ RAISE_SIGSTOP(MAKE_CHILD); apr_signal(SIGTERM, just_die); - child_main(slot, bucket); + child_main(slot); /* NOTREACHED */ ap_assert(0); return -1; @@ -1480,7 +1491,7 @@ if (ap_scoreboard_image->parent[i].pid != 0) { continue; } - if (make_child(ap_server_conf, i, i % num_buckets) < 0) { + if (make_child(ap_server_conf, i, i % ap_num_buckets) < 0) { break; } --number_to_start; @@ -1555,7 +1566,7 @@ } active_thread_count += child_threads_active; if (any_dead_threads && totally_free_length < retained->idle_spawn_rate[child_bucket] - && free_length < MAX_SPAWN_RATE / num_buckets + && free_length < MAX_SPAWN_RATE / ap_num_buckets && (!ps->pid /* no process in the slot */ || ps->quiescing)) { /* or at least one is going away */ if (all_dead_threads) { @@ -1611,12 +1622,12 @@ retained->max_daemons_limit = last_non_dead + 1; - if (idle_thread_count > max_spare_threads / num_buckets) { + if (idle_thread_count > max_spare_threads / ap_num_buckets) { /* Kill off one child */ ap_mpm_podx_signal(pod[child_bucket], AP_MPM_PODX_GRACEFUL); retained->idle_spawn_rate[child_bucket] = 1; } - else if (idle_thread_count < min_spare_threads / num_buckets) { + else if (idle_thread_count < min_spare_threads / ap_num_buckets) { /* terminate the free list */ if (free_length == 0) { /* scoreboard is full, can't fork */ @@ -1673,7 +1684,7 @@ --retained->hold_off_on_exponential_spawning; } else if (retained->idle_spawn_rate[child_bucket] - < MAX_SPAWN_RATE / num_buckets) { + < MAX_SPAWN_RATE / ap_num_buckets) { retained->idle_spawn_rate[child_bucket] *= 2; } } @@ -1755,7 +1766,7 @@ if (processed_status == APEXIT_CHILDSICK && old_gen == retained->my_generation) { /* resource shortage, minimize the fork rate */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { retained->idle_spawn_rate[i] = 1; } } @@ -1796,7 +1807,7 @@ continue; } - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { perform_idle_server_maintenance(i); } } @@ -1810,9 +1821,11 @@ ap_log_pid(pconf, ap_pid_fname); + child = apr_palloc(_pconf, sizeof(*child)); + /* Initialize cross-process accept lock */ - accept_mutex = apr_palloc(_pconf, sizeof(apr_proc_mutex_t *) * num_buckets); - for (i = 0; i < num_buckets; i++) { + accept_mutex = apr_palloc(_pconf, sizeof(apr_proc_mutex_t *) * ap_num_buckets); + for (i = 0; i < ap_num_buckets; i++) { rv = ap_proc_mutex_create(&accept_mutex[i], NULL, AP_ACCEPT_MUTEX_TYPE, NULL, s, _pconf, 0); if (rv != APR_SUCCESS) { @@ -1836,10 +1849,10 @@ set_signals(); /* Don't thrash... */ - if (min_spare_threads < threads_per_child * num_buckets) - min_spare_threads = threads_per_child * num_buckets; - if (max_spare_threads < min_spare_threads + threads_per_child * num_buckets) - max_spare_threads = min_spare_threads + threads_per_child * num_buckets; + if (min_spare_threads < threads_per_child * ap_num_buckets) + min_spare_threads = threads_per_child * ap_num_buckets; + if (max_spare_threads < min_spare_threads + threads_per_child * ap_num_buckets) + max_spare_threads = min_spare_threads + threads_per_child * ap_num_buckets; /* If we're doing a graceful_restart then we're going to see a lot * of children exiting immediately when we get into the main loop @@ -1884,7 +1897,7 @@ /* Time to shut down: * Kill child processes, tell them to call child_exit, etc... */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); } ap_reclaim_child_processes(1, /* Start with SIGTERM */ @@ -1907,7 +1920,11 @@ /* Close our listeners, and then ask our children to do same */ ap_close_listeners(); - for (i = 0; i < num_buckets; i++) { + if (ap_have_so_reuseport) { + ap_close_duplicated_listeners(worker_listener, ap_num_buckets); + } + + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_GRACEFUL); } ap_relieve_child_processes(worker_note_child_killed); @@ -1949,7 +1966,7 @@ * way, try and make sure that all of our processes are * really dead. */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); } ap_reclaim_child_processes(1, worker_note_child_killed); @@ -1976,7 +1993,7 @@ ap_log_error(APLOG_MARK, APLOG_NOTICE, 0, ap_server_conf, APLOGNO(00297) AP_SIG_GRACEFUL_STRING " received. Doing graceful restart"); /* wake up the children...time to die. But we'll have more soon */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_GRACEFUL); } @@ -1990,7 +2007,7 @@ * and a SIGHUP, we may as well use the same signal, because some user * pthreads are stealing signals from us left and right. */ - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { ap_mpm_podx_killpg(pod[i], ap_daemons_limit, AP_MPM_PODX_RESTART); } @@ -2021,7 +2038,6 @@ level_flags |= APLOG_STARTUP; } - enable_default_listener = 0; if ((num_listensocks = ap_setup_listeners(ap_server_conf)) < 1) { ap_log_error(APLOG_MARK, APLOG_ALERT | level_flags, 0, (startup ? NULL : s), @@ -2029,12 +2045,12 @@ return DONE; } - enable_default_listener = 1; - ap_duplicate_listeners(ap_server_conf, pconf, num_buckets); + ap_duplicate_listeners(ap_server_conf, pconf, &worker_listener, + &ap_num_buckets, retained->is_graceful, one_process); - pod = apr_palloc(pconf, sizeof(ap_pod_t *) * num_buckets); + pod = apr_palloc(pconf, sizeof(ap_pod_t *) * ap_num_buckets); if (!one_process) { - for (i = 0; i < num_buckets; i++) { + for (i = 0; i < ap_num_buckets; i++) { if ((rv = ap_mpm_podx_open(pconf, &pod[i]))) { ap_log_error(APLOG_MARK, APLOG_CRIT | level_flags, rv, (startup ? NULL : s), @@ -2076,17 +2092,6 @@ retained = ap_retained_data_create(userdata_key, sizeof(*retained)); retained->max_daemons_limit = -1; } - if (!retained->is_graceful) { - num_buckets = 1; -#ifdef _SC_NPROCESSORS_ONLN - if (have_so_reuseport) { - int num_online_cores = sysconf(_SC_NPROCESSORS_ONLN); - if (num_online_cores > 8) { - num_buckets = num_online_cores / 8; - } - } -#endif - } ++retained->module_loads; if (retained->module_loads == 2) { if (!one_process && !foreground) { @@ -2100,8 +2105,8 @@ return HTTP_INTERNAL_SERVER_ERROR; } } - retained->idle_spawn_rate = apr_palloc(pconf, sizeof(int) * num_buckets); - for (i = 0; i< num_buckets; i++) { + retained->idle_spawn_rate = apr_palloc(pconf, sizeof(int) * ap_num_buckets); + for (i = 0; i< ap_num_buckets; i++) { retained->idle_spawn_rate[i] = 1; } } @@ -2319,11 +2324,11 @@ } ap_daemons_limit = server_limit; } - else if (ap_daemons_limit < num_buckets) { - /* Don't thrash since num_buckets depends on + else if (ap_daemons_limit < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - ap_daemons_limit = num_buckets; + ap_daemons_limit = ap_num_buckets; } /* ap_daemons_to_start > ap_daemons_limit checked in worker_run() */ @@ -2339,11 +2344,11 @@ } ap_daemons_to_start = 1; } - if (ap_daemons_to_start < num_buckets) { - /* Don't thrash since num_buckets depends on + if (ap_daemons_to_start < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - ap_daemons_to_start = num_buckets; + ap_daemons_to_start = ap_num_buckets; } if (min_spare_threads < 1) { @@ -2362,11 +2367,11 @@ } min_spare_threads = 1; } - if (min_spare_threads < num_buckets) { - /* Don't thrash since num_buckets depends on + if (min_spare_threads < ap_num_buckets) { + /* Don't thrash since ap_num_buckets depends on * the system and the number of CPU cores. */ - min_spare_threads = num_buckets; + min_spare_threads = ap_num_buckets; } /* max_spare_threads < min_spare_threads + threads_per_child diff -ru httpd-trunk/server/mpm_unix.c httpd-trunk.new/server/mpm_unix.c --- httpd-trunk/server/mpm_unix.c 2014-10-10 10:39:26.815416871 -0400 +++ httpd-trunk.new/server/mpm_unix.c 2014-10-10 10:47:17.888409341 -0400 @@ -627,89 +627,87 @@ * plain-HTTP, not SSL; using an SSL port would either be * expensive to do correctly (performing a complete SSL handshake) * or cause log spam by doing incorrectly (simply sending EOF). */ - for (i = 0; i < num_buckets; i++) { - lp = mpm_listen[i]; - while (lp && lp->protocol && strcasecmp(lp->protocol, "http") != 0) { - lp = lp->next; - } - if (!lp) { - lp = mpm_listen[i]; - } - - rv = apr_socket_create(&sock, lp->bind_addr->family, SOCK_STREAM, 0, p); - if (rv != APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, APLOGNO(00054) - "get socket to connect to listener"); - apr_pool_destroy(p); - return rv; - } + lp = ap_listeners; + while (lp && lp->protocol && strcasecmp(lp->protocol, "http") != 0) { + lp = lp->next; + } + if (!lp) { + lp = ap_listeners; + } - /* on some platforms (e.g., FreeBSD), the kernel won't accept many - * queued connections before it starts blocking local connects... - * we need to keep from blocking too long and instead return an error, - * because the MPM won't want to hold up a graceful restart for a - * long time - */ - rv = apr_socket_timeout_set(sock, apr_time_from_sec(3)); - if (rv != APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, APLOGNO(00055) - "set timeout on socket to connect to listener"); - apr_socket_close(sock); - apr_pool_destroy(p); - return rv; - } + rv = apr_socket_create(&sock, lp->bind_addr->family, SOCK_STREAM, 0, p); + if (rv != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, APLOGNO(00054) + "get socket to connect to listener"); + apr_pool_destroy(p); + return rv; + } - rv = apr_socket_connect(sock, lp->bind_addr); - if (rv != APR_SUCCESS) { - int log_level = APLOG_WARNING; - - if (APR_STATUS_IS_TIMEUP(rv)) { - /* probably some server processes bailed out already and there - * is nobody around to call accept and clear out the kernel - * connection queue; usually this is not worth logging - */ - log_level = APLOG_DEBUG; - } - - ap_log_error(APLOG_MARK, log_level, rv, ap_server_conf, APLOGNO(00056) - "connect to listener on %pI", lp->bind_addr); - apr_pool_destroy(p); - return rv; - } + /* on some platforms (e.g., FreeBSD), the kernel won't accept many + * queued connections before it starts blocking local connects... + * we need to keep from blocking too long and instead return an error, + * because the MPM won't want to hold up a graceful restart for a + * long time + */ + rv = apr_socket_timeout_set(sock, apr_time_from_sec(3)); + if (rv != APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, APLOGNO(00055) + "set timeout on socket to connect to listener"); + apr_socket_close(sock); + apr_pool_destroy(p); + return rv; + } - if (lp->protocol && strcasecmp(lp->protocol, "https") == 0) { - /* Send a TLS 1.0 close_notify alert. This is perhaps the - * "least wrong" way to open and cleanly terminate an SSL - * connection. It should "work" without noisy error logs if - * the server actually expects SSLv3/TLSv1. With - * SSLv23_server_method() OpenSSL's SSL_accept() fails - * ungracefully on receipt of this message, since it requires - * an 11-byte ClientHello message and this is too short. */ - static const unsigned char tls10_close_notify[7] = { - '\x15', /* TLSPlainText.type = Alert (21) */ - '\x03', '\x01', /* TLSPlainText.version = {3, 1} */ - '\x00', '\x02', /* TLSPlainText.length = 2 */ - '\x01', /* Alert.level = warning (1) */ - '\x00' /* Alert.description = close_notify (0) */ - }; - data = (const char *)tls10_close_notify; - len = sizeof(tls10_close_notify); - } - else /* ... XXX other request types here? */ { - /* Create an HTTP request string. We include a User-Agent so - * that adminstrators can track down the cause of the - * odd-looking requests in their logs. A complete request is - * used since kernel-level filtering may require that much - * data before returning from accept(). */ - data = apr_pstrcat(p, "OPTIONS * HTTP/1.0\r\nUser-Agent: ", - ap_get_server_description(), - " (internal dummy connection)\r\n\r\n", NULL); - len = strlen(data); - } + rv = apr_socket_connect(sock, lp->bind_addr); + if (rv != APR_SUCCESS) { + int log_level = APLOG_WARNING; + + if (APR_STATUS_IS_TIMEUP(rv)) { + /* probably some server processes bailed out already and there + * is nobody around to call accept and clear out the kernel + * connection queue; usually this is not worth logging + */ + log_level = APLOG_DEBUG; + } + + ap_log_error(APLOG_MARK, log_level, rv, ap_server_conf, APLOGNO(00056) + "connect to listener on %pI", lp->bind_addr); + apr_pool_destroy(p); + return rv; + } - apr_socket_send(sock, data, &len); - apr_socket_close(sock); + if (lp->protocol && strcasecmp(lp->protocol, "https") == 0) { + /* Send a TLS 1.0 close_notify alert. This is perhaps the + * "least wrong" way to open and cleanly terminate an SSL + * connection. It should "work" without noisy error logs if + * the server actually expects SSLv3/TLSv1. With + * SSLv23_server_method() OpenSSL's SSL_accept() fails + * ungracefully on receipt of this message, since it requires + * an 11-byte ClientHello message and this is too short. */ + static const unsigned char tls10_close_notify[7] = { + '\x15', /* TLSPlainText.type = Alert (21) */ + '\x03', '\x01', /* TLSPlainText.version = {3, 1} */ + '\x00', '\x02', /* TLSPlainText.length = 2 */ + '\x01', /* Alert.level = warning (1) */ + '\x00' /* Alert.description = close_notify (0) */ + }; + data = (const char *)tls10_close_notify; + len = sizeof(tls10_close_notify); } + else /* ... XXX other request types here? */ { + /* Create an HTTP request string. We include a User-Agent so + * that adminstrators can track down the cause of the + * odd-looking requests in their logs. A complete request is + * used since kernel-level filtering may require that much + * data before returning from accept(). */ + data = apr_pstrcat(p, "OPTIONS * HTTP/1.0\r\nUser-Agent: ", + ap_get_server_description(), + " (internal dummy connection)\r\n\r\n", NULL); + len = strlen(data); + } + + apr_socket_send(sock, data, &len); + apr_socket_close(sock); apr_pool_destroy(p); return rv;