Graceful restart not so graceful?
Would folks comment on Nathan's, Joe's and Stefan's work on https://issues.apache.org/bugzilla/show_bug.cgi?id=42829 and offer any comments on why this patch; https://issues.apache.org/bugzilla/attachment.cgi?id=22822 should not be applied to trunk/ and branches/2.2.x/ in time for the next releases? If I hear no objections by noon Jan 8 I'll apply. According to Stefan this current patch is improving the situation. Bill
graceful
Is there a way for a child httpd to trigger a graceful restart, e.g. from a watchdog? It seems to me that our unixd security model forbids this in most setups (User/Group config). If I understand correctly, httpd under Windows is one process, so triggering the mpm_signal_service() would work? Thanks for your help, Stefan
graceful?
Has anybody torture tested graceful restarts lately? I only ask because we just got a PR that gave me "that sinking feeling". Maybe not a real problem, but just figured I'd ask. --Cliff
Re: Graceful restart not so graceful?
On Tue, Jan 6, 2009 at 1:10 PM, William A. Rowe, Jr. wrote: > Would folks comment on Nathan's, Joe's and Stefan's work on > > https://issues.apache.org/bugzilla/show_bug.cgi?id=42829 > > and offer any comments on why this patch; > > https://issues.apache.org/bugzilla/attachment.cgi?id=22822 > > should not be applied to trunk/ and branches/2.2.x/ in time for > the next releases? > > If I hear no objections by noon Jan 8 I'll apply. According to > Stefan this current patch is improving the situation. > > Bill > > See also http://marc.info/?l=apache-httpd-dev&m=119945416529706&w=2
Re: Graceful restart not so graceful?
On 01/06/2009 07:10 PM, William A. Rowe, Jr. wrote:
> Would folks comment on Nathan's, Joe's and Stefan's work on
>
> https://issues.apache.org/bugzilla/show_bug.cgi?id=42829
>
> and offer any comments on why this patch;
>
> https://issues.apache.org/bugzilla/attachment.cgi?id=22822
>
> should not be applied to trunk/ and branches/2.2.x/ in time for
> the next releases?
@@ -573,6 +588,11 @@ static void child_main(int child_num_arg
apr_int32_t numdesc;
const apr_pollfd_t *pdesc;
+if (die_now) {
+status = !APR_SUCCESS;
+goto unlock;
+}
+
Hm, what happens if we get the signal exactly here?
Then stop_listening only set die_now to 1. So we reenter the poll and stay in
until we get a new request to OUR process on one of the listeners.
Is this the desired behaviour?
/* timeout == -1 == wait forever */
status = apr_pollset_poll(pollset, -1, &numdesc, &pdesc);
if (status != APR_SUCCESS) {
How about this one instead:
Index: server/mpm/prefork/prefork.c
===
--- server/mpm/prefork/prefork.c(Revision 731601)
+++ server/mpm/prefork/prefork.c(Arbeitskopie)
@@ -99,6 +99,8 @@
static int mpm_state = AP_MPMQ_STARTING;
static ap_pod_t *pod;
+static int dummy_listener;
+
/*
* The max child slot ever assigned, preserved across restarts. Necessary
* to deal with MaxClients changes across AP_SIG_GRACEFUL restarts. We
@@ -136,6 +138,8 @@
#endif /* TPF */
static volatile int die_now = 0;
+static volatile int listeners_closed = 0;
+static int close_connection = 1;
#ifdef GPROF
/*
@@ -304,8 +308,19 @@
static void stop_listening(int sig)
{
-ap_close_listeners();
+if (close_connection && !listeners_closed) {
+ap_listen_rec *lr;
+for (lr = ap_listeners; lr; lr = lr->next) {
+apr_os_sock_t fd;
+
+apr_os_sock_get(&fd, lr->sd);
+
+dup2(dummy_listener, fd);
+ }
+listeners_closed = 1;
+}
+
/* For a graceful stop, we want the child to exit when done */
die_now = 1;
}
@@ -533,6 +548,7 @@
if (num_listensocks == 1) {
/* There is only one listener record, so refer to that one. */
lr = ap_listeners;
+close_connection = 0;
}
else {
/* multiple listening sockets - need to poll */
@@ -540,8 +556,14 @@
apr_int32_t numdesc;
const apr_pollfd_t *pdesc;
+if (die_now) {
+status = !APR_SUCCESS;
+goto unlock;
+}
+
/* timeout == -1 == wait forever */
status = apr_pollset_poll(pollset, -1, &numdesc, &pdesc);
+close_connection = 0;
if (status != APR_SUCCESS) {
if (APR_STATUS_IS_EINTR(status)) {
if (one_process && shutdown_pending) {
@@ -595,8 +617,15 @@
/* if we accept() something we don't want to die, so we have to
* defer the exit
*/
-status = lr->accept_func(&csd, lr, ptrans);
+if (!die_now) {
+status = lr->accept_func(&csd, lr, ptrans);
+}
+else {
+status = !APR_SUCCESS;
+}
+unlock:
+close_connection = 1;
SAFE_ACCEPT(accept_mutex_off()); /* unlock after "accept" */
if (status == APR_EGENERAL) {
@@ -612,6 +641,11 @@
* socket options, file descriptors, and read/write buffers.
*/
+if (die_now && !listeners_closed) {
+ap_close_listeners();
+listeners_closed = 1;
+}
+
current_conn = ap_run_create_connection(ptrans, ap_server_conf, csd,
my_child_num, sbh, bucket_alloc);
if (current_conn) {
ap_process_connection(current_conn, csd);
@@ -890,6 +924,17 @@
mpm_state = AP_MPMQ_STOPPING;
return 1;
}
+
+if (dummy_listener == 0) {
+dummy_listener = socket(AF_INET, SOCK_STREAM, 0);
+if (dummy_listener < 0) {
+rv = errno;
+ap_log_error(APLOG_MARK, APLOG_EMERG, rv, s,
+ "Couldn't create dummy listener socket");
+mpm_state = AP_MPMQ_STOPPING;
+return 1;
+}
+}
#if APR_USE_SYSVSEM_SERIALIZE
if (ap_accept_lock_mech == APR_LOCK_DEFAULT ||
Regards
Rüdiger
Index: server/mpm/prefork/prefork.c
===
--- server/mpm/prefork/prefork.c (Revision 731601)
+++ server/mpm/prefork/prefork.c (Arbeitskopie)
@@ -99,6 +99,8 @@
static int mpm_state = AP_MPMQ_STARTING;
static ap_pod_t *pod;
+st
Re: Graceful restart not so graceful?
On Tue, Jan 06, 2009 at 12:10:25PM -0600, William Rowe wrote: > Would folks comment on Nathan's, Joe's and Stefan's work on > > https://issues.apache.org/bugzilla/show_bug.cgi?id=42829 > > and offer any comments on why this patch; > > https://issues.apache.org/bugzilla/attachment.cgi?id=22822 > > should not be applied to trunk/ and branches/2.2.x/ in time for > the next releases? I never found the time to commit this for a bunch of reasons: 1) I never convinced myself that adding a bunch of complexity to prefork, per Stefan's patch, was the right way to address this 2) I never worked out what impact the lack of check on the poll descriptor event type (POLLERR etc) had on this, but that needs resolving too 3) I hadn't checked whether worker et al had similar issues 4) I think that a simpler solution to this problem would be to add a timeout (e.g. 30s) on the child pollset_poll() call so they are guaranteed to pop at some point, but I haven't had time to try this or work out whether that will suck in some other way. (Also, my stuff to dup2() against the listener was demonstrably useless, per the thread Jeff referenced, so that should definitely not be committed, regardless.) Regards, Joe
Re: Graceful restart not so graceful?
On Wed, Jan 7, 2009 at 6:03 AM, Joe Orton wrote:
> On Tue, Jan 06, 2009 at 12:10:25PM -0600, William Rowe wrote:
> > Would folks comment on Nathan's, Joe's and Stefan's work on
> >
> > https://issues.apache.org/bugzilla/show_bug.cgi?id=42829
> >
> > and offer any comments on why this patch;
> >
> > https://issues.apache.org/bugzilla/attachment.cgi?id=22822
> >
> > should not be applied to trunk/ and branches/2.2.x/ in time for
> > the next releases?
>
> I never found the time to commit this for a bunch of reasons:
>
> 1) I never convinced myself that adding a bunch of complexity to
> prefork, per Stefan's patch, was the right way to address this
>
> 2) I never worked out what impact the lack of check on the poll
> descriptor event type (POLLERR etc) had on this, but that needs
> resolving too
>
> 3) I hadn't checked whether worker et al had similar issues
>
> 4) I think that a simpler solution to this problem would be to add a
> timeout (e.g. 30s) on the child pollset_poll() call so they are
> guaranteed to pop at some point, but I haven't had time to try this or
> work out whether that will suck in some other way.
Initial testing of your idea for a timeout was promising. I'll try to test
it with graceful stop as well.
Index: server/mpm/prefork/prefork.c
===
--- server/mpm/prefork/prefork.c(revision 731724)
+++ server/mpm/prefork/prefork.c(working copy)
@@ -540,10 +540,12 @@
apr_int32_t numdesc;
const apr_pollfd_t *pdesc;
-/* timeout == -1 == wait forever */
-status = apr_pollset_poll(pollset, -1, &numdesc, &pdesc);
+/* timeout == 10 seconds to avoid a hang at graceful
restart/stop
+ * caused by the closing of sockets by the signal handler
+ */
+status = apr_pollset_poll(pollset, apr_time_from_sec(10),
&numdesc, &pdesc);
if (status != APR_SUCCESS) {
-if (APR_STATUS_IS_EINTR(status)) {
+if (APR_STATUS_IS_TIMEUP(status) ||
APR_STATUS_IS_EINTR(status)) {
if (one_process && shutdown_pending) {
return;
}
Re: Graceful restart not so graceful?
> -Ursprüngliche Nachricht-
> Von: Jeff Trawick [mailto:traw...@gmail.com]
> Gesendet: Mittwoch, 7. Januar 2009 13:03
> An: dev@httpd.apache.org
> Betreff: Re: Graceful restart not so graceful?
>
> On Wed, Jan 7, 2009 at 6:03 AM, Joe Orton wrote:
>
>
> On Tue, Jan 06, 2009 at 12:10:25PM -0600, William Rowe wrote:
> > Would folks comment on Nathan's, Joe's and Stefan's work on
> >
> > https://issues.apache.org/bugzilla/show_bug.cgi?id=42829
> >
> > and offer any comments on why this patch;
> >
> > https://issues.apache.org/bugzilla/attachment.cgi?id=22822
> >
> > should not be applied to trunk/ and branches/2.2.x/
> in time for
> > the next releases?
>
>
> I never found the time to commit this for a bunch of reasons:
>
> 1) I never convinced myself that adding a bunch of complexity to
> prefork, per Stefan's patch, was the right way to address this
>
> 2) I never worked out what impact the lack of check on the poll
> descriptor event type (POLLERR etc) had on this, but that needs
> resolving too
>
> 3) I hadn't checked whether worker et al had similar issues
>
> 4) I think that a simpler solution to this problem
> would be to add a
> timeout (e.g. 30s) on the child pollset_poll() call so they are
> guaranteed to pop at some point, but I haven't had time
> to try this or
> work out whether that will suck in some other way.
>
>
> Initial testing of your idea for a timeout was promising.
> I'll try to test it with graceful stop as well.
>
> Index: server/mpm/prefork/prefork.c
> ==
> =
> --- server/mpm/prefork/prefork.c(revision 731724)
> +++ server/mpm/prefork/prefork.c(working copy)
> @@ -540,10 +540,12 @@
> apr_int32_t numdesc;
> const apr_pollfd_t *pdesc;
>
> -/* timeout == -1 == wait forever */
> -status = apr_pollset_poll(pollset, -1,
> &numdesc, &pdesc);
> +/* timeout == 10 seconds to avoid a hang at
> graceful restart/stop
> + * caused by the closing of sockets by the
> signal handler
> + */
> +status = apr_pollset_poll(pollset,
> apr_time_from_sec(10), &numdesc, &pdesc);
> if (status != APR_SUCCESS) {
> -if (APR_STATUS_IS_EINTR(status)) {
> +if (APR_STATUS_IS_TIMEUP(status) ||
> APR_STATUS_IS_EINTR(status)) {
> if (one_process && shutdown_pending) {
> return;
> }
Don't we need to add
SAFE_ACCEPT(accept_mutex_off());
before the continue later on in the
if (APR_STATUS_IS_TIMEUP(status) || APR_STATUS_IS_EINTR(status)) {
block or is it guaranteed that if we allready have the lock
that another SAFE_ACCEPT(accept_mutex_on()); doesn't do any harm?
Regards
Rüdiger
Re: Graceful restart not so graceful?
Hi, thanks for following up on this and sorry for the late response. On Wednesday 07 January 2009, Jeff Trawick wrote: > Initial testing of your idea for a timeout was promising. I couldn't reproduce any hangs under linux with the patch you commited to trunk. In my patch I tried to avoid that an idle apache wakes periodically for no good reasons. But if you don't think that is a problem, please backport your patch to 2.2.x. Cheers, Stefan
Re: Re: Graceful restart not so graceful?
On Jan 11, 2009 11:34am, Stefan Fritsch wrote: Hi, thanks for following up on this and sorry for the late response. On Wednesday 07 January 2009, Jeff Trawick wrote: > Initial testing of your idea for a timeout was promising. I couldn't reproduce any hangs under linux with the patch you commited to trunk. Thanks for all of your work on this issue! In my patch I tried to avoid that an idle apache wakes periodically for no good reasons. But if you don't think that is a problem, please backport your patch to 2.2.x. I'd also prefer that prefork doesn't wake up unless it has work, but I think the simpler, localized change which was committed will be less trouble in the long run. wrt backporting: I think that modern Linux and modern Solaris, perhaps the only platforms tested so far, both work for the same reason under the covers. Has anyone tested with regular poll()? If not, I can try to test another platform or at least dumb down APR to bypass epoll/event queue/etc.
Graceful stop
This has probably been discussed here before, and if it has I'd like to know the reasons why it's a bad idea. Anyway, I've been working on a graceful stop mechanism for httpd, which is reasonably trivial to work in - the problem is that there are no portable signal numbers left (or am I wrong in that?) which means one has to go. Why does graceful restart exist? Wouldn't it be better to have a graceful stop, and then just start a new instance? This would mean; * The existing behaviour of a graceful restart would be more-or-less preserved, httpd would be restarted without dropping clients. Though there would be issues with things like mod_perl/php/python persistent variables, and maybe a small window where requests are dropped. * httpd could be upgraded without dropping clients That last reason is my reason, right now when I need to upgrade httpd my only option is to kill httpd to free up port 80, this means kicking off a few thousand users who are in the middle of some very lengthy downloads. Since we use off-peak times like the weekend for doing things like this, this is really bad for dialup users - many of whom leave a long multi-day download running on a Friday - and arnt amused when they get back to the office on the Monday to find I killed it. But basically, graceful stop is trivially doable - it's the behaviour os most other daemons (things like sshd for example), it allows for graceful restarts, it allows graceful upgrades. But then, maybe I'm missing something, please enlighten me :) -- Colm MacCárthaighPublic Key: [EMAIL PROTECTED]
graceful restart
To the Knowledgable out there: how can I detect that the server is gracefully restarting? Backlground is that HTTP/2 1. is sometimes in the state of being IDLE, but unable to return to the MPM, *could* however terminate the HTTP2 session. 2. HTTP2 sessions can be brought down by first announcing that no more streams will be processed and then closing, once the existing ones are done. Thanks for the help. -Stefan
graceful dead
hello list, just a quick question: is there any Apache api to request from within a module a child process (or thread for that matter) to die? I have a module where returning 500 followed by a suicide seems to be the best way to deal with some error conditions. TIA Ralf Mattes
Re: graceful?
Cliff Woolley wrote: > > > Has anybody torture tested graceful restarts lately? I only ask because > we just got a PR that gave me "that sinking feeling". Maybe not a real > problem, but just figured I'd ask. > Not me lately... What's the # ? -- === Jim Jagielski [|] [EMAIL PROTECTED] [|] http://www.jaguNET.com/ "A society that will trade a little liberty for a little order will lose both and deserve neither" - T.Jefferson
Re: graceful?
On Mon, 23 Sep 2002, Jim Jagielski wrote: > Not me lately... What's the # ? Oy... um, the last one? ;-] http://nagoya.apache.org/bugzilla/show_bug.cgi?id=12033 Oh, and not to startle the 1.3.27 RM... I should have mentioned it's a 2.0 PR. =] --Cliff
Re: graceful?
Cliff Woolley wrote: > > On Mon, 23 Sep 2002, Jim Jagielski wrote: > > > Not me lately... What's the # ? > > Oy... um, the last one? ;-] > > http://nagoya.apache.org/bugzilla/show_bug.cgi?id=12033 > > Oh, and not to startle the 1.3.27 RM... I should have mentioned it's a 2.0 > PR. =] > I assumed it was 2.0 :) -- === Jim Jagielski [|] [EMAIL PROTECTED] [|] http://www.jaguNET.com/ "A society that will trade a little liberty for a little order will lose both and deserve neither" - T.Jefferson
Re: graceful?
Cliff Woolley wrote: > > Has anybody torture tested graceful restarts lately? I only ask because > we just got a PR that gave me "that sinking feeling". Maybe not a real > problem, but just figured I'd ask. They work on daedalus with prefork. But that's typically just once a night - not sure if that qualifies as a torture test. Greg
Re: graceful?
On Mon, 23 Sep 2002 [EMAIL PROTECTED] wrote: > > Has anybody torture tested graceful restarts lately? I only ask because > > we just got a PR that gave me "that sinking feeling". Maybe not a real > > problem, but just figured I'd ask. > > They work on daedalus with prefork. But that's typically just once a night - > not sure if that qualifies as a torture test. It fails sometimes for me on Linux (pretty standard RedHat 7.3) with the worker MPM. (The original "parent" process (if that's what you call it) just hangs until you kill it). Actually, it might not have failed since sometime around .40; I haven't paid that much attention. I will try to look closer if it happens again. (how was that for a useless bug report? :-) ) - ask -- ask bjoern hansen, http://www.askbjoernhansen.com/ !try; do();
Re: Re: Re: Graceful restart not so graceful?
On Jan 12, 2009 8:49am, traw...@gmail.com wrote: wrt backporting: I think that modern Linux and modern Solaris, perhaps the only platforms tested so far, both work for the same reason under the covers. Has anyone tested with regular poll()? If not, I can try to test another platform or at least dumb down APR to bypass epoll/event queue/etc. I tested prefork on OpenSolaris with ac_cv_func_port_create=no (ie, use poll()) and it worked with/without the patch. I tested on Leopard (kevent) and it appeared to work with/without the patch, but I'm not very confident that I was able to reproduce the right timing window. I've proposed for backport to 2.2.x.
Re: Graceful stop
Hello, CM>Why does graceful restart exist? Wouldn't it be better to have CM>a graceful stop, and then just start a new instance? CM>... CM> * httpd could be upgraded without dropping clients I don't see how this would work--a graceful stop followed by a start would mean clients would drop in between the window when you signal the graceful stop (and Apache therefore stops accepting new requests) and when the new instance starts. Humbly, Andrew -- Andrew Ho http://www.tellme.com/ [EMAIL PROTECTED] Engineer1-800-555-TELL Voice 650-930-9062 Tellme Networks, Inc. Fax 650-930-9101 --
Re: Graceful stop
On Tue, Jan 20, 2004 at 10:17:07AM -0800, Andrew Ho wrote: > I don't see how this would work--a graceful stop followed by a start would > mean clients would drop in between the window when you signal the graceful > stop (and Apache therefore stops accepting new requests) and when the new > instance starts. The idea behind a graceful restart is that it won't destroy current requests, it already adds a delay to serving new requets during the restart - and frequently drops them (depending on your queue size). With a graceful stop, and then a start, the outage can be pretty minimal. Ideally you would have the process issuing the signal to be the new instance of httpd, so it wait until it's exactly ready to listen() before calling kill(). I'd wager you'd drop less new connections this way than the current one, and in any case I think the benifits of upgradability are much more worth it. -- Colm MacCárthaighPublic Key: [EMAIL PROTECTED]
Re: graceful restart
On Tue, Jan 5, 2016 at 9:21 AM, Stefan Eissing wrote: > To the Knowledgable out there: how can I detect that the server is gracefully > restarting? > > Backlground is that HTTP/2 > 1. is sometimes in the state of being IDLE, but unable to return to the MPM, > *could* however terminate the HTTP2 session. > 2. HTTP2 sessions can be brought down by first announcing that no more > streams will be processed and then closing, once the existing ones are done. > > Thanks for the help. More generally, do you want to know if the child process is trying to gracefully exit? I think currently that is tied up in the MPMs and would need to be exposed, but let's wait for others. who will know better.
Re: graceful restart
On Tue, Jan 5, 2016 at 8:24 AM, Eric Covener wrote: > On Tue, Jan 5, 2016 at 9:21 AM, Stefan Eissing > wrote: > > To the Knowledgable out there: how can I detect that the server is > gracefully restarting? > > > > Backlground is that HTTP/2 > > 1. is sometimes in the state of being IDLE, but unable to return to the > MPM, *could* however terminate the HTTP2 session. > > 2. HTTP2 sessions can be brought down by first announcing that no more > streams will be processed and then closing, once the existing ones are done. > > > > Thanks for the help. > > More generally, do you want to know if the child process is trying to > gracefully exit? > > I think currently that is tied up in the MPMs and would need to be > exposed, but let's wait for others. > who will know better. > ap_mpm_query should answer such questions, we should expand the answer-set if not. Does AP_MPMQ_MPM_STATE report AP_MPMQ_STOPPING? If not there is clearly a fourth state, AP_MPMQ_CHILD_STOPPING we should report. The API change will be a bit of a hassle for existing modules though... if they wanted to truly know that the parent will still be running or is stopping.
Re: graceful restart
On Tue, Jan 5, 2016 at 11:26 AM, William A Rowe Jr wrote: > If not there is clearly a fourth state, AP_MPMQ_CHILD_STOPPING we should > report. +1
Re: graceful restart
Thanks, I will investigate tomorrow and report back. > Am 05.01.2016 um 17:26 schrieb William A Rowe Jr : > > On Tue, Jan 5, 2016 at 8:24 AM, Eric Covener wrote: > On Tue, Jan 5, 2016 at 9:21 AM, Stefan Eissing > wrote: > > To the Knowledgable out there: how can I detect that the server is > > gracefully restarting? > > > > Backlground is that HTTP/2 > > 1. is sometimes in the state of being IDLE, but unable to return to the > > MPM, *could* however terminate the HTTP2 session. > > 2. HTTP2 sessions can be brought down by first announcing that no more > > streams will be processed and then closing, once the existing ones are done. > > > > Thanks for the help. > > More generally, do you want to know if the child process is trying to > gracefully exit? > > I think currently that is tied up in the MPMs and would need to be > exposed, but let's wait for others. > who will know better. > > ap_mpm_query should answer such questions, we should expand the answer-set if > not. > > Does AP_MPMQ_MPM_STATE report AP_MPMQ_STOPPING? If not there is clearly a > fourth state, AP_MPMQ_CHILD_STOPPING we should report. The API change will > be a bit of a hassle for existing modules though... if they wanted to truly > know that the parent will still be running or is stopping. > > > > > > >
Re: graceful restart
On Tue, Jan 5, 2016 at 10:27 AM, Eric Covener wrote: > On Tue, Jan 5, 2016 at 11:26 AM, William A Rowe Jr > wrote: > > If not there is clearly a fourth state, AP_MPMQ_CHILD_STOPPING we > should > > report. > > +1 > Better yet, an AP_MPMQ_CHILD_STATE that uses the same response set but distinguishes server-stopping from process-stopping. In the single process -X model, the two responses would be identical.
Regarding graceful restart
Hi Apache Gurus,
I am seeing the following issue when I sent a graceful restart signal to
httpd [run with proxy enabled]
Processes on receiving SIGUSR1 seems to send a GET / HTTP/1.0 to the httpd
itself.
This seems to be a dummy request that each process is trying to send
127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284
127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284
127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284
127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284
127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284
Wondering if this is a known issue, and if there is any solution for it.
I saw recent bug report regarding strange behaviour of mod_ssl on graceful
restart, but in my case, I dont have mod_ssl enabled.
Initially I saw this issue even when processes are in LISTEN state and
sigusr1 is sent.
I put in the following fix and that worked.
File : httpd/server/mpm/prefork/prefork.c
Function :child_main
+ if (ap_mpm_pod_check(pod) == APR_SUCCESS) { /* selected as idle? */
+ ap_log_error(APLOG_MARK, APLOG_ERR, status, ap_server_conf,
+ "Dont process connection as graceful restart received from
pod");
+ break;
+ }
+ if (die_now) {
+ ap_log_error(APLOG_MARK, APLOG_ERR, status, ap_server_conf,
+"Dont process connection as graceful restart received");
+ break;
+ }
current_conn = ap_run_create_connection(ptrans, ap_server_conf, csd,
my_child_num, sbh, bucket_alloc);
if (current_conn) {
ap_process_connection(current_conn, csd);
ap_lingering_close(current_conn);
}
However, for connections in ESTABLISHED state and other TCP states, I still
see this issue.
Your inputs would be of great help!
Thanks & Regards
Devi
mod_fcgid, graceful restart
Hi. It seems to me that graceful restart doesn't work in mod_fcgid at all. My configuration: FcgidProcessTableFile /tmp/fcgidshm_aux7 FcgidIPCDir /tmp/.fcgidsock_aux7 FcgidMinProcessesPerClass 0 FcgidIdleTimeout 300 FcgidIdleScanInterval 1 FcgidProcessLifeTime 150 FcgidErrorScanInterval 150 FcgidFixPathinfo 1 ... FcgidWrapper /full/path/to/wrapper .php FcgidIOTimeout 150 I made a simple script to check my guess: $ cat sleep.php Checking without Apache restart: $ time wget http://test-aux7/sleep.php --2010-01-12 16:53:08-- http://test-l12-aux7/sleep.php Resolving test-aux7... 111.222.11.22 Connecting to test-aux7|111.222.11.22|:80... connected. HTTP request sent, awaiting response... 200 OK Length: unspecified [text/html] Saving to: `sleep.php' [ <=> ] 2 --.-K/s in 0s 2010-01-12 16:54:08 (258 KB/s) - `sleep.php' saved real1m0.466s user0m0.000s sys 0m0.004s $ cat sleep.php OK Here is everything ok. Now with graceful restart: time wget http://test-aux7:80/sleep.php --2010-01-12 16:57:15-- http://test-aux7:80/sleep.php Resolving test-aux7... 111.222.11.22 Connecting to test-aux7|111.222.11.22|:80... connected. HTTP request sent, awaiting response... No data received. Retrying. --2010-01-12 16:57:22-- (try: 2) http://test-aux7:80/sleep.php Connecting to test-aux7|111.222.11.22|:80... connected. HTTP request sent, awaiting response... Here what goes in error_log: [Tue Jan 12 16:55:27 2010] [emerg] [client 10.0.2.11] (22)Invalid argument: mod_fcgid: can't lock process table in pid 4172 -- BRGDS. Alexey Vlasov.
mod_fcgid graceful restarts
Hi All, Currently graceful restart while using mod_fcgid just kill all subprocesses, this is not sefe for applications and slows down reloads. John Lightsey provided a patch to make real graceful restarts on mod_fcgid, now graceful part is separated as requested in the bug report https://issues.apache.org/bugzilla/show_bug.cgi?id=48769 I am running with this patch on over 10 machines doing daily 24+ graceful restarts, so far without any issues (reloads are faster, and there are no orphan left behind.) Regards, Michal Grzedzicki
Threaded MPM Graceful segfaults
Howdy,I adjusted the patch from Yann ( https://svn.apache.org/viewvc?view=revision&revision=1783849 ) and got it compiling into Apache 2.4 mainline. The patch I’m using is attached to this email.During testing of this patch, we noticed that CGId is now crashing and not correctly handling the restarts properly. This causes any following requests to be returned with a 503.[Thu Mar 02 12:28:40.926742 2017] [cgid:error] [pid 7247:tid 13133972608] (98)Address already in use: AH01243: Couldn't bind unix domain socket /etc/apache2/run/cgid_sock.6322[Thu Mar 02 12:28:41.927489 2017] [cgid:crit] [pid 6322:tid 13133972608] AH01238: cgid daemon failed to initialize[Thu Mar 02 12:37:19.159596 2017] [cgid:error] [pid 8552:tid 139765481343104] (98)Address already in use: AH01243: Couldn't bind unix domain socket /etc/apache2/run/cgid_sock.7971[Thu Mar 02 12:37:19.168898 2017] [cgid:crit] [pid 7971:tid 139765481343104] AH01238: cgid daemon failed to initialize[Thu Mar 02 12:42:50.097123 2017] [cgid:error] [pid 8876:tid 139765481343104] (98)Address already in use: AH01243: Couldn't bind unix domain socket /etc/apache2/run/cgid_sock.7971[Thu Mar 02 12:42:51.097846 2017] [cgid:crit] [pid 7971:tid 139765481343104] AH01238: cgid daemon failed to initialize[Thu Mar 02 12:42:59.215291 2017] [cgid:error] [pid 9046:tid 139765481343104] (98)Address already in use: AH01243: Couldn't bind unix domain socket /etc/apache2/run/cgid_sock.7971[Thu Mar 02 12:42:59.220158 2017] [cgid:crit] [pid 7971:tid 139765481343104] AH01238: cgid daemon failed to initializeI was hoping I might be able to get some pointers as to how to solve this issue. As a note, this is the only issue we saw with this patch. We are no longer getting segfaults, just having some issues with CGId. 0002-Handle-Gracefuls-Better.patch Description: Binary data Thanks!—Jacob PerkinsProduct OwnercPanel Inc.jacob.perk...@cpanel.netOffice: 713-529-0800 x 4046Cell: 713-560-8655 smime.p7s Description: S/MIME cryptographic signature
Graceful shutdown in 2.0
Has anyone ever thought about the best way to implement graceful shutdown (perhaps with a timeout) in the server? This would be a useful function to have where there is a cluster of http servers (in a DMZ for instance) proxying requests to a farm backend application servers. If you need to take an http server completely offline, you could issue something like apachectl shutdown_graceful [timeout] and prevent that server from accepting any new connections but allow the existing requests to be completed w/o disruption (subject to a timeout). Bill
Graceful stop implementation nits
Now that the ap_close_listeners() code is committed, implementing a graceful stop is relatively trivial, I already have it working here for me. However there are some complicated nits which I thought I'd solicit feedback on. One of the main benefits of a graceful stop is to allow for upgrading httpd without kicking off long downloads. For this situation, it's desirable that it be possible to do; make install apachectl graceful-stop apachectl start and have it work as the admin would expect, but this leads to some complications. Files like the ScriptSock, ScoreBoardFile, SSLSessionCache, LockFile, LDAPSharedCacheFile and so on will get clobbered. Ther will be two instances of httpd, pointing at the same files. To solve this, I'd like allow the administrator do something like; ScoreBoardFile /var/run/apache_status-%p ScriptSock cgi_sock.%p SSLSessionCache shm:/path/to/datafile.%p where %p is the PID of the server. Or even better I'd like to be able to do; # Create a directory for server-related state, gets # removed on server exit. StateDirectory %p.state/ ScoreBoardFile $statedir/apache_status ScriptSock $statedir/cgi_sock SSLSessionCache shm:$statedir/datafile But either way, it starts to look like an embedded macro engine, and it's not at all clear to admin that they can't use those expansions elsewhere. This is a mean problem. What's really needed is a lightweight embedded scripting engine for config parsing, but that's a lot of overkill for simply making graceful stops working. Has anyone got any comments on any other approaches? or on the suitability of adding a small ammount of expansion to the state-related directives above? -- Colm MacCárthaighPublic Key: [EMAIL PROTECTED]
Re: Regarding graceful restart
On Feb 5, 2007, at 12:44 PM, Devi Krishna wrote: Wondering if this is a known issue, and if there is any solution for it. Not having written the code, I think the Pipe Of Death (POD) is a mechanism for the server to draw the attention of workers that are sitting in accept(). Receiving a request causes them to return from accept() and become aware of the restart or shutdown signal. S. -- [EMAIL PROTECTED]http://www.temme.net/sander/ PGP FP: 51B4 8727 466A 0BC3 69F4 B7B8 B2BE BC40 1529 24AF smime.p7s Description: S/MIME cryptographic signature
Re: Regarding graceful restart
Hi Sander Thanks for the quick response ! However, I am surprised that when it returns from accept, why would accept return a valid socket id and not -1 ? Thanks & Regards Devi On 2/5/07, Sander Temme <[EMAIL PROTECTED]> wrote: On Feb 5, 2007, at 12:44 PM, Devi Krishna wrote: > Wondering if this is a known issue, and if there is any solution > for it. Not having written the code, I think the Pipe Of Death (POD) is a mechanism for the server to draw the attention of workers that are sitting in accept(). Receiving a request causes them to return from accept() and become aware of the restart or shutdown signal. S. -- [EMAIL PROTECTED]http://www.temme.net/sander/ PGP FP: 51B4 8727 466A 0BC3 69F4 B7B8 B2BE BC40 1529 24AF
Re: Regarding graceful restart
Hi,
Resending this mail, just in case anyone would have suggestions/inputs as
how to fix this for connections that are in the ESTABLISHED state or FIN
state or any other TCP state other than LISTEN
Thanks & Regards
Devi
On 2/5/07, Devi Krishna <[EMAIL PROTECTED]> wrote:
Hi Apache Gurus,
I am seeing the following issue when I sent a graceful restart signal to
httpd [run with proxy enabled]
Processes on receiving SIGUSR1 seems to send a GET / HTTP/1.0 to the
httpd
itself.
This seems to be a dummy request that each process is trying to send
127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284
127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284
127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284
127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284
127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284
Wondering if this is a known issue, and if there is any solution for it.
I saw recent bug report regarding strange behaviour of mod_ssl on graceful
restart, but in my case, I dont have mod_ssl enabled.
Initially I saw this issue even when processes are in LISTEN state and
sigusr1 is sent.
I put in the following fix and that worked.
File : httpd/server/mpm/prefork/prefork.c
Function :child_main
+ if (ap_mpm_pod_check(pod) == APR_SUCCESS) { /* selected as idle?
*/
+ ap_log_error(APLOG_MARK, APLOG_ERR, status, ap_server_conf,
+ "Dont process connection as graceful restart received from
pod");
+ break;
+ }
+ if (die_now) {
+ ap_log_error(APLOG_MARK, APLOG_ERR, status, ap_server_conf,
+"Dont process connection as graceful restart received");
+ break;
+ }
current_conn = ap_run_create_connection(ptrans, ap_server_conf,
csd,
my_child_num, sbh, bucket_alloc);
if (current_conn) {
ap_process_connection(current_conn, csd);
ap_lingering_close(current_conn);
}
However, for connections in ESTABLISHED state and other TCP states, I
still
see this issue.
Your inputs would be of great help!
Thanks & Regards
Devi
Re: Regarding graceful restart
tor 2007-02-08 klockan 17:15 -0800 skrev Devi Krishna: > Hi, > > Resending this mail, just in case anyone would have > suggestions/inputs as how to fix this for connections that are in the > ESTABLISHED state or FIN state or any other TCP state other than > LISTEN Maybe change the "wake up call" to just connect briefly without actually sending a full HTTP request? This should be sufficient to wake up any processes sleeping in accept() and will not cause anything to get processed.. But I am not sure I understand the original problem so I may be completely off here.. Regards Henrik signature.asc Description: Detta är en digitalt signerad meddelandedel
Re: Regarding graceful restart
> -Ursprüngliche Nachricht- > Von: Henrik Nordstrom > Gesendet: Freitag, 9. Februar 2007 16:33 > An: dev@httpd.apache.org > Betreff: Re: Regarding graceful restart > > > tor 2007-02-08 klockan 17:15 -0800 skrev Devi Krishna: > > Hi, > > > > Resending this mail, just in case anyone would have > > suggestions/inputs as how to fix this for connections that > are in the > > ESTABLISHED state or FIN state or any other TCP state other than > > LISTEN > > Maybe change the "wake up call" to just connect briefly > without actually > sending a full HTTP request? This should be sufficient to wake up any > processes sleeping in accept() and will not cause anything to get > processed.. Not if BSD accept filters are in place. In this case the kernel waits until it sees a HTTP request until it wakes up the process. And on Linux with TCP_DEFER_ACCEPT enabled you need to sent a least one byte of data. Regards Rüdiger
Re: Regarding graceful restart
fre 2007-02-09 klockan 18:34 +0100 skrev Plüm, Rüdiger, VF EITO: > Not if BSD accept filters are in place. In this case the kernel waits until it > sees a HTTP request until it wakes up the process. > And on Linux with TCP_DEFER_ACCEPT enabled you need to sent a least one byte > of data. So send two blank lines. Should satisfy both.. Regards Henrik signature.asc Description: Detta är en digitalt signerad meddelandedel
Re: Regarding graceful restart
Hi Folks, Thanks for the replies. What seems to be happening in my case , is when I send a graceful restart to apache, I get the following messages in the access log 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 I am trying to find the root cause of what cause this request when we do graceful restart. I have been digging around for some days but not able to find out which code is causing this. Any cluse ? Thanks & Regards Devi On 2/9/07, Plüm, Rüdiger, VF EITO <[EMAIL PROTECTED]> wrote: > -Ursprüngliche Nachricht- > Von: Henrik Nordstrom > Gesendet: Freitag, 9. Februar 2007 16:33 > An: dev@httpd.apache.org > Betreff: Re: Regarding graceful restart > > > tor 2007-02-08 klockan 17:15 -0800 skrev Devi Krishna: > > Hi, > > > > Resending this mail, just in case anyone would have > > suggestions/inputs as how to fix this for connections that > are in the > > ESTABLISHED state or FIN state or any other TCP state other than > > LISTEN > > Maybe change the "wake up call" to just connect briefly > without actually > sending a full HTTP request? This should be sufficient to wake up any > processes sleeping in accept() and will not cause anything to get > processed.. Not if BSD accept filters are in place. In this case the kernel waits until it sees a HTTP request until it wakes up the process. And on Linux with TCP_DEFER_ACCEPT enabled you need to sent a least one byte of data. Regards Rüdiger
Re: Regarding graceful restart
Please have a look at dummy_connection in mpm_common.c Regards Rüdiger On 02/10/2007 05:15 AM, Devi Krishna wrote: > Hi Folks, Thanks for the replies. > What seems to be happening in my case , is when I send a graceful > restart to > apache, I get the following messages in the access log > > > 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 > 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 > 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 > 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 > 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 > > I am trying to find the root cause of what cause this request when we do > graceful restart. > > I have been digging around for some days but not able to find out which > code > is causing this. > > Any cluse ? > > Thanks & Regards > Devi > > > > On 2/9/07, Plüm, Rüdiger, VF EITO <[EMAIL PROTECTED]> wrote: > >> >> >> >> > -----Ursprüngliche Nachricht- >> > Von: Henrik Nordstrom >> > Gesendet: Freitag, 9. Februar 2007 16:33 >> > An: dev@httpd.apache.org >> > Betreff: Re: Regarding graceful restart >> > >> > >> > tor 2007-02-08 klockan 17:15 -0800 skrev Devi Krishna: >> > > Hi, >> > > >> > > Resending this mail, just in case anyone would have >> > > suggestions/inputs as how to fix this for connections that >> > are in the >> > > ESTABLISHED state or FIN state or any other TCP state other than >> > > LISTEN >> > >> > Maybe change the "wake up call" to just connect briefly >> > without actually >> > sending a full HTTP request? This should be sufficient to wake up any >> > processes sleeping in accept() and will not cause anything to get >> > processed.. >> >> Not if BSD accept filters are in place. In this case the kernel waits >> until it >> sees a HTTP request until it wakes up the process. >> And on Linux with TCP_DEFER_ACCEPT enabled you need to sent a least one >> byte of data. >> >> Regards >> >> Rüdiger >> >> > >
Re: Regarding graceful restart
Hi Ruediger, Thats exactly what I was looking for ! Thanks so much!!! Thanks & Regards Devi On 2/10/07, Ruediger Pluem <[EMAIL PROTECTED]> wrote: Please have a look at dummy_connection in mpm_common.c Regards Rüdiger On 02/10/2007 05:15 AM, Devi Krishna wrote: > Hi Folks, Thanks for the replies. > What seems to be happening in my case , is when I send a graceful > restart to > apache, I get the following messages in the access log > > > 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 > 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 > 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 > 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 > 127.0.0.1 [05/Feb/2007:13:35:25 +] "GET / HTTP/1.0" 400 284 > > I am trying to find the root cause of what cause this request when we do > graceful restart. > > I have been digging around for some days but not able to find out which > code > is causing this. > > Any cluse ? > > Thanks & Regards > Devi > > > > On 2/9/07, Plüm, Rüdiger, VF EITO <[EMAIL PROTECTED]> wrote: > >> >> >> >> > -Ursprüngliche Nachricht- >> > Von: Henrik Nordstrom >> > Gesendet: Freitag, 9. Februar 2007 16:33 >> > An: dev@httpd.apache.org >> > Betreff: Re: Regarding graceful restart >> > >> > >> > tor 2007-02-08 klockan 17:15 -0800 skrev Devi Krishna: >> > > Hi, >> > > >> > > Resending this mail, just in case anyone would have >> > > suggestions/inputs as how to fix this for connections that >> > are in the >> > > ESTABLISHED state or FIN state or any other TCP state other than >> > > LISTEN >> > >> > Maybe change the "wake up call" to just connect briefly >> > without actually >> > sending a full HTTP request? This should be sufficient to wake up any >> > processes sleeping in accept() and will not cause anything to get >> > processed.. >> >> Not if BSD accept filters are in place. In this case the kernel waits >> until it >> sees a HTTP request until it wakes up the process. >> And on Linux with TCP_DEFER_ACCEPT enabled you need to sent a least one >> byte of data. >> >> Regards >> >> Rüdiger >> >> > >
leak on graceful restarts
Looking at a problem that seems easy to re-produce using un-patched trunk, 2.2.10 and 2.0.63. Using a graceful restart causes higher memory usage in the parent, which is then passed on to the 'new' children processes. The issue seems to appear in all the Worker, Event and Prefork MPMs. Compile up httpd without apr pool debugging enabled, start it as normal, and then check the memory usage: $ ./apachectl start $ ps alx | grep 18459 1 1000 18459 1 20 0 152752 2672 674009 Ss ? 0:00 /home/chip/temp/httpd/bin/httpd -k start $ ./apachectl graceful $ ps alx | grep 18459 1 1000 18459 1 20 0 152752 3192 674009 Ss ? 0:00 /home/chip/temp/httpd/bin/httpd -k start $ ./apachectl graceful $ ps alx | grep 18459 1 1000 18459 1 20 0 152752 3236 674009 Ss ? 0:00 /home/chip/temp/httpd/bin/httpd -k start $ ./apachectl graceful $ ps alx | grep 18459 1 1000 18459 1 20 0 152752 3280 674009 Ss ? 0:00 /home/chip/temp/httpd/bin/httpd -k start (2672 -> 3192 -> 3236 -> 3280) And, at least over here, httpd consistently grows in RSS, without any obvious cause. Seems reproducible on Ubuntu and Darwin, using 2.2.10, 2.0.63 and trunk. Any ideas? Thanks, Paul
graceful-stop and SIGWINCH
Hi everyone, A couple of days a go i noticed that apache2ctl uses the signal SIGWINCH to send a graceful-stop. This is obviously a very bad idea. e.g. try starting httpd -X and change your window size ;) I am proposing to use the SIGUSR2 signal (see the diff). The first diff is for apache2 and the last one for apr. Please note that i don't know the apache or apr stuff that well. Regards, Alexander Index: include/mpm_common.h === --- include/mpm_common.h(revision 780410) +++ include/mpm_common.h(working copy) @@ -72,13 +72,13 @@ #define AP_SIG_GRACEFUL_STRING "SIGUSR1" /* Signal used to gracefully stop */ -#define AP_SIG_GRACEFUL_STOP SIGWINCH +#define AP_SIG_GRACEFUL_STOP SIGUSR2 /* Signal used to gracefully stop (without SIG prefix) */ -#define AP_SIG_GRACEFUL_STOP_SHORT WINCH +#define AP_SIG_GRACEFUL_STOP_SHORT USR2 /* Signal used to gracefully stop (as a quoted string) */ -#define AP_SIG_GRACEFUL_STOP_STRING "SIGWINCH" +#define AP_SIG_GRACEFUL_STOP_STRING "SIGUSR2" /** * Make sure all child processes that have been spawned by the parent process Index: threadproc/unix/signals.c === --- threadproc/unix/signals.c (revision 780413) +++ threadproc/unix/signals.c (working copy) @@ -307,13 +307,7 @@ /* the rest of the signals removed from the mask in this function * absolutely must be removed; you cannot block synchronous signals * (requirement of pthreads API) - * - * SIGUSR2 is being removed from the mask for the convenience of - * Purify users (Solaris, HP-UX, SGI) since Purify uses SIGUSR2 */ -#ifdef SIGUSR2 -sigdelset(sig_mask, SIGUSR2); -#endif } APR_DECLARE(apr_status_t) apr_signal_thread(int(*signal_handler)(int signum))
Re: mod_fcgid, graceful restart
On a slightly related note - I have seen that the cgi-handler continues to function for a GET request after a graceful restart, but not for a POST request. Cheers, Brad On Wed, 13 Jan 2010 12:57:10 +0300 Alexey Vlasov wrote: > Hi. > > It seems to me that graceful restart doesn't work in mod_fcgid at all. > > My configuration: > > FcgidProcessTableFile /tmp/fcgidshm_aux7 > FcgidIPCDir /tmp/.fcgidsock_aux7 > FcgidMinProcessesPerClass 0 > FcgidIdleTimeout 300 > FcgidIdleScanInterval 1 > FcgidProcessLifeTime 150 > FcgidErrorScanInterval 150 > FcgidFixPathinfo 1 > > ... > > > FcgidWrapper /full/path/to/wrapper .php > FcgidIOTimeout 150 > > > > I made a simple script to check my guess: > $ cat sleep.php > > > Checking without Apache restart: > $ time wget http://test-aux7/sleep.php > --2010-01-12 16:53:08-- http://test-l12-aux7/sleep.php > Resolving test-aux7... 111.222.11.22 > Connecting to test-aux7|111.222.11.22|:80... connected. > HTTP request sent, awaiting response... 200 OK > Length: unspecified [text/html] > Saving to: `sleep.php' > > [ <=> >] 2 --.-K/s in 0s > > 2010-01-12 16:54:08 (258 KB/s) - `sleep.php' saved > > real1m0.466s > user0m0.000s > sys 0m0.004s > > $ cat sleep.php > OK > Here is everything ok. > > Now with graceful restart: > time wget http://test-aux7:80/sleep.php > --2010-01-12 16:57:15-- http://test-aux7:80/sleep.php > Resolving test-aux7... 111.222.11.22 > Connecting to test-aux7|111.222.11.22|:80... connected. > HTTP request sent, awaiting response... No data received. > Retrying. > > --2010-01-12 16:57:22-- (try: 2) http://test-aux7:80/sleep.php > Connecting to test-aux7|111.222.11.22|:80... connected. > HTTP request sent, awaiting response... > > Here what goes in error_log: > [Tue Jan 12 16:55:27 2010] [emerg] [client 10.0.2.11] (22)Invalid argument: > mod_fcgid: can't lock process table in pid 4172 >
Solaris graceful restart issue
On Mon, Feb 27, 2012 at 4:51 PM, csross wrote: > > Hi, > > It was suggested that I try and use apr 1.3.9 and apu 1.3.9 to try and stop > the problem with graceful restarts and processes being stuck in "G" state if > you have more than 1 listener (ie) port 80 and port 443. I have been trying > the newer versions of apache and I still have that problem with 2.2.22 > (compile on solaris with gcc). The only way to clear them is to restart > apache. > > Can I download apr and apr-util 1.3.9, and specify to use both apr and > apr-util on the ./configure line? > > Do you know if there are any plans to fix this issue? What bug number(s) apply to this issue? (I assume that a suggestion to use 1.3.9 of either apr or apr-util is a rather old one.) > > thanks > > > > Ruediger Pluem wrote: >> >> >> >> Mikhail T. wrote: >>> On 27.12.2011 08:24, Ruediger Pluem wrote: >>>> Some modules in 2.3 require the apr-util crypto API. These >>>> won't work with older apr-util versions. >>> Oh, Ok -- so a module may just not be built, if apr(-util) is too old at >>> compile time. But if it is available, it is >>> fully-featured, right? In other words, there are no silent #ifs in the >>> code like: >>> >>> #if APR_UTIL_VERSION < 1.4 >>> disable some really cool feature or performance improvement >>> #endif >> >> Not directly, but you need to have apr-util 1.4 available during compile >> time to get these modules compiled. >> >> Regards >> >> Rüdiger >> >> > > -- > View this message in context: > http://old.nabble.com/which-apr-to-use%2C-version-numbering-confusion-tp33039336p33402972.html > Sent from the Apache HTTP Server - Dev mailing list archive at Nabble.com. > -- Born in Roswell... married an alien...
trunk crash on graceful
Hi, when I run the mod_http2 tests on trunk, I get a crash when they do a graceful restart during a h2 download. This is relatively new and it does not happen on 2.4.x. In case someone gets bored or the weather is dull and cloudy in southern France, someone might have a look at this: Cheers, Stefan Crashed Thread:1 Exception Type:EXC_CRASH (SIGSEGV) Exception Codes: 0x, 0x Exception Note:EXC_CORPSE_NOTIFY Termination Signal:Segmentation fault: 11 Termination Reason:Namespace SIGNAL, Code 0xb Terminating Process: httpd [526] Application Specific Information: crashed on child side of fork pre-exec Thread 0:: Dispatch queue: com.apple.main-thread 0 libsystem_kernel.dylib 0x7fff7f4a12c0 __sigreturn + 0 1 libsystem_platform.dylib0x7fff7f5d2f6d _sigtramp + 45 2 ??? 0x1fa0 0 + 281470681751456 3 libapr-1.0.dylib0x00010ad2e310 apr_pool_destroy + 128 (apr_pools.c:820) 4 mod_mpm_event.so0x00010b1c222b clean_child_exit + 43 (event.c:770) 5 mod_mpm_event.so0x00010b1c2103 child_main + 899 (event.c:2725) 6 mod_mpm_event.so0x00010b1c1d76 make_child + 470 7 mod_mpm_event.so0x00010b1c10a0 event_run + 2880 (event.c:2957) 8 httpd 0x00010abc630b ap_run_mpm + 75 (mpm_common.c:101) 9 httpd 0x00010abb8149 main + 2233 (main.c:844) 10 libdyld.dylib 0x7fff7f351115 start + 1 Thread 1 Crashed: 0 libsystem_kernel.dylib 0x7fff7f4a0fca __select + 10 1 libapr-1.0.dylib0x00010ad394f9 apr_sleep + 73 (time.c:248) 2 mod_watchdog.so 0x00010ae562fb wd_worker + 155 3 libsystem_pthread.dylib 0x7fff7f5dc6c1 _pthread_body + 340 4 libsystem_pthread.dylib 0x7fff7f5dc56d _pthread_start + 377 5 libsystem_pthread.dylib 0x7fff7f5dbc5d thread_start + 13
Errors while using graceful shutdown
Hi, While running the bench marking tool, ab (options are -c 30 -n 400), on a cgi script that takes about 30 seconds to complete, I notice the following error in the logfile when I do a graceful shutdown. This is on 2.0.55 version of Apache. The error I get is : [Thu Mar 02 17:23:11 2006] [error] [client 15.42.227.146] daemon couldn't find C GI process for connection 5 [Thu Mar 02 17:23:11 2006] [error] [client 15.42.227.146] daemon couldn't find C GI process for connection 67 For every concurrent request issued, there seems to be an error. I noticed that CGI daemon dies immediately after a graceful shutdown is issued and the message is printed by the other httpd children that were spawned before the graceful shutdown is issued. Any idea why this should happen ? Thanks, Kiran
Re: Graceful shutdown in 2.0
The same effect is already possible by configuring your proxying machine to stop forwarding new requests to that box first Of course, it's possible that different people manage the proxying service vs the back end apache services, so I can see how it can be desireable to have this feature in apache too, but still those two people should always be working pretty closely together anyway... Dave - Original Message - From: "Bill Stoddard" <[EMAIL PROTECTED]> To: <[EMAIL PROTECTED]> Sent: Tuesday, February 04, 2003 8:53 AM Subject: Graceful shutdown in 2.0 > Has anyone ever thought about the best way to implement graceful > shutdown (perhaps with a timeout) in the server? This would be a > useful function to have where there is a cluster of http servers (in a > DMZ for instance) proxying requests to a farm backend application > servers. If you need to take an http server completely offline, you > could issue something like apachectl shutdown_graceful [timeout] and > prevent that server from accepting any new connections but allow the > existing requests to be completed w/o disruption (subject to a timeout). > > Bill >
Re: Graceful shutdown in 2.0
David Burry wrote: The same effect is already possible by configuring your proxying machine to stop forwarding new requests to that box first Yep, that's the idea. In the scenario I'm interested in, Apache httpd -is- the proxy machine! Bill
Re: Graceful shutdown in 2.0
um, but if you're talking about shutting down the proxy itself (i.e. the whole service, cutting off all load balanced machines behind it) that's hardly graceful to begin with so why bother to make it graceful... I assumed you meant just gracefully shutting down one single load balanced machine behind the proxy machine... you can do that already now by a) configuring the proxy machine to stop routing (new) requests to it, b) graceful restart the proxy machine to make the new config go into effect, c) wait till the existing connections to that behind-the-proxy machine are finished with a timeout if necessary (sort of part of the graceful restart process in the proxy machine), then d) shut down the machine behind the proxy, in that order. External users should not notice anything at all in this scenario. - Original Message - From: "Bill Stoddard" <[EMAIL PROTECTED]> To: <[EMAIL PROTECTED]> Sent: Tuesday, February 04, 2003 12:25 PM Subject: Re: Graceful shutdown in 2.0 > David Burry wrote: > > The same effect is already possible by configuring your proxying machine to > > stop forwarding new requests to that box first > > Yep, that's the idea. In the scenario I'm interested in, Apache httpd > -is- the proxy machine! > > Bill > > >
Re: Graceful shutdown in 2.0
David Burry wrote: um, but if you're talking about shutting down the proxy itself (i.e. the whole service, cutting off all load balanced machines behind it) that's hardly graceful to begin with so why bother to make it graceful... If your proxy machines were being load balanced by an Alteon switch (or whatever) you could probably tell the switch to stop sending new connections to the proxy in question, while allowing the old connections to finish. Of course this is based on whether the Alteon (or whatever) config will let you do this of course. On the proxy side, maybe some generic function in Apache that is not proxy specific could signal Apache to go into a "disabled" state, where new connections get "connection refused", while old connections run their course. The load balancer I assume would detect this state and pass any new requests on to other machines. You could use ./apachectl disable and ./apachectl enable to achieve this. Regards, Graham -- - [EMAIL PROTECTED] "There's a moon over Bourbon Street tonight..."
Re: Graceful shutdown in 2.0
--- Graham Leggett <[EMAIL PROTECTED]> wrote: > David Burry wrote: > > > um, but if you're talking about shutting down the proxy itself (i.e. the > > whole service, cutting off all load balanced machines behind it) that's > > hardly graceful to begin with so why bother to make it graceful... > > If your proxy machines were being load balanced by an Alteon switch (or > whatever) you could probably tell the switch to stop sending new > connections to the proxy in question, while allowing the old connections > to finish. Of course this is based on whether the Alteon (or whatever) > config will let you do this of course. > > On the proxy side, maybe some generic function in Apache that is not > proxy specific could signal Apache to go into a "disabled" state, where > new connections get "connection refused", while old connections run > their course. The load balancer I assume would detect this state and > pass any new requests on to other machines. You could use ./apachectl > disable and ./apachectl enable to achieve this. I've wondered about this as well. Even for just single servers, it'd be nice to shutdown the server "gracefully". Often, I'm more comfortable with stopping and then starting the server than doing a reload, so a graceful shutdown would be useful, unless I'm missing something that already exists. Hans
Re: Graceful shutdown in 2.0
At 07:29 AM 2/5/2003, Hans Zaunere wrote: >> David Burry wrote: >> >> > um, but if you're talking about shutting down the proxy itself (i.e. the >> > whole service, cutting off all load balanced machines behind it) that's >> > hardly graceful to begin with so why bother to make it graceful... > >I've wondered about this as well. Even for just single servers, it'd be nice >to shutdown the server "gracefully". Often, I'm more comfortable with >stopping and then starting the server than doing a reload, so a graceful >shutdown would be useful, unless I'm missing something that already exists. With the new interdependencies with DAV content... e.g. live files are being modified while the server is operating, it seems like a forced shutdown of a DAV server could cause havoc. Correct me if I'm off base here. So perhaps graceful shutdown would be a good thing, perhaps even ultra-graceful (allowing open keep alive connections to finish serving all requests.) But at minimum we don't want to always cut off POST or PUT requests mid-stream. Bill
Re: Graceful shutdown in 2.0
Hello, GL>If your proxy machines were being load balanced by an Alteon switch (or GL>whatever) you could probably tell the switch to stop sending new GL>connections to the proxy in question, while allowing the old connections GL>to finish. Of course this is based on whether the Alteon (or whatever) GL>config will let you do this of course. For the Alteons on our production networks, we do exactly this. It works pretty well. GL>On the proxy side, maybe some generic function in Apache that is not GL>proxy specific could signal Apache to go into a "disabled" state, where GL>new connections get "connection refused", while old connections run GL>their course. The load balancer I assume would detect this state and GL>pass any new requests on to other machines. You could use ./apachectl GL>disable and ./apachectl enable to achieve this. This is exactly the graceful shutdown case that was suggested, which I am wholeheartedly supportive of. (Cutting off requests that are in service is, in my opinion, never good; graceful shutdown seems like a really useful feature to have.) On a more load balancer specific note, Alteons (and some other load balancers) use the concept of a health check URL. Our Alteons are configured for example to check for a specific URL (for example, the Alteon might do a "GET alteoncheck.txt HTTP/1.0" every 2 seconds). I had a plan originally to write a handler that accepts requests for this heartbeat check... on some signal (a particular request? an OS signal?) it would start returning an error for the heartbeat check case, but keep servicing all other requests as normal. Eventually, the Alteon would decide that that machine was bad, and the number of connections would fall to zero; it would then be safe to take the server out of rotation. The benefit of this scenario is that you don't have to touch the load balancer at all to get individual machines in and out of the load balancer. Also, this type of scenario is far more automatable (rather than telnetting into, say, a load balancer console interface and navigating menus, ugh). Humbly, Andrew -- Andrew Ho http://www.tellme.com/ [EMAIL PROTECTED] Engineer [EMAIL PROTECTED] Voice 650-930-9062 Tellme Networks, Inc. 1-800-555-TELLFax 650-930-9101 --
Re: Graceful shutdown in 2.0
On our systems we just rename that "alteoncheck.txt" file to "alteoncheck_DOWN.txt" when we're going to bring a server down (causing a 404 error for the health check, which stops all new requests), it effectively does the same thing you describe without the hassle of writing a handler. And yes it is very nice in that it's easily automated... So, yes, it would be very nice to have a graceful shutdown, but it's not necessarily high priority for those who have some sort of load balancer box (not round robin DNS ;) because there are other relatively simple ways to achieve the same effect... Dave - Original Message - From: "Andrew Ho" <[EMAIL PROTECTED]> > On a more load balancer specific note, Alteons (and some other load > balancers) use the concept of a health check URL. Our Alteons are > configured for example to check for a specific URL (for example, the > Alteon might do a "GET alteoncheck.txt HTTP/1.0" every 2 seconds). > > I had a plan originally to write a handler that accepts requests for this > heartbeat check... on some signal (a particular request? an OS signal?) it > would start returning an error for the heartbeat check case, but keep > servicing all other requests as normal. Eventually, the Alteon would > decide that that machine was bad, and the number of connections would fall > to zero; it would then be safe to take the server out of rotation. > > The benefit of this scenario is that you don't have to touch the load > balancer at all to get individual machines in and out of the load > balancer. Also, this type of scenario is far more automatable (rather than > telnetting into, say, a load balancer console interface and navigating > menus, ugh).
prototype for worker graceful *shutdown*
the attached patch, based on one bill started, presses SIGINT
into service as a 'graceful shutdown' signal. (alternative
suggestions highly welcomed.) if the config directive
GracefulShutdownTimeout is set (default is 10), when the
server is sent a SIGINT it will go into shutdown mode but
not actually go all the way down until that many seconds have
passed. SIGTERM works normally for a graceless shutdown.
i'd rather not re-add another signal, but oh well.. the
alternative is to set GracefulShutdownTimeout to zero as
the default (either in .c or .conf) to keep the current
behaviour -- but then to take advantage of it you'd need to
edit the .conf file, do a restart, and then do a shutdown.
bleah.
expansion to other mpms and command-line interface waiting
for non-vetoed acceptance of this proof-of-concept.
--
#kenP-)}
Ken Coar, Sanagendamgagwedweinini http://Golux.Com/coar/
Author, developer, opinionist http://Apache-Server.Com/
"Millennium hand and shrimp!"
Index: configure.in
===
RCS file: /home/cvs/httpd-2.0/configure.in,v
retrieving revision 1.246
diff -u -r1.246 configure.in
--- configure.in8 Mar 2003 13:54:41 - 1.246
+++ configure.in28 Mar 2003 19:30:52 -
@@ -164,6 +164,7 @@
APACHE_SUBST(LTCFLAGS)
AP_SIG_GRACEFUL=USR1
+AP_SIG_GRACEFUL_STOP=INT
case $host in
*-apple-aux3*)
@@ -477,11 +478,14 @@
AC_DEFINE_UNQUOTED(AP_SIG_GRACEFUL, SIG$AP_SIG_GRACEFUL, [Signal used to gracefully
restart])
AC_DEFINE_UNQUOTED(AP_SIG_GRACEFUL_STRING, "SIG$AP_SIG_GRACEFUL", [Signal used to
gracefully restart (as a quoted string)])
AC_DEFINE_UNQUOTED(AP_SIG_GRACEFUL_SHORT, $AP_SIG_GRACEFUL, [Signal used to
gracefully restart (without SIG prefix)])
+AC_DEFINE_UNQUOTED(AP_SIG_GRACEFUL_STOP, SIG$AP_SIG_GRACEFUL_STOP, [Signal used to
gracefully shut down])
AP_SIG_GRACEFUL_SHORT=$AP_SIG_GRACEFUL
AP_SIG_GRACEFUL=SIG$AP_SIG_GRACEFUL_SHORT
+AP_SIG_GRACEFUL_STOP=SIG$AP_SIG_GRACEFUL_STOP
AC_SUBST(AP_SIG_GRACEFUL)
AC_SUBST(AP_SIG_GRACEFUL_STRING)
AC_SUBST(AP_SIG_GRACEFUL_SHORT)
+AC_SUBST(AP_SIG_GRACEFUL_STOP)
dnl check for endianness
if test "$cross_compiling" = "no"; then
Index: server/mpm/worker/worker.c
===
RCS file: /home/cvs/httpd-2.0/server/mpm/worker/worker.c,v
retrieving revision 1.134
diff -u -r1.134 worker.c
--- server/mpm/worker/worker.c 3 Feb 2003 17:53:26 - 1.134
+++ server/mpm/worker/worker.c 28 Mar 2003 19:30:52 -
@@ -157,6 +157,7 @@
*/
int ap_threads_per_child = 0; /* Worker threads per child */
+static int graceful_shutdown_timeout = 10;
static int ap_daemons_to_start = 0;
static int min_spare_threads = 0;
static int max_spare_threads = 0;
@@ -436,7 +437,7 @@
* child to force an exit) and so do an exit anyway.
*/
-static void ap_start_shutdown(void)
+static void ap_start_shutdown(int sig)
{
if (shutdown_pending == 1) {
/* Um, is this _probably_ not an error, if the user has
@@ -445,6 +446,13 @@
*/
return;
}
+/*
+ * If it isn't a graceful shutdown, force it to be graceless by
+ * setting the gracefule timeout to zero.
+ */
+if (sig != AP_SIG_GRACEFUL_STOP) {
+graceful_shutdown_timeout = 0;
+}
shutdown_pending = 1;
}
@@ -462,7 +470,7 @@
static void sig_term(int sig)
{
-ap_start_shutdown();
+ap_start_shutdown(sig);
}
static void restart(int sig)
@@ -1743,11 +1751,16 @@
server_main_loop(remaining_children_to_start);
if (shutdown_pending) {
-/* Time to gracefully shut down:
- * Kill child processes, tell them to call child_exit, etc...
- * (By "gracefully" we don't mean graceful in the same sense as
- * "apachectl graceful" where we allow old connections to finish.)
+/* If a graceful_shutdown_timeout is set, allow some time
+ * for in-flight transactions to complete. If they are not
+ * done by the timeout, take them down hard.
+ * Todo: poll to see if the children have shutdown
+ * to save us from waiting the full timeout period.
*/
+if (graceful_shutdown_timeout != 0) {
+ap_mpm_pod_killpg(pod, ap_daemons_limit, TRUE);
+apr_sleep(apr_time_from_sec(graceful_shutdown_timeout));
+}
ap_mpm_pod_killpg(pod, ap_daemons_limit, FALSE);
ap_reclaim_child_processes(1);/* Start with SIGTERM */
@@ -1937,7 +1950,16 @@
ap_hook_open_logs(worker_open_logs, NULL, aszSucc, APR_HOOK_MIDDLE);
ap_hook_pre_config(worker_pre_config, NULL, NULL, APR_HOOK_MIDDLE);
}
-
+static const char *set_graceful_shutdown(cmd_parms *cmd, void *dummy,
+ const char *arg)
+{
+const char *err = ap_check_cmd_context(cmd, GLOBAL
[PATCH] prefork graceful restart fix
Graceful restarts can be really slow in 2.0 with prefork... the parent is making a dummy connection on the pod MaxClients times, rather than a connection for as many children as it has had; is that really necessary? At least one of the connections will get the 3 second timeout without the below change in my testing, which accounts for the delay. Anecdotally we've seen it take up to a minute or more on idle-ish servers during testing, and had user reports of the same. Index: prefork.c === RCS file: /home/cvs/httpd-2.0/server/mpm/prefork/prefork.c,v retrieving revision 1.279 diff -u -r1.279 prefork.c --- prefork.c 27 Aug 2003 22:33:11 - 1.279 +++ prefork.c 16 Oct 2003 18:57:15 - @@ -1123,7 +1123,7 @@ "Graceful restart requested, doing restart"); /* kill off the idle ones */ -ap_mpm_pod_killpg(pod, ap_daemons_limit); +ap_mpm_pod_killpg(pod, ap_max_daemons_limit); /* This is mostly for debugging... so that we know what is still * gracefully dealing with existing request. This will break
Re: Graceful stop implementation nits
On Thu, Aug 25, 2005 at 11:01:14AM +0100, Colm MacCarthaigh wrote: > Now that the ap_close_listeners() code is committed, implementing a > graceful stop is relatively trivial, I already have it working here for > me. However there are some complicated nits which I thought I'd solicit > feedback on. > > One of the main benefits of a graceful stop is to allow for upgrading > httpd without kicking off long downloads. For this situation, it's > desirable that it be possible to do; > > make install [fixing "make install" to avoid overwriting already-mapped DSOs here would probably useful at some point!] > apachectl graceful-stop > apachectl start > > and have it work as the admin would expect, but this leads to some > complications. Files like the ScriptSock, ScoreBoardFile, > SSLSessionCache, LockFile, LDAPSharedCacheFile and so on will get > clobbered. Ther will be two instances of httpd, pointing at the same > files. I think the right way to fix this is to make the default behaviour DTRT without requiring fancy configuration hacks. - all shmem stuff should be using anonymous shm segments by default on the trunk; so the scoreboard and SSL/LDAP caches should all be fine; do you still see issues there really? - lock files created in the parent should append the parent pid to the filename automatically, LockFile already does in prefork at least. It should be done for ScriptSock too, I think that's been suggested before once already. joe
Re: Graceful stop implementation nits
On Thu, Aug 25, 2005 at 11:17:29AM +0100, Joe Orton wrote:
> > apachectl graceful-stop
> > apachectl start
> >
> > and have it work as the admin would expect, but this leads to some
> > complications. Files like the ScriptSock, ScoreBoardFile,
> > SSLSessionCache, LockFile, LDAPSharedCacheFile and so on will get
> > clobbered. Ther will be two instances of httpd, pointing at the same
> > files.
>
> I think the right way to fix this is to make the default behaviour DTRT
> without requiring fancy configuration hacks.
>
> - all shmem stuff should be using anonymous shm segments by default on
> the trunk; so the scoreboard and SSL/LDAP caches should all be fine; do
> you still see issues there really?
Not for me on Linux, but I'm trying to think out the scenarious where
the filename will matter. From looking at apr, there seem to be some
cases when neither anon shm or mmap("/dev/zero") is available, or is
that near-impossible?
I guess just documenting this loudly in stopping.xml is an approach?
> - lock files created in the parent should append the parent pid to the
> filename automatically, LockFile already does in prefork at least. It
> should be done for ScriptSock too, I think that's been suggested before
> once already.
O.k. that sounds good, LockFile, ScriptSock, and SSLMutex
fcntl|flock|file all need a pid appendage, anyone know am I missing any?
Also, DavLockDB needs be the same for multiple instances right? And
sdbm should work it out.
--
Colm MacCárthaighPublic Key: [EMAIL PROTECTED]
Re: Graceful stop implementation nits
On Aug 25, 2005, at 6:01 AM, Colm MacCarthaigh wrote: Now that the ap_close_listeners() code is committed, implementing a graceful stop is relatively trivial, I already have it working here for me. However there are some complicated nits which I thought I'd solicit feedback on. One of the main benefits of a graceful stop is to allow for upgrading httpd without kicking off long downloads. For this situation, it's desirable that it be possible to do; make install apachectl graceful-stop apachectl start and have it work as the admin would expect, but this leads to some complications. Files like the ScriptSock, ScoreBoardFile, SSLSessionCache, LockFile, LDAPSharedCacheFile and so on will get clobbered. Ther will be two instances of httpd, pointing at the same files. To solve this, I'd like allow the administrator do something like; ScoreBoardFile/var/run/apache_status-%p ScriptSockcgi_sock.%p SSLSessionCache shm:/path/to/datafile.%p where %p is the PID of the server. Or even better I'd like to be able to do; # Create a directory for server-related state, gets # removed on server exit. StateDirectory%p.state/ ScoreBoardFile$statedir/apache_status ScriptSock$statedir/cgi_sock SSLSessionCache shm:$statedir/datafile But either way, it starts to look like an embedded macro engine, and it's not at all clear to admin that they can't use those expansions elsewhere. This is a mean problem. What's really needed is a lightweight embedded scripting engine for config parsing, but that's a lot of overkill for simply making graceful stops working. Has anyone got any comments on any other approaches? or on the suitability of adding a small ammount of expansion to the state- related directives above? Although StateDirectory is nice, I think it would complicate things. Instead, the '%p' idea is cleaner; Of course, it's been proposed before that all server-specific files and resources that are clobberable are internally prepended-appended with PID, as we already do with some. That, I think, would be the best solution.
Re: Graceful stop implementation nits
Quoting Colm MacCarthaigh <[EMAIL PROTECTED]> (2005-08-25 11:01:14 BST): > > Has anyone got any comments on any other approaches? or on the > suitability of adding a small ammount of expansion to the state-related > directives above? I've always felt it was a shame that you could do -DFOO but not -DFOO=bar. If it were allowed, it would be a simple matter to allow definitions to be instantiated from within the config file. -- SOLE LUNDY FASTNET WEST OR NORTHWEST 5 TO 7, BACKING SOUTHWEST 4 OR 5, INCREASING 6 IN WEST SOLE LATER. SHOWERS, RAIN IN SOLE LATER. MAINLY GOOD
Re: leak on graceful restarts
On 10/18/2008 01:25 AM, Paul Querna wrote: > Looking at a problem that seems easy to re-produce using un-patched > trunk, 2.2.10 and 2.0.63. > > Using a graceful restart causes higher memory usage in the parent, which > is then passed on to the 'new' children processes. > > The issue seems to appear in all the Worker, Event and Prefork MPMs. > > Compile up httpd without apr pool debugging enabled, start it as normal, > and then check the memory usage: > > $ ./apachectl start > $ ps alx | grep 18459 > 1 1000 18459 1 20 0 152752 2672 674009 Ss ? 0:00 > /home/chip/temp/httpd/bin/httpd -k start > > $ ./apachectl graceful > $ ps alx | grep 18459 > 1 1000 18459 1 20 0 152752 3192 674009 Ss ? 0:00 > /home/chip/temp/httpd/bin/httpd -k start > > $ ./apachectl graceful > $ ps alx | grep 18459 > 1 1000 18459 1 20 0 152752 3236 674009 Ss ? 0:00 > /home/chip/temp/httpd/bin/httpd -k start > > $ ./apachectl graceful > $ ps alx | grep 18459 > 1 1000 18459 1 20 0 152752 3280 674009 Ss ? 0:00 > /home/chip/temp/httpd/bin/httpd -k start > > (2672 -> 3192 -> 3236 -> 3280) > > And, at least over here, httpd consistently grows in RSS, without any > obvious cause. > > Seems reproducible on Ubuntu and Darwin, using 2.2.10, 2.0.63 and trunk. > > Any ideas? Two quick thoughts: 1. Memory fragmentation in the allocator lists (we had this discussion either here or on [EMAIL PROTECTED] a short time ago). 2. At some locations we use a global pool (process->pool) to allocate memory, e.g. mod_ssl and when setting up the listeners. I haven't checked so far if this global pool usage is justified. Regards Rüdiger
Re: leak on graceful restarts
Ruediger Pluem schrieb: > > On 10/18/2008 01:25 AM, Paul Querna wrote: >> Looking at a problem that seems easy to re-produce using un-patched >> trunk, 2.2.10 and 2.0.63. >> >> Using a graceful restart causes higher memory usage in the parent, which >> is then passed on to the 'new' children processes. >> >> The issue seems to appear in all the Worker, Event and Prefork MPMs. >> >> Compile up httpd without apr pool debugging enabled, start it as normal, >> and then check the memory usage: >> >> $ ./apachectl start >> $ ps alx | grep 18459 >> 1 1000 18459 1 20 0 152752 2672 674009 Ss ? 0:00 >> /home/chip/temp/httpd/bin/httpd -k start >> >> $ ./apachectl graceful >> $ ps alx | grep 18459 >> 1 1000 18459 1 20 0 152752 3192 674009 Ss ? 0:00 >> /home/chip/temp/httpd/bin/httpd -k start >> >> $ ./apachectl graceful >> $ ps alx | grep 18459 >> 1 1000 18459 1 20 0 152752 3236 674009 Ss ? 0:00 >> /home/chip/temp/httpd/bin/httpd -k start >> >> $ ./apachectl graceful >> $ ps alx | grep 18459 >> 1 1000 18459 1 20 0 152752 3280 674009 Ss ? 0:00 >> /home/chip/temp/httpd/bin/httpd -k start >> >> (2672 -> 3192 -> 3236 -> 3280) >> >> And, at least over here, httpd consistently grows in RSS, without any >> obvious cause. >> >> Seems reproducible on Ubuntu and Darwin, using 2.2.10, 2.0.63 and trunk. >> >> Any ideas? > > Two quick thoughts: > > 1. Memory fragmentation in the allocator lists (we had this discussion either > here >or on [EMAIL PROTECTED] a short time ago). > > 2. At some locations we use a global pool (process->pool) to allocate memory, > e.g. mod_ssl >and when setting up the listeners. I haven't checked so far if this global > pool usage is >justified. Using my production configurations on Solaris with 2.2.10 worker I can only reproduce a leak during graceful restart when loading mod_ssl. The memory size does not always increase though, after a couple of restarts it decreases again, but not back to the previous minimum so over all there is a small leak related to restarts. Regards, Rainer
Re: leak on graceful restarts
On Oct 18, 2008, at 3:04 PM, Rainer Jung wrote: Ruediger Pluem schrieb: On 10/18/2008 01:25 AM, Paul Querna wrote: Looking at a problem that seems easy to re-produce using un-patched trunk, 2.2.10 and 2.0.63. Using a graceful restart causes higher memory usage in the parent, which is then passed on to the 'new' children processes. And, at least over here, httpd consistently grows in RSS, without any obvious cause. Seems reproducible on Ubuntu and Darwin, using 2.2.10, 2.0.63 and trunk. Any ideas? Two quick thoughts: 1. Memory fragmentation in the allocator lists (we had this discussion either here or on [EMAIL PROTECTED] a short time ago). 2. At some locations we use a global pool (process->pool) to allocate memory, e.g. mod_ssl and when setting up the listeners. I haven't checked so far if this global pool usage is justified. Using my production configurations on Solaris with 2.2.10 worker I can only reproduce a leak during graceful restart when loading mod_ssl. The memory size does not always increase though, after a couple of restarts it decreases again, but not back to the previous minimum so over all there is a small leak related to restarts. This is weird... I can recreate this under OS X but not under Sol10, and only with mod_ssl in the mix as well. But at least it appears that mod_ssl is the main culprit.
Re: leak on graceful restarts
On 10/19/2008 07:25 PM, Jim Jagielski wrote: > > On Oct 18, 2008, at 3:04 PM, Rainer Jung wrote: > >> >> >> Ruediger Pluem schrieb: >>> >>> On 10/18/2008 01:25 AM, Paul Querna wrote: >>>> Looking at a problem that seems easy to re-produce using un-patched >>>> trunk, 2.2.10 and 2.0.63. >>>> >>>> Using a graceful restart causes higher memory usage in the parent, >>>> which >>>> is then passed on to the 'new' children processes. >>>> >>>> And, at least over here, httpd consistently grows in RSS, without any >>>> obvious cause. >>>> >>>> Seems reproducible on Ubuntu and Darwin, using 2.2.10, 2.0.63 and >>>> trunk. >>>> >>>> Any ideas? >>> >>> Two quick thoughts: >>> >>> 1. Memory fragmentation in the allocator lists (we had this >>> discussion either here >>> or on [EMAIL PROTECTED] a short time ago). >>> >>> 2. At some locations we use a global pool (process->pool) to allocate >>> memory, e.g. mod_ssl >>> and when setting up the listeners. I haven't checked so far if this >>> global pool usage is >>> justified. >> >> Using my production configurations on Solaris with 2.2.10 worker I can >> only reproduce a leak during graceful restart when loading mod_ssl. The >> memory size does not always increase though, after a couple of restarts >> it decreases again, but not back to the previous minimum so over all >> there is a small leak related to restarts. >> > > This is weird... I can recreate this under OS X but not under Sol10, > and only with mod_ssl in the mix as well. But at least it appears that > mod_ssl is the main culprit. In 2.2.x I guess we leak in ssl_scache_shmcb_init when we create a shared memory segment passing apr_shm_create a global pool. Maybe the actual amount of the leak depends on the platform specific details of the shm implementation. As said this is just a guess. AFAICT this leak does not happen on trunk. Regards Rüdiger
apache prefork mpm graceful restarts
httpd version : 2.2.9 It looking at how the prefork mpm handles graceful restarts. Even with a keep-alive connection, I would like graceful restarts to handle the current in-flight request and come out as soon as the request is processed. But that doesnt seem to be the case currently. Looks like we wait either for the connection to be closed or the keep-alive timeout to kickin. Shouldn't ap_graceful_stop_signalled() return die_now rather than 0 ? I understand that this is done on purpose, but couldn't figure out the reason. Can someone shed light on this. Thanks, Padhu.
Re: trunk crash on graceful
On Fri, Mar 23, 2018 at 1:07 PM, Stefan Eissing wrote:> > > In case someone gets bored or the weather is dull and cloudy in > southern France, How so? The sun shines here (as always), do you want a picture or it might hurt? :) > someone might have a look at this: Is it in ONE_PROCESS mode (-X) only or it happens w/o too?
Re: trunk crash on graceful
> Am 23.03.2018 um 13:19 schrieb Yann Ylavic : > > On Fri, Mar 23, 2018 at 1:07 PM, Stefan Eissing > wrote:> >> >> In case someone gets bored or the weather is dull and cloudy in >> southern France, > > How so? The sun shines here (as always), do you want a picture or it > might hurt? :) Ok, here in almost-as-beautiful Germany it is cloudy... >:-) > >> someone might have a look at this: > > Is it in ONE_PROCESS mode (-X) only or it happens w/o too? I run tests in real mode with several children.
worker graceful restart warning message
I'm seeing a race condition in which the worker MPM logs the "long lost child came home!" warning message. The test case is: - run "ab -c5" to create a steady load on the httpd - while it's running, do a graceful restart. This will sometimes yield the "long lost child" message. I added a bit of diagnostic logging and found that the order of events looks like this: - child process for scoreboard slot X finishes its work and exits - parent process forks a new child process and assigns it to scoreboard slot X - parent process notices that the first child process has exited, looks for its pid in scoreboard, and doesn't find it Is this a harmless (and expected) warning case, or cause for alarm? --Brian
kill -HUP vs. httpd -k graceful
Is there a difference between kill -HUP and httpd -k graceful? One user has reported problems with using kill -HUP [Fri Nov 19 00:00:05 2004] [warn] child process 12783 still did not exit, sending a SIGTERM [Fri Nov 19 00:00:05 2004] [notice] SIGHUP received. Attempting to restart panic: pthread_key_create and dumps core... whereas httpd -k graceful doesn't have this problem. Unfortunately I can't tell you which MPM he was using, since he didn't bother to tell. -- __ Stas BekmanJAm_pH --> Just Another mod_perl Hacker http://stason.org/ mod_perl Guide ---> http://perl.apache.org mailto:[EMAIL PROTECTED] http://use.perl.org http://apacheweek.com http://modperlbook.org http://apache.org http://ticketmaster.com
Re: Errors while using graceful shutdown
Kiran Mendonce wrote: For every concurrent request issued, there seems to be an error. I think this is because ab closes the last set of connections without completing the HTTP "transaction." We noticed this on some of our apps. This may or may not be your issue as well. -- Brian Akins Lead Systems Engineer CNN Internet Technologies
Re: Errors while using graceful shutdown
I could fix this with some modifications to mod_cgid.c. I found that the httpd process serving the CGI request tries to contact the daemon in the cleanup callback even when a graceful restart has been issued. A check for a graceful restart situation should be made before the connect. The following condition at the beginning of cleanup_script() solves the problem : if (ap_graceful_stop_signalled()) return APR_EGENERAL; Regards, Kiran Brian Akins wrote: Kiran Mendonce wrote: For every concurrent request issued, there seems to be an error. I think this is because ab closes the last set of connections without completing the HTTP "transaction." We noticed this on some of our apps. This may or may not be your issue as well.
Re: prototype for worker graceful *shutdown*
--On Friday, March 28, 2003 3:34 PM -0500 Rodent of Unusual Size <[EMAIL PROTECTED]> wrote: the attached patch, based on one bill started, presses SIGINT into service as a 'graceful shutdown' signal. (alternative suggestions highly welcomed.) if the config directive SIGUSR2? SIGINT should definitely not be used. -- justin
Re: prototype for worker graceful *shutdown*
Justin Erenkrantz wrote: > > SIGUSR2? i considered that. unfortunately, i think it is less than universal in its presence on all our platforms. maybe SIGSTOP? its canonical interactive use is obviated by being in a daemon environment.. except i think that would be problematic when debugging interactively. > SIGINT should definitely not be used. -- justin why not? there are precedents; named, for example. i couldn't see any place we're using ourselves; did i miss something? or is it the same problem with interactive debuggined processing and not having an instant shutdown on ctrl/c? -- #kenP-)} Ken Coar, Sanagendamgagwedweinini http://Golux.Com/coar/ Author, developer, opinionist http://Apache-Server.Com/ "Millennium hand and shrimp!"
Re: [PATCH] prefork graceful restart fix
Joe Orton wrote: Graceful restarts can be really slow in 2.0 with prefork... the parent is making a dummy connection on the pod MaxClients times, rather than a connection for as many children as it has had; is that really necessary? We've had problems in this area in the past (not necessarily in prefork) due to not being able to aim the dummy connections at specific old generation server instances. Then we start the new generation before the old generation is completely down, and the old generation processes hang around indefinately. If we can insure that won't happen, then go for it. This is an area where I think signals are valuable because they target specific processes. Yes, we've been avoiding signals in 2.0. But IMO that was mostly a workaround for linuxthread problems. That's not an issue for prefork, and it shouldn't be an issue for the new Linux pthread library either. Other approaches could work too, like: 1. send ap_max_daemons_limit dummy connections. 2. scan the scoreboard. Are all the children gone? Great, on to step 5. 3. send one dummy connection 4. sleep a little, then back to step 2. 5. start up the new generation With the current design, you shouldn't see more than one connection time out. A minute or more sounds like breakage, or possibly funky sysctl settings for TCP. Greg
Re: [PATCH] prefork graceful restart fix
Joe Orton wrote: Graceful restarts can be really slow in 2.0 with prefork... the parent is making a dummy connection on the pod MaxClients times, rather than a connection for as many children as it has had; is that really necessary? you're right, of course At least one of the connections will get the 3 second timeout without the below change in my testing, which accounts for the delay. Anecdotally we've seen it take up to a minute or more on idle-ish servers during testing, and had user reports of the same. I've seen this too on Linux 2.4 kernel. Index: prefork.c === RCS file: /home/cvs/httpd-2.0/server/mpm/prefork/prefork.c,v retrieving revision 1.279 diff -u -r1.279 prefork.c --- prefork.c 27 Aug 2003 22:33:11 - 1.279 +++ prefork.c 16 Oct 2003 18:57:15 - @@ -1123,7 +1123,7 @@ "Graceful restart requested, doing restart"); /* kill off the idle ones */ -ap_mpm_pod_killpg(pod, ap_daemons_limit); +ap_mpm_pod_killpg(pod, ap_max_daemons_limit); as Greg mentioned, there are aspects of this mechanism that leave a lot to be desired... but this should be an improvement +1 from me for 2.1-dev and 2.0.49-dev
Re: [PATCH] prefork graceful restart fix
On Fri, Oct 17, 2003 at 10:41:09AM -0400, Greg Ames wrote: > Joe Orton wrote: > >Graceful restarts can be really slow in 2.0 with prefork... the parent > >is making a dummy connection on the pod MaxClients times, rather than a > >connection for as many children as it has had; is that really necessary? > > We've had problems in this area in the past (not necessarily in prefork) > due to not being able to aim the dummy connections at specific old > generation server instances. Then we start the new generation before the > old generation is completely down, and the old generation processes hang > around indefinately. If we can insure that won't happen, then go for it. The ap_my_generation != parent generation check in the child should give a good guarantee that children don't stick around too long after a restart? > This is an area where I think signals are valuable because they target > specific processes. Yes, we've been avoiding signals in 2.0. But IMO that > was mostly a workaround for linuxthread problems. That's not an issue for > prefork, and it shouldn't be an issue for the new Linux pthread library > either. OK, why was prefork switched to use the pod rather than signals - concerns about using threads from modules or something? > Other approaches could work too, like: > > 1. send ap_max_daemons_limit dummy connections. > 2. scan the scoreboard. Are all the children gone? Great, on to step 5. > 3. send one dummy connection > 4. sleep a little, then back to step 2. > 5. start up the new generation > > With the current design, you shouldn't see more than one connection time > out. A minute or more sounds like breakage, or possibly funky sysctl > settings for TCP. It's actually quite easy to reproduce, with default TCP settings; just set MaxClients to something high, then try a graceful restart on an idle server a few times. [Wed Oct 22 12:54:11 2003] [notice] Graceful restart requested, doing restart [Wed Oct 22 12:54:44 2003] [notice] Digest: generating secret for digest authentication ... StartServers 8 MinSpareServers5 MaxSpareServers 20 ServerLimit 1500 MaxClients 1500 MaxRequestsPerChild 1000 Regards, joe
detecting stop|graceful|restart from httpd.conf
My last post included a patch providing the mechanism to know when the server starts and immediately restarts. Unfortunately it won't work for 'httpd -k [stop|graceful|restart]' which can't reach the pool of the running server. We have users asking for a flag which will allow them to disable the execution of their startup code when the server is shutdown. Can the mpm architecture be extended to set some flag telling what state it's in (for the benefit of the config phase and may be others as well)? I imagine that the optional signal_server() should be moved before the config parsing code, so the mpm implementation will register what should be done and raise the flag (visible from the outside) telling whether it's one of stop|graceful|restart. Now when config is done, call signal_server_finalize() which will do what signal_server() originally did. __ Stas BekmanJAm_pH --> Just Another mod_perl Hacker http://stason.org/ mod_perl Guide ---> http://perl.apache.org mailto:[EMAIL PROTECTED] http://use.perl.org http://apacheweek.com http://modperlbook.org http://apache.org http://ticketmaster.com
[PATCH] graceful restart bug as opportunity
configure.in makes a big deal about determining AP_SIG_GRACEFUL, which
defaults to SIGUSR1, but uses SIGWINCH on Linux 2.0. But then
mpm_common.c goes ahead and ignores this for actually sending the
signal, SIGUSR1 is hard-coded;
if (!strcmp(dash_k_arg, "graceful")) {
if (!running) {
printf("httpd not running, trying to start\n");
}
else {
*exit_status = send_signal(otherpid, SIGUSR1);
return 1;
}
}
I can only surmise that there just arn't very many linux 2.0 users who
try to do graceful restarts :-)
Anyway, an easy and obvious fix would be to patch that code to use
AP_SIG_GRACEFUL. However this sucks, so I've attached a totally more
insane patch to just use SIGUSR1 everywhere, and not support "graceful"
on Linux 2,0 (which doesn't work anyway, so it's not exactly a change).
I'm working on adding "graceful stop" (httpd -k drain) [1] and well
there's a shortage of genuinely usable signals. SIGUSR2 would be the
obvious choice, but;
/*
* SIGUSR2 is being removed from the mask for the convenience of
* Purify users (Solaris, HP-UX, SGI) since Purify uses SIGUSR2
*/
#ifdef SIGUSR2
sigdelset(sig_mask, SIGUSR2);
#endif
Which really leaves SIGWINCH as the only semi-reliable signal to use,
but that isn't free because of the stupid Linux 2.0 brokenness. So
rather than seeing this as a lunatic patch, I'm asking you to look into
your hearts and see this as an opportunity to free up a portable signal
so some nifty functionality can be added more easily.
Failing this, can anyone suggest a non-WINCH signal which is portable
and reliable? Or, how would people feel about ditching kill-style
signalling altogher and using some other IPC mechanism, for more
adaptable and futureproofing ?
[1] When you have a few hundred users downloading DVD ISO's over dialup,
not killing their downloads when upgrading the webserver would be a
real nice feature, and well I'm tired of JoS gloating about it in IIS;
http://joelonsoftware.com/items/2005/06/15.html ;)
--
Colm MacCárthaighPublic Key: [EMAIL PROTECTED]
Index: server/mpm/prefork/prefork.c
===
--- server/mpm/prefork/prefork.c(revision 225483)
+++ server/mpm/prefork/prefork.c(working copy)
@@ -104,7 +104,7 @@
/*
* The max child slot ever assigned, preserved across restarts. Necessary
- * to deal with MaxClients changes across AP_SIG_GRACEFUL restarts. We
+ * to deal with MaxClients changes across graceful restarts. We
* use this value to optimize routines that have to scan the entire scoreboard.
*/
int ap_max_daemons_limit = -1;
@@ -348,7 +348,7 @@
shutdown_pending = 1;
}
-/* restart() is the signal handler for SIGHUP and AP_SIG_GRACEFUL
+/* restart() is the signal handler for SIGHUP and SIGUSR1
* in the parent process, unless running in ONE_PROCESS mode
*/
static void restart(int sig)
@@ -358,7 +358,7 @@
return;
}
restart_pending = 1;
-is_graceful = (sig == AP_SIG_GRACEFUL);
+is_graceful = (sig == SIGUSR1);
}
static void set_signals(void)
@@ -398,16 +398,16 @@
ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
"sigaction(SIGPIPE)");
#endif
-/* we want to ignore HUPs and AP_SIG_GRACEFUL while we're busy
+/* we want to ignore HUPs and USR1 while we're busy
* processing one
*/
sigaddset(&sa.sa_mask, SIGHUP);
-sigaddset(&sa.sa_mask, AP_SIG_GRACEFUL);
+sigaddset(&sa.sa_mask, SIGUSR1);
sa.sa_handler = restart;
if (sigaction(SIGHUP, &sa, NULL) < 0)
ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
"sigaction(SIGHUP)");
-if (sigaction(AP_SIG_GRACEFUL, &sa, NULL) < 0)
-ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
"sigaction(" AP_SIG_GRACEFUL_STRING ")");
+if (sigaction(SIGUSR1, &sa, NULL) < 0)
+ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
"sigaction(SIGUSR1)");
#else
if (!one_process) {
#ifdef SIGXCPU
@@ -422,9 +422,9 @@
#ifdef SIGHUP
apr_signal(SIGHUP, restart);
#endif /* SIGHUP */
-#ifdef AP_SIG_GRACEFUL
-apr_signal(AP_SIG_GRACEFUL, restart);
-#endif /* AP_SIG_GRACEFUL */
+#ifdef AP_ENABLE_GRACEFUL
+apr_signal(SIGUSR1, restart);
+#endif /* AP_ENABLE_GRACEFUL */
#ifdef SIGPIPE
apr_signal(SIGPIPE, SIG_IGN);
#endif /* SIGPIPE */
@@ -657,7 +657,7 @@
if (one_process) {
apr_signal(SIGHUP, sig_term);
-/* Don't catch AP_SIG_GRACEFUL in ONE_PROCESS mode :) */
+/* Don't catch SIGUSR1 in ONE_PROCESS mode :) */
apr_signal(SIGINT, sig_term);
#ifdef SIGQUIT
apr_signal(SIGQUIT, SIG_DFL);
@@ -715,1
fixing graceful-stop with event mpm
Short: We need to call ap_close_listeners() earlier or more aggressively. Question: Where/How? Looking at the Event MPM in both trunk and 2.2.x, the listener_thread is where we call ap_close_listeners(). This does not seem to be working quickly enough, per PR 43081: http://issues.apache.org/bugzilla/show_bug.cgi?id=43081 I haven't quite got my head around how the listener thread is apparently blocking for so long, but I can't figure out another way that the listeners would stay online for so long either. Solution: Add call to ap_close_listeners() in wakeup_listener(). I think this would pop out all of the listening sockets quickly from the listener pollset. Solution: Actually start working on 3.0 with a sane event/start/shutdown model? Thoughts? Thanks, -Paul
prefork: hung processes on graceful reload
On Monday 08 October 2007, Jim Jagielski wrote: > > On Oct 5, 2007, at 2:07 PM, Stefan Fritsch wrote: > >> Maybe someone could also look at > >> > >> http://issues.apache.org/bugzilla/show_bug.cgi?id=42829 > A quick review seems to indicate that the suggested patch could > result in a worker accepting a request and then being killed > off, something that we've tried to prevent... The patch was broken in other ways, too. I have attached a different patch that is definitely an improvement over 2.2.6 (though a small race condition remains). Can you look at it again and maybe include it in 2.2.7? Thanks. Cheers, Stefan
2.4.4 graceful restart taking long time
Hello, On apache 2.2.22, 2.2.23 and 2.4.4 I'm able to reproduce a problem where graceful restart takes very long time. Linux 3.7.10, glibc 2.17 here. keepalive is off, ServerLimit 600; StartServers 64; MinSpareServers 32; MaxSpareServers 64; MaxClients 600; MaxRequestsPerChild 2000; minimal number of apache modules I'm running ab where 192.168.1.121 is my local ip: /usr/bin/ab -n 10 -c 64 http://192.168.1.121/ (ususally I'm interrupting ab in first 1-2 seconds of its run, starting it again and leaving running) and then doing graceful restart while ab is still running: [Thu Apr 11 13:27:32.624875 2013] [mpm_prefork:notice] [pid 10202] AH00171: Graceful restart requested, doing restart [Thu Apr 11 13:28:15.836068 2013] [mpm_prefork:notice] [pid 10202] AH00163: Apache/2.4.4 (Unix) configured -- resuming normal operations It takes almost 1 minute to resume operations. Now if I don't do ab "magic" then resume is immediate (1-2s). The whole time is spend in prefork ap_mpm_pod_killpg ap_log_error(APLOG_MARK, APLOG_NOTICE, 0, ap_server_conf, APLOGNO(00171) "Graceful restart requested, doing restart"); /* kill off the idle ones */ ap_mpm_pod_killpg(pod, retained->max_daemons_limit); where ap_mpm_pod_killpg iterates over childrens (?) and that takes ~1 minute. I also did testing with StartServers 128. ap_mpm_pod_killpg had to iterate over 128 childrens where 72 of them were done in the same second while the rest in over minute - log here http://pastebin.com/3tP5hW1f. Example strace of main httpd process while doing graceful restart: http://pastebin.com/QFH5TjT6 The whole problem is that until ap_mpm_pod_killpg finishes apache is not responding to new requests and the downtime is between 30s to even 4 minutes. -- Arkadiusz Miśkiewicz, arekm / maven.pl
Catching graceful restart in apache2 module
Hello all, sorry for bother you with this question, but I do not know how to handle graceful restart in apache2 module. I will try to explain the situation: To the our apache2 configuration file are dynamically added directives with the authorizations, options, etc. When the apache2 is running than all web pages are visible and work correctly, All applications are running over HTTPS therefore our module handled sessions with clients. When I am adding some directive Directory I would like to tell to the apache2 to reuse/reopen changed configuration file. When I am calling /usr/sbin/apache2ctl -k graceful then apache2 is restarted gracefully unfortunatelly all sessions with clients are deleted during the stopping case. Is there any way how to detect that gracefull restart has been started and where should I place the code for detection in our module. Thank you in advance best regards Petr Hracek
graceful restart detection in prefork mode
Dear developers, My situation is following: In my Apache2 running in prefork mode I have following situation. Whole pages (except loging page) are run over https (port 443) and authentication is done over my own module. When the most users are looking on the pages (https) then I would like to reload configuration of my apache over command apache2ctl -k graceful I suggest that new users will see new configuration and old users will have their session till they will not logout. But unfortunatelly all sessions are closed and as new as old users have to login again for other work. Is there any solution for that like detection for gracefull restart? Do you now any idea how to do that? My apache2 which is delivered by SUSE is pretty old (I know). linux:~ # httpd2 -l Compiled in modules: core.c prefork.c http_core.c mod_so.c linux:~ # httpd2 -V Server version: Apache/2.2.3 Server built: Apr 14 2010 11:41:47 Server's Module Magic Number: 20051115:3 Server loaded: APR 1.2.2, APR-Util 1.2.2 Compiled using: APR 1.2.2, APR-Util 1.2.2 Architecture: 32-bit Server MPM: Prefork threaded: no forked: yes (variable process count) Server compiled with -D APACHE_MPM_DIR="server/mpm/prefork" -D APR_HAS_SENDFILE -D APR_HAS_MMAP -D APR_HAVE_IPV6 (IPv4-mapped addresses enabled) -D APR_USE_SYSVSEM_SERIALIZE -D APR_USE_PTHREAD_SERIALIZE -D SINGLE_LISTEN_UNSERIALIZED_ACCEPT -D APR_HAS_OTHER_CHILD -D AP_HAVE_RELIABLE_PIPED_LOGS -D DYNAMIC_MODULE_LIMIT=128 -D HTTPD_ROOT="/srv/www" -D SUEXEC_BIN="/usr/sbin/suexec2" -D DEFAULT_PIDLOG="/var/run/httpd2.pid" -D DEFAULT_SCOREBOARD="logs/apache_runtime_status" -D DEFAULT_LOCKFILE="/var/run/accept.lock" -D DEFAULT_ERRORLOG="/var/log/apache2/error_log" -D AP_TYPES_CONFIG_FILE="/etc/apache2/mime.types" -D SERVER_CONFIG_FILE="/etc/apache2/httpd.conf" linux:~ # Thank you in advance. -- Best Regards / S pozdravem Petr Hracek
Piped logging, graceful restart, broken pipe
I configured CustomLog with a pipe: CustomLog "|/usr/bin/logger -p local1.info -t apache2" combined Now I start downloading a large file. During this download I gracefully restart Apache. The download continues of course. But when the download is finished I don't get an entry in the access log, instead I get this message in the error log: (32)Broken pipe: [...] AH00646: Error writing to |/usr/bin/logger -p local1.info -t apache2 It looks like the graceful restart doesn't account for the /usr/bin/logger process still being required until the last worker finished. I.e. the graceful restart kills the running /usr/bin/logger immediately and starts a new one. Is this something that can be fixed? Or am I doing something wrong here?
Do graceful restarts actually work anywhere?
Just looking through the code and graceful restarts really shouldn't be working right now. The problem is that we pass pconf into the pre_mpm phase, but on Unix we only call the pre_mpm phase on graceless shutdown or when the server first starts. We clear the pconf pool on every restart, and we register a cleanup with pconf to destroy the scoreboard. If I have traced the code correctly, this means that we should be deleting the scoreboard on every graceful restart and not re-creating it. I can't see how this is not causing severe seg faults. Also, Windows is calling ap_pre_mpm on every restart, and I believe that the correct solution is to pass pglobal to the pre_mpm hook. Ryan -- Ryan Bloom [EMAIL PROTECTED] 645 Howard St. [EMAIL PROTECTED] San Francisco, CA
RE: worker graceful restart warning message
> From: Brian Pane [mailto:[EMAIL PROTECTED]] > Sent: 13 April 2002 10:12 > I'm seeing a race condition in which the worker MPM logs the > "long lost child came home!" warning message. The test case > is: > - run "ab -c5" to create a steady load on the httpd > - while it's running, do a graceful restart. > This will sometimes yield the "long lost child" message. > > I added a bit of diagnostic logging and found that the order > of events looks like this: > - child process for scoreboard slot X finishes its work and exits > - parent process forks a new child process and assigns it > to scoreboard slot X Why does it reuse slot X when it hasn't seen the child at slot X exit yet? > - parent process notices that the first child process has > exited, looks for its pid in scoreboard, and doesn't find it > > > Is this a harmless (and expected) warning case, or cause for > alarm? > > --Brian Sander
Re: worker graceful restart warning message
Brian Pane wrote: > > I'm seeing a race condition in which the worker MPM logs the > "long lost child came home!" warning message. The test case > is: > - run "ab -c5" to create a steady load on the httpd > - while it's running, do a graceful restart. > This will sometimes yield the "long lost child" message. > > I added a bit of diagnostic logging and found that the order > of events looks like this: > - child process for scoreboard slot X finishes its work and exits > - parent process forks a new child process and assigns it > to scoreboard slot X > - parent process notices that the first child process has > exited, looks for its pid in scoreboard, and doesn't find it > > > Is this a harmless (and expected) warning case, or cause for > alarm? I think it's harmless. If there are no scoreboard slots available that are completely empty, one new process is allowed to "squat" on a scoreboard slot for a process that is quiescing and has some unused thread slots. Look at how perform_idle_server_maintenance manipulates the free_slots array, starting around line 1417 in worker.c. Assuming the "squatting" scenario happens, the new process could overwrite the pid field(s) in the scoreboard. Then when the SIGCHILD logic in the parent kicks in, it may not be able to find the dying process's pid in the scoreboard. Bumping up ServerLimit to be significantly bigger than the number of processes allowed by MaxClients will reduce the likelihood of scoreboard squatting. Could we do something to make the messages go away? This is software, so the only safe answer is "maybe". I don't remember enough of the details of how this all works to say if it would be easy. Greg Greg
Re: kill -HUP vs. httpd -k graceful
On Thu, 02 Dec 2004 13:41:07 -0500, Stas Bekman <[EMAIL PROTECTED]> wrote: > Is there a difference between kill -HUP and httpd -k graceful? One user > has reported problems with using kill -HUP graceful uses signal USR1; ungraceful restart uses HUP; not expected that HUP would cause a crash
Re: kill -HUP vs. httpd -k graceful
Jeff Trawick wrote: On Thu, 02 Dec 2004 13:41:07 -0500, Stas Bekman <[EMAIL PROTECTED]> wrote: Is there a difference between kill -HUP and httpd -k graceful? One user has reported problems with using kill -HUP graceful uses signal USR1; ungraceful restart uses HUP; Thanks Jeff. not expected that HUP would cause a crash It's quite expected if it's ungraceful. I've reported this problem long long time ago. If something is run for longer than the shutdown is willing to wait after SIGKILL was sent (e.g. pool cleanups), SIGTERM is sent and then crashes are normal. I can easily reproduce that at will. Please see: http://issues.apache.org/bugzilla/show_bug.cgi?id=23238 -- __ Stas BekmanJAm_pH --> Just Another mod_perl Hacker http://stason.org/ mod_perl Guide ---> http://perl.apache.org mailto:[EMAIL PROTECTED] http://use.perl.org http://apacheweek.com http://modperlbook.org http://apache.org http://ticketmaster.com
Apache 2.2 - graceful restart causes cpu load
I'm hoping someone on the development list can comment on this.
I've noticed these in httpd access log starting with Apache2.2:
::1 - - [23/Jun/2006:04:03:04 -0700] "GET / HTTP/1.0" 200 117 "-"
"Apache/2.2.0 (Fedora) (internal dummy connection)"
I see a large number of those internal dummy connection requests during
an apache graceful restart (SIGUSR1) and at the same time the cpu load
on the Apache2.2 server maxes out at nearly 100%. I do not see this cpu
load during a graceful restart on apache 2.0 httpd servers.
I'm wondering if anyone else sees this high cpu usage with a graceful
restart of Apache2.2.
My home page (GET /) is dynamically generated (although running under
mod_perl) but I use the following mod_rewrite rule to make / a very
uncostly http request when the HTTP_USER_AGENT is "internal dummy
request". I'm not sure it is all those requests of / that is causing the
cpu load, but I created the RewriteRule, just in case.
RewriteEngine on
RewriteCond %{HTTP_USER_AGENT} ^.*internal\ dummy\ connection.*$ [NC]
RewriteRule ^/$ /small_static_page.html [L]
Can anyone comment or confirm a similar experience with high cpu usage
during a graceful restart of Apache2.2?
My OS is Linux 2.6.16-1.2133_FC5smp
--
Jim Albert
RE: detecting stop|graceful|restart from httpd.conf
> From: Stas Bekman [mailto:[EMAIL PROTECTED] > Sent: Friday, October 24, 2003 11:14 PM > My last post included a patch providing the mechanism to know when the server > starts and immediately restarts. Unfortunately it won't work for 'httpd -k > [stop|graceful|restart]' which can't reach the pool of the running server. > > We have users asking for a flag which will allow them to disable the execution > of their startup code when the server is shutdown. Can the mpm architecture be > extended to set some flag telling what state it's in (for the benefit of the > config phase and may be others as well)? Now this would be useful. Especially since I don't want all the initialization (with all the potential failures during) when I'm trying to shut down the server. Ever had a config error when trying to shut down your server? Sander
Re: detecting stop|graceful|restart from httpd.conf
Sander Striker wrote: From: Stas Bekman [mailto:[EMAIL PROTECTED] Sent: Friday, October 24, 2003 11:14 PM My last post included a patch providing the mechanism to know when the server starts and immediately restarts. Unfortunately it won't work for 'httpd -k [stop|graceful|restart]' which can't reach the pool of the running server. We have users asking for a flag which will allow them to disable the execution of their startup code when the server is shutdown. Can the mpm architecture be extended to set some flag telling what state it's in (for the benefit of the config phase and may be others as well)? Now this would be useful. Especially since I don't want all the initialization (with all the potential failures during) when I'm trying to shut down the server. Ever had a config error when trying to shut down your server? Config error is a trivial issue. It's a much bigger issue with running code at the startup, which is not supposed to be run. I thought that this and my previous post's patch could be combined into one API. So we could have the following states: state httpd.conf parsed # of times --- prestart first time startsecond time graceful first time restart first time stop first time the first two states are the normal startup, phases. As you can see from this table, the state gives a better state mapping, rather then the currently used init_if_first_time. __ Stas BekmanJAm_pH --> Just Another mod_perl Hacker http://stason.org/ mod_perl Guide ---> http://perl.apache.org mailto:[EMAIL PROTECTED] http://use.perl.org http://apacheweek.com http://modperlbook.org http://apache.org http://ticketmaster.com
Re: detecting stop|graceful|restart from httpd.conf
On Mon, Oct 27, 2003 at 04:53:22PM -0800, Stas Bekman wrote:
> Sander Striker wrote:
> I thought that this and my previous post's patch could be combined into one
> API. So we could have the following states:
>
> state httpd.conf parsed # of times
> ---
> prestart first time
> start second time
> graceful first time
> restart first time
> stop first time
>
> the first two states are the normal startup, phases. As you can see from
> this table, the state gives a better state mapping, rather then the
> currently used init_if_first_time.
Doesn't "graceful" re-run both first and second configuration phases?
Couldn't "stop" be used to execute code as an AP-style atexit()/on_exit()?
If so, the function would need to return some alternate code besides
"first time" or "second time".
I see a problem with both "prestart" and "start" indicating "first time"
because memory pools are cleared after preflight and after graceful.
Some modules might need to do some memory allocation during what you call
the "start" phase. You would not want to do this during "prestart"
because it would be cleared. But you would want it to run after
"graceful", because memory pools (except the main pool?) are cleared.
Doing it in child_init might be wasteful for forked MPMs, and a hassle to
do in child_init for threaded MPMs (e.g. having to keep tabs with SysV
conditionals PTHREAD_INIT_ONCE or atomics). (Or should the plog log pool
be used here instead of the primary pool?)
Or am I confused about memory pools and Apache configuration phase passes?
(quite likely :o))
Prior to this thread, I had needed to keep this state myself and so I
reworked my previously posted macros into something which could just as
easily be made a generic function. It is included below, but not well
tested. If any of the comments below are incorrect, I'd be thrilled if
someone would correct me.
Thanks.
Glenn
(Feel free to rename to AP_INIT_ or whatever)
/* Apache runs init routines twice. This keeps state and returns which mode,
* preflight (first pass through config) or runtime (second pass), the routine
* is being called in.
* (The pool must have the same lifetime as the server (i.e. s->process->pool)
* so that when server is restarted, the userdata set below will be cleaned up
* and the module will be reinitialized properly. A file-scoped global flag
* is not suitable for this purpose.)
* (modified from Apache2 modules/generators/mod_cgid.c)
*/
enum { INIT_PREFLIGHT = 0, INIT_RUNTIME, INIT_DONE };
# define INIT_MODE(mode, pool, tag)\
do {\
void *userdata_data;\
apr_pool_userdata_get(&userdata_data, tag "_init_once", (pool));\
if (userdata_data == NULL) {\
apr_pool_userdata_set((const void *)1, tag "_init_once",\
apr_pool_cleanup_null, (pool)); \
(mode) = INIT_PREFLIGHT;\
} \
else if (userdata_data == (const void *)1) {\
apr_pool_userdata_set((const void *)2, tag "_init_once",\
apr_pool_cleanup_null, (pool)); \
(mode) = INIT_RUNTIME; \
} \
else { \
(mode) = INIT_DONE; \
} \
} while (0)
static int
example_init(apr_pool_t *p, apr_pool_t *l, apr_pool_t *t, server_rec *s)
{
/* Check mode in which this code is being called: preflight, runtime, done*/
int mode;
INIT_MODE(mode, s->process->pool, "mod_example");
/* (init routines should only be called twice, but be defensive) */
if (mode != INIT_DONE) {
ap_add_version_component("mod_example/"MOD_EXAMPLE_VERSION);
}
if (mode == INIT_PREFLIGHT) {
/* Run as much as possible during first pass through the config
* (so that certain errors can be detected early).
* Do not run anything that depends on persistent memory pools,
* which are cleared before the runtime pass through the config.
* (or used the plog pool? (above as apr_pool_t *l))
*/
}
else if (mode == INIT_RUNTIME) {
/* Run things dependent on persistent memory pools in second init pass*/
}
else { /* (mode == INIT_DONE) */
/* (When (if at all) does this happen?
* Will this happen when shutting down?
* Only when shutting down?)
*/
}
return OK;
}
Re: detecting stop|graceful|restart from httpd.conf
Ooops. Reposted, changing:
> I see a problem with both "prestart" and "start" indicating "first time"
changed to:
> I see a problem with both "prestart" and "graceful" indicating "first time"
Sorry. -Glenn
On Mon, Oct 27, 2003 at 04:53:22PM -0800, Stas Bekman wrote:
> Sander Striker wrote:
> I thought that this and my previous post's patch could be combined into one
> API. So we could have the following states:
>
> state httpd.conf parsed # of times
> ---
> prestart first time
> startsecond time
> graceful first time
> restart first time
> stop first time
>
> the first two states are the normal startup, phases. As you can see from
> this table, the state gives a better state mapping, rather then the
> currently used init_if_first_time.
Doesn't "graceful" re-run both first and second configuration phases?
Couldn't "stop" be used to execute code as an AP-style atexit()/on_exit()?
If so, the function would need to return some alternate code besides
"first time" or "second time".
I see a problem with both "prestart" and "graceful" indicating "first time"
because memory pools are cleared after preflight and after graceful.
Some modules might need to do some memory allocation during what you call
the "start" phase. You would not want to do this during "prestart"
because it would be cleared. But you would want it to run after
"graceful", because memory pools (except the main pool?) are cleared.
Doing it in child_init might be wasteful for forked MPMs, and a hassle to
do in child_init for threaded MPMs (e.g. having to keep tabs with SysV
conditionals PTHREAD_INIT_ONCE or atomics). (Or should the plog log pool
be used here instead of the primary pool?)
Or am I confused about memory pools and Apache configuration phase passes?
(quite likely :o))
Prior to this thread, I had needed to keep this state myself and so I
reworked my previously posted macros into something which could just as
easily be made a generic function. It is included below, but not well
tested. If any of the comments below are incorrect, I'd be thrilled if
someone would correct me.
Thanks.
Glenn
(Feel free to rename to AP_INIT_ or whatever)
/* Apache runs init routines twice. This keeps state and returns which mode,
* preflight (first pass through config) or runtime (second pass), the routine
* is being called in.
* (The pool must have the same lifetime as the server (i.e. s->process->pool)
* so that when server is restarted, the userdata set below will be cleaned up
* and the module will be reinitialized properly. A file-scoped global flag
* is not suitable for this purpose.)
* (modified from Apache2 modules/generators/mod_cgid.c)
*/
enum { INIT_PREFLIGHT = 0, INIT_RUNTIME, INIT_DONE };
# define INIT_MODE(mode, pool, tag)\
do {\
void *userdata_data;\
apr_pool_userdata_get(&userdata_data, tag "_init_once", (pool));\
if (userdata_data == NULL) {\
apr_pool_userdata_set((const void *)1, tag "_init_once",\
apr_pool_cleanup_null, (pool)); \
(mode) = INIT_PREFLIGHT;\
} \
else if (userdata_data == (const void *)1) {\
apr_pool_userdata_set((const void *)2, tag "_init_once",\
apr_pool_cleanup_null, (pool)); \
(mode) = INIT_RUNTIME; \
} \
else { \
(mode) = INIT_DONE; \
} \
} while (0)
static int
example_init(apr_pool_t *p, apr_pool_t *l, apr_pool_t *t, server_rec *s)
{
/* Check mode in which this code is being called: preflight, runtime, done*/
int mode;
INIT_MODE(mode, s->process->pool, "mod_example");
/* (init routines should only be called twice, but be defensive) */
if (mode != INIT_DONE) {
ap_add_version_component("mod_example/"MOD_EXAMPLE_VERSION);
}
if (mode == INIT_PREFLIGHT) {
/* Run as much as possible during first pass through the config
* (so that certain errors can be detected early).
* Do not run anything that depends
Re: detecting stop|graceful|restart from httpd.conf
Glenn wrote: Ooops. Reposted, changing: [answering to the corrected context by Glenn] > I see a problem with both "prestart" and "graceful" indicating "first time" > because memory pools are cleared after preflight and after graceful. > Some modules might need to do some memory allocation during what you call > the "start" phase. You would not want to do this during "prestart" > because it would be cleared. But you would want it to run after > "graceful", because memory pools (except the main pool?) are cleared. > Doing it in child_init might be wasteful for forked MPMs, and a hassle to > do in child_init for threaded MPMs (e.g. having to keep tabs with SysV > conditionals PTHREAD_INIT_ONCE or atomics). (Or should the plog log pool > be used here instead of the primary pool?) These states will be set as soon as the config phase is finished. You have the open_logs and post_config phases to do the initialization. When these phases are run the state has already been set. I didn't know graceful was rerunning preflight as well. I'm used to do a full stop and start. In which case it's possible that an additional state (pregraceful) is needed. Or I have a better idea, drop the pre- states completely. Instead indicate whether it's preflight/first_time or not. So these two flags give you all the information you need: 1. action: start|stop|graceful|restart 2. cycle: first_time|second_time Prior to this thread, I had needed to keep this state myself and so I reworked my previously posted macros into something which could just as easily be made a generic function. It is included below, but not well tested. If any of the comments below are incorrect, I'd be thrilled if someone would correct me. what I dislike about the proposed macros is that each module needing this functionality will set a different key and read that different key, whereas the patch I have sent earlier provides this exact service for any consumer and does only one set operation besides simplifying the logic. Of course your set macro is more flexible as it can be called at any moment, whereas my patch suggested to run that same macro at the end of the config phase. __ Stas BekmanJAm_pH --> Just Another mod_perl Hacker http://stason.org/ mod_perl Guide ---> http://perl.apache.org mailto:[EMAIL PROTECTED] http://use.perl.org http://apacheweek.com http://modperlbook.org http://apache.org http://ticketmaster.com
Re: detecting stop|graceful|restart from httpd.conf
Sander Striker wrote: From: Stas Bekman [mailto:[EMAIL PROTECTED] Sent: Friday, October 24, 2003 11:14 PM My last post included a patch providing the mechanism to know when the server starts and immediately restarts. Unfortunately it won't work for 'httpd -k [stop|graceful|restart]' which can't reach the pool of the running server. We have users asking for a flag which will allow them to disable the execution of their startup code when the server is shutdown. Can the mpm architecture be extended to set some flag telling what state it's in (for the benefit of the config phase and may be others as well)? Now this would be useful. Especially since I don't want all the initialization (with all the potential failures during) when I'm trying to shut down the server. Ever had a config error when trying to shut down your server? Sander It's really weird how everyone stumbles across the same fooness at the same time. +1's all the way round. bill
Re: [PATCH] graceful restart bug as opportunity
On Wed, Jul 27, 2005 at 12:40:58PM +0100, Colm MacCarthaigh wrote:
> configure.in makes a big deal about determining AP_SIG_GRACEFUL, which
> defaults to SIGUSR1, but uses SIGWINCH on Linux 2.0. But then
> mpm_common.c goes ahead and ignores this for actually sending the
> signal, SIGUSR1 is hard-coded;
>
> if (!strcmp(dash_k_arg, "graceful")) {
> if (!running) {
> printf("httpd not running, trying to start\n");
> }
> else {
> *exit_status = send_signal(otherpid, SIGUSR1);
> return 1;
> }
> }
Heh :)
> I can only surmise that there just arn't very many linux 2.0 users who
> try to do graceful restarts :-)
>
> Anyway, an easy and obvious fix would be to patch that code to use
> AP_SIG_GRACEFUL. However this sucks, so I've attached a totally more
> insane patch to just use SIGUSR1 everywhere, and not support "graceful"
> on Linux 2,0 (which doesn't work anyway, so it's not exactly a change).
SIGUSR1 is unavailable on Linux 2.0 iff linuxthreads is used, i.e. in a
threaded MPM. It'd be better simply to refuse to allow use of threaded
MPMs on such platforms (which nobody will notice) and allow graceful to
use SIGUSR1 everywhere.
+1 on the patch if that much is changed.
joe
Re: [PATCH] graceful restart bug as opportunity
Colm MacCarthaigh wrote:
configure.in makes a big deal about determining AP_SIG_GRACEFUL, which
defaults to SIGUSR1, but uses SIGWINCH on Linux 2.0. But then
mpm_common.c goes ahead and ignores this for actually sending the
signal, SIGUSR1 is hard-coded;
if (!strcmp(dash_k_arg, "graceful")) {
if (!running) {
printf("httpd not running, trying to start\n");
}
else {
*exit_status = send_signal(otherpid, SIGUSR1);
return 1;
}
}
I can only surmise that there just arn't very many linux 2.0 users who
try to do graceful restarts :-)
Anyway, an easy and obvious fix would be to patch that code to use
AP_SIG_GRACEFUL. However this sucks, so I've attached a totally more
insane patch to just use SIGUSR1 everywhere, and not support "graceful"
on Linux 2,0 (which doesn't work anyway, so it's not exactly a change).
I'm working on adding "graceful stop" (httpd -k drain) [1] and well
there's a shortage of genuinely usable signals. SIGUSR2 would be the
obvious choice, but;
/*
* SIGUSR2 is being removed from the mask for the convenience of
* Purify users (Solaris, HP-UX, SGI) since Purify uses SIGUSR2
*/
#ifdef SIGUSR2
sigdelset(sig_mask, SIGUSR2);
#endif
Which really leaves SIGWINCH as the only semi-reliable signal to use,
but that isn't free because of the stupid Linux 2.0 brokenness. So
rather than seeing this as a lunatic patch, I'm asking you to look into
your hearts and see this as an opportunity to free up a portable signal
so some nifty functionality can be added more easily.
+1. Ken Coar and I have looked into the need for a 'graceful shutdown' and there may even be a patch posted
to the dev list using an IPC (so long ago I don't recall the exact details). Freeing up SIGWINCH sounds like a
good solution.
Bill
Re: [PATCH] graceful restart bug as opportunity
On Wed, Jul 27, 2005 at 12:59:05PM +0100, Joe Orton wrote:
> SIGUSR1 is unavailable on Linux 2.0 iff linuxthreads is used, i.e. in a
> threaded MPM. It'd be better simply to refuse to allow use of threaded
> MPMs on such platforms (which nobody will notice) and allow graceful to
> use SIGUSR1 everywhere.
>
> +1 on the patch if that much is changed.
That's even slightly easier to implement. Patch attached, though it is
rather blunt and simply disables the threaded MPM's on Linux 2.0.
I can remember there being other threading libraries for 2.0, but I
don't have a 2.0 test system and I'm not sure if that can easily be
autoconfed either.
--
Colm MacCárthaighPublic Key: [EMAIL PROTECTED]
Index: server/mpm/config.m4
===
--- server/mpm/config.m4(revision 225483)
+++ server/mpm/config.m4(working copy)
@@ -38,6 +38,22 @@
AC_MSG_CHECKING(checking for replacement)
AC_MSG_RESULT(prefork selected)
apache_cv_mpm=prefork
+ else
+case $host in
+ *-linux-*)
+case `uname -r` in
+ 2.0* )
+dnl Threaded MPM's are not supported on Linux 2.0
+dnl as on 2.0 the linuxthreads library uses SIGUSR1
+dnl and SIGUSR2 internally
+echo "Threaded MPM's are not supported on this platform"
+AC_MSG_CHECKING(checking for replacement)
+AC_MSG_RESULT(prefork selected)
+apache_cv_mpm=prefork
+ ;;
+esac
+ ;;
+esac
fi
fi
Index: server/mpm/prefork/prefork.c
===
--- server/mpm/prefork/prefork.c(revision 225483)
+++ server/mpm/prefork/prefork.c(working copy)
@@ -104,7 +104,7 @@
/*
* The max child slot ever assigned, preserved across restarts. Necessary
- * to deal with MaxClients changes across AP_SIG_GRACEFUL restarts. We
+ * to deal with MaxClients changes across graceful restarts. We
* use this value to optimize routines that have to scan the entire scoreboard.
*/
int ap_max_daemons_limit = -1;
@@ -348,7 +348,7 @@
shutdown_pending = 1;
}
-/* restart() is the signal handler for SIGHUP and AP_SIG_GRACEFUL
+/* restart() is the signal handler for SIGHUP and SIGUSR1
* in the parent process, unless running in ONE_PROCESS mode
*/
static void restart(int sig)
@@ -358,7 +358,7 @@
return;
}
restart_pending = 1;
-is_graceful = (sig == AP_SIG_GRACEFUL);
+is_graceful = (sig == SIGUSR1);
}
static void set_signals(void)
@@ -398,16 +398,17 @@
ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
"sigaction(SIGPIPE)");
#endif
-/* we want to ignore HUPs and AP_SIG_GRACEFUL while we're busy
+/* we want to ignore HUPs and USR1 while we're busy
* processing one
*/
sigaddset(&sa.sa_mask, SIGHUP);
-sigaddset(&sa.sa_mask, AP_SIG_GRACEFUL);
+sigaddset(&sa.sa_mask, SIGUSR1);
sa.sa_handler = restart;
if (sigaction(SIGHUP, &sa, NULL) < 0)
ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
"sigaction(SIGHUP)");
-if (sigaction(AP_SIG_GRACEFUL, &sa, NULL) < 0)
-ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
"sigaction(" AP_SIG_GRACEFUL_STRING ")");
+if (sigaction(SIGUSR1, &sa, NULL) < 0)
+ap_log_error(APLOG_MARK, APLOG_WARNING, errno, ap_server_conf,
"sigaction(SIGUSR1)");
+
#else
if (!one_process) {
#ifdef SIGXCPU
@@ -422,9 +423,9 @@
#ifdef SIGHUP
apr_signal(SIGHUP, restart);
#endif /* SIGHUP */
-#ifdef AP_SIG_GRACEFUL
-apr_signal(AP_SIG_GRACEFUL, restart);
-#endif /* AP_SIG_GRACEFUL */
+#ifdef SIGUSR1
+apr_signal(SIGUSR1, restart);
+#endif /* SIGUSR1 */
#ifdef SIGPIPE
apr_signal(SIGPIPE, SIG_IGN);
#endif /* SIGPIPE */
@@ -657,7 +658,7 @@
if (one_process) {
apr_signal(SIGHUP, sig_term);
-/* Don't catch AP_SIG_GRACEFUL in ONE_PROCESS mode :) */
+/* Don't catch SIGUSR1 in ONE_PROCESS mode :) */
apr_signal(SIGINT, sig_term);
#ifdef SIGQUIT
apr_signal(SIGQUIT, SIG_DFL);
@@ -715,10 +716,10 @@
*/
apr_signal(SIGHUP, just_die);
apr_signal(SIGTERM, just_die);
-/* The child process doesn't do anything for AP_SIG_GRACEFUL.
+/* The child process doesn't do anything for SIGUSR1.
* Instead, the pod is used for signalling graceful restart.
*/
-apr_signal(AP_SIG_GRACEFUL, SIG_IGN);
+apr_signal(SIGUSR1, SIG_IGN);
child_main(slot);
}
@@ -947,7 +948,7 @@
/* If we're doing a graceful_restart then we're going to see a lot
* of children exiting immediately when we get into the main loop
-
