Re: [RFC] ?

2017-02-27 Thread Joe Orton
On Wed, Feb 22, 2017 at 10:00:08PM +0100, Yann Ylavic wrote:
> On Wed, Feb 22, 2017 at 11:47 AM, Joe Orton  wrote:
> > (b) for  match both "foo" and " 
> I'd vote for this, it's very unlikely that some day we want to add a
> directive called VirtualHost or If (w/o the leading '<') which may
> conflict here, so it shouldn't hurt.

I'm fine with that, I'll commit like this unless anybody else has strong 
opinions.

Regards, Joe

Index: server/config.c
===
--- server/config.c (revision 1784534)
+++ server/config.c (working copy)
@@ -2668,6 +2668,24 @@
 printf("  %s\n", ap_loaded_modules[n]->name);
 }
 
+AP_DECLARE(int) ap_exists_directive(apr_pool_t *p, const char *name)
+{
+char *lname;
+
+if (!ap_config_hash)
+return 0;
+
+lname = apr_pstrdup(p, name);
+ap_str_tolower(lname);
+
+if (apr_hash_get(ap_config_hash, lname, APR_HASH_KEY_STRING) != NULL)
+return 1;
+
+/* Try '<'-prefixed form. */
+return apr_hash_get(ap_config_hash, apr_pstrcat(p, "<", lname, NULL), 
+APR_HASH_KEY_STRING) != NULL;
+ }
+
 AP_DECLARE(void *) ap_retained_data_get(const char *key)
 {
 void *retained;
Index: include/http_config.h
===
--- include/http_config.h   (revision 1784534)
+++ include/http_config.h   (working copy)
@@ -992,6 +992,15 @@
 AP_DECLARE(void) ap_show_directives(void);
 
 /**
+ * Returns non-zero if a configuration directive of the given name has
+ * been registered by a module at the time of calling.
+ * @param p Temporary pool
+ * @param name Directive name to match, case-insensitively, either
+ * directly or against '<'-prefixed container form.
+ */
+AP_DECLARE(int) ap_exists_directive(apr_pool_t *p, const char *name);
+
+/**
  * Show the preloaded module names.  Used for httpd -l.
  */
 AP_DECLARE(void) ap_show_modules(void);
Index: server/core.c
===
--- server/core.c   (revision 1784534)
+++ server/core.c   (working copy)
@@ -2894,6 +2894,46 @@
 }
 }
 
+
+static const char *start_ifdirective(cmd_parms *cmd, void *dummy, const char 
*arg)
+{
+const char *endp;
+int defined;
+int not = 0;
+
+endp = ap_strrchr_c(arg, '>');
+if (endp == NULL) {
+return unclosed_directive(cmd);
+}
+
+arg = apr_pstrmemdup(cmd->temp_pool, arg, endp - arg);
+
+if (arg[0] == '!') {
+not = 1;
+arg++;
+}
+
+if (!arg[0]) {
+return missing_container_arg(cmd);
+}
+
+defined = ap_exists_directive(cmd->temp_pool, arg);
+if ((!not && defined) || (not && !defined)) {
+ap_directive_t *parent = NULL;
+ap_directive_t *current = NULL;
+const char *retval;
+
+retval = ap_build_cont_config(cmd->pool, cmd->temp_pool, cmd,
+  ¤t, &parent, " business */
 
 static const char *virtualhost_section(cmd_parms *cmd, void *dummy,
@@ -4509,6 +4549,8 @@
   "Container for directives based on existence of command line defines"),
 AP_INIT_TAKE1("

Re: release v1.9.2

2017-02-27 Thread Stefan Priebe - Profihost AG
Hi Stefan,

two new crashes here.

Reading symbols from /usr/local/apache/bin/httpd...Reading symbols from
/usr/lib/debug//usr/local/apache2/bin/httpd...done.
done.
(gdb) (gdb) quit
Reading symbols from /usr/local/apache/bin/httpd...Reading symbols from
/usr/lib/debug//usr/local/apache2/bin/httpd...done.
done.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/local/apache/bin/httpd -DFOREGROUND'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  allocator_free (node=0x0, allocator=0x7f458005a320)
at memory/unix/apr_pools.c:381
#0  allocator_free (node=0x0, allocator=0x7f458005a320)
at memory/unix/apr_pools.c:381
#1  apr_pool_clear (pool=0x7f458004bec8) at memory/unix/apr_pools.c:793
#2  0x560ce83e5718 in ap_push_pool (queue_info=0x0,
pool_to_recycle=0x7f458005a328) at fdqueue.c:234
#3  0x560ce83e0a5a in process_lingering_close (cs=0x7f458004c158,
pfd=0x560ce8b8dda8) at event.c:1513
#4  0x560ce83e4590 in listener_thread (thd=0x0, dummy=0x5497f5242585c)
at event.c:1837
#5  0x7f45967610a4 in start_thread ()
   from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x7f459649662d in clone () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) (gdb) quit


Reading symbols from /usr/local/apache/bin/httpd...Reading symbols from
/usr/lib/debug//usr/local/apache2/bin/httpd...done.
done.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/local/apache/bin/httpd -DFOREGROUND'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  allocator_free (node=0x0, allocator=0x7f4580062840)
at memory/unix/apr_pools.c:381
#0  allocator_free (node=0x0, allocator=0x7f4580062840)
at memory/unix/apr_pools.c:381
#1  apr_pool_clear (pool=0x7f4580069188) at memory/unix/apr_pools.c:793
#2  0x560ce83e5718 in ap_push_pool (queue_info=0x0,
pool_to_recycle=0x7f4580062848) at fdqueue.c:234
#3  0x560ce83e0a5a in process_lingering_close (cs=0x7f4580069418,
pfd=0x560ce8b8dda8) at event.c:1513
#4  0x560ce83e4590 in listener_thread (thd=0x0, dummy=0x549845279ce62)
at event.c:1837
#5  0x7f45967610a4 in start_thread ()
   from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x7f459649662d in clone () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) (gdb) quit

Stefan
Am 26.02.2017 um 19:46 schrieb Stefan Priebe - Profihost AG:
> Hi Stefan,
> 
> currently everything fine. No segfaults.
> 
> Stefan
> 
> Am 25.02.2017 um 20:40 schrieb Stefan Priebe - Profihost AG:
>> Hi Stefan,
>> Am 25.02.2017 um 13:51 schrieb Stefan Eissing:
>>> Stefan,
>>>
>>> whenever you have time, please deploy 
>>> https://github.com/icing/mod_h2/releases/tag/v1.9.2
>>>
>>> I added an own allocator with mutex to the http/2 main session. That is 
>>> something of a middle-ground between placing that on the main conn (as we 
>>> had in the crash free version) and 1.9.1 behaviour. Thanks!
>>
>> done. But please keep in mind that this crash might be very rare and
>> might even have happened with v1.9.0 if we've waited more time.
>>
>> Greets,
>> Stefan
>>
>>> -Stefan
>>>
 Am 24.02.2017 um 19:31 schrieb Stefan Priebe - Profihost AG 
 :

 Hi Yann,

 here we go:

 (gdb) p *ipsub
 $1 = {family = 2, sub = {1367753145, 0, 0, 0}, mask = {4294967295,
 4294967295, 4294967295, 4294967295}}

 (gdb) p *sa
 $2 = {pool = 0x10b500c4ff0b0a, hostname = 0x503040203030102 >>> Cannot access memory at address 0x503040203030102>,
  servname = 0x17d01040405 >>> 0x17d01040405>, port = 770, family = 554829073,
  salen = 319177009, ipaddr_len = 570909009, addr_str_len = -2127424399,
 ipaddr_ptr = 0x24f0d15215c1b142,
  next = 0x17160a0982726233, sa = {sin = {sin_family = 6424, sin_port =
 9498, sin_addr = {s_addr = 690497318},
  sin_zero = "*456789:"}, sin6 = {sin6_family = 6424, sin6_port =
 9498, sin6_flowinfo = 690497318, sin6_addr = {__in6_u = {
  __u6_addr8 = "*456789:CDEFGHIJ", __u6_addr16 = {13354, 13877,
 14391, 14905, 17475, 17989, 18503, 19017}, __u6_addr32 = {
909456426, 976828471, 1178944579, 1246316615}}},
 sin6_scope_id = 1448432723}, sas = {ss_family = 6424,
  __ss_align = 4195446337656140842,
  __ss_padding =
 "CDEFGHIJSTUVWXYZcdefghijstuvwxyz\203\204\205\206\207\210\211\212\222\223\224\225\226\227\230\231\232\242\243\244\245\246\247\250\251\252\262\263\264\265\266\267\270\271\272\302\303\304\305\306\307\310\311\312\322\323\324\325\326\327\330\331\332\341\342\343\344\345\346\347\350\351\352\361\362\363\364\365\366\367\370\371\372\377\304\000\037\001\000\003"}}}

 (gdb) p *(struct in6_addr *)sa
 $3 = {__in6_u = {__u6_addr8 =
 "\n\v\377\304\000\265\020\000\002\001\003\003\002\004\003\005",
 __u6_addr16 = {2826, 50431, 46336,
  16, 258, 771, 1026, 1283}, 

Re: httpd 2.4.25, mpm_event, ssl: segfaults

2017-02-27 Thread Jacob Champion

[combining two replies]

On 02/23/2017 04:47 PM, Yann Ylavic wrote:

On Thu, Feb 23, 2017 at 7:16 PM, Jacob Champion  wrote:

Power users can break the system, and this is a power tool, right?


It's not about power users, I don't think we can recommend anyone to
use 4MB buffers even if they (seem to) have RAM for it.


Even if this is the case (and I'm still skeptical), there's a middle 
ground between "don't let users configure it at all" and "let users 
configure it to dangerous extents".



I'm not talking about hardcoding anything, nor reading or sendind hard
limited sizes on filesystem/sockets.
I'm proposing that the configuration relates to how much we "coalesce"
data on output, and all buffers' reuses (though each of fixed size)
should follow the needs.


I consider the fixed block size to be a major tuning point. That's what 
I'm referring to with "hardcoding". Coalescing doesn't help performance 
if we're bottlenecking on the overhead of tiny blocks. (Even if 
*everything* else can be vectored -- and right now, OpenSSL can't be -- 
allocation itself would still still be a fixed overhead.)



I've no idea how much it costs to have 8K vs 16K records, though.
Maybe in the mod_ssl case we'd want 16K buffers, still reasonable?


We can't/shouldn't hardcode this especially. People who want maximum
throughput may want nice big records, but IIRC users who want progressive
rendering need smaller records so that they don't have to wait as long for
the first decrypted chunk. It needs to be tunable, possibly per-location.


Well, the limit is 16K at the TLS level.


Maximum limit, yes. We also need to not set a minimum limit, which is, 
IIUC, what we're currently doing. mod_ssl calls SSL_write() once for 
every chunk read from apr_bucket_read().


On 02/23/2017 04:48 PM, Yann Ylavic wrote:
> On Wed, Feb 22, 2017 at 8:55 PM, Daniel Lescohier wrote:


IOW: read():Three copies: copy from filesystem cache to httpd
read() buffer to encrypted-data buffer to kernel socket buffer.

>
> Not really, "copy from filesystem cache to httpd read() buffer" is
> likely mapping to userspace, so no copy (on read) here.

Oh, cool. Which kernels do this? It seems like the VM tricks would have 
to be incredibly intricate for this to work; reads typically don't 
happen in page-sized chunks, nor to aligned addresses. Linux in 
particular has comments in the source explaining that they *don't* do it 
for other syscalls (e.g. vmsplice)... but I don't have much experience 
with non-Linux systems.


--Jacob


Re: release v1.9.2

2017-02-27 Thread Stefan Eissing
Damnit. Can you try the attached patch on top of mod_http2 v1.9.2? This one 
sets a mutex on the main connection allocator if missing and is pretty close to 
the version we ran successfully with on your site for days.

Thanks again!

-Stefan



h2_192_alloc_mutex.diff
Description: Binary data

> Am 27.02.2017 um 16:22 schrieb Stefan Priebe - Profihost AG 
> :
> 
> Hi Stefan,
> 
> two new crashes here.
> 
> Reading symbols from /usr/local/apache/bin/httpd...Reading symbols from
> /usr/lib/debug//usr/local/apache2/bin/httpd...done.
> done.
> (gdb) (gdb) quit
> Reading symbols from /usr/local/apache/bin/httpd...Reading symbols from
> /usr/lib/debug//usr/local/apache2/bin/httpd...done.
> done.
> [Thread debugging using libthread_db enabled]
> Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
> Core was generated by `/usr/local/apache/bin/httpd -DFOREGROUND'.
> Program terminated with signal SIGSEGV, Segmentation fault.
> #0  allocator_free (node=0x0, allocator=0x7f458005a320)
>at memory/unix/apr_pools.c:381
> #0  allocator_free (node=0x0, allocator=0x7f458005a320)
>at memory/unix/apr_pools.c:381
> #1  apr_pool_clear (pool=0x7f458004bec8) at memory/unix/apr_pools.c:793
> #2  0x560ce83e5718 in ap_push_pool (queue_info=0x0,
>pool_to_recycle=0x7f458005a328) at fdqueue.c:234
> #3  0x560ce83e0a5a in process_lingering_close (cs=0x7f458004c158,
>pfd=0x560ce8b8dda8) at event.c:1513
> #4  0x560ce83e4590 in listener_thread (thd=0x0, dummy=0x5497f5242585c)
>at event.c:1837
> #5  0x7f45967610a4 in start_thread ()
>   from /lib/x86_64-linux-gnu/libpthread.so.0
> #6  0x7f459649662d in clone () from /lib/x86_64-linux-gnu/libc.so.6
> (gdb) (gdb) quit
> 
> 
> Reading symbols from /usr/local/apache/bin/httpd...Reading symbols from
> /usr/lib/debug//usr/local/apache2/bin/httpd...done.
> done.
> [Thread debugging using libthread_db enabled]
> Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
> Core was generated by `/usr/local/apache/bin/httpd -DFOREGROUND'.
> Program terminated with signal SIGSEGV, Segmentation fault.
> #0  allocator_free (node=0x0, allocator=0x7f4580062840)
>at memory/unix/apr_pools.c:381
> #0  allocator_free (node=0x0, allocator=0x7f4580062840)
>at memory/unix/apr_pools.c:381
> #1  apr_pool_clear (pool=0x7f4580069188) at memory/unix/apr_pools.c:793
> #2  0x560ce83e5718 in ap_push_pool (queue_info=0x0,
>pool_to_recycle=0x7f4580062848) at fdqueue.c:234
> #3  0x560ce83e0a5a in process_lingering_close (cs=0x7f4580069418,
>pfd=0x560ce8b8dda8) at event.c:1513
> #4  0x560ce83e4590 in listener_thread (thd=0x0, dummy=0x549845279ce62)
>at event.c:1837
> #5  0x7f45967610a4 in start_thread ()
>   from /lib/x86_64-linux-gnu/libpthread.so.0
> #6  0x7f459649662d in clone () from /lib/x86_64-linux-gnu/libc.so.6
> (gdb) (gdb) quit
> 
> Stefan
> Am 26.02.2017 um 19:46 schrieb Stefan Priebe - Profihost AG:
>> Hi Stefan,
>> 
>> currently everything fine. No segfaults.
>> 
>> Stefan
>> 
>> Am 25.02.2017 um 20:40 schrieb Stefan Priebe - Profihost AG:
>>> Hi Stefan,
>>> Am 25.02.2017 um 13:51 schrieb Stefan Eissing:
 Stefan,
 
 whenever you have time, please deploy 
 https://github.com/icing/mod_h2/releases/tag/v1.9.2
 
 I added an own allocator with mutex to the http/2 main session. That is 
 something of a middle-ground between placing that on the main conn (as we 
 had in the crash free version) and 1.9.1 behaviour. Thanks!
>>> 
>>> done. But please keep in mind that this crash might be very rare and
>>> might even have happened with v1.9.0 if we've waited more time.
>>> 
>>> Greets,
>>> Stefan
>>> 
 -Stefan
 
> Am 24.02.2017 um 19:31 schrieb Stefan Priebe - Profihost AG 
> :
> 
> Hi Yann,
> 
> here we go:
> 
> (gdb) p *ipsub
> $1 = {family = 2, sub = {1367753145, 0, 0, 0}, mask = {4294967295,
> 4294967295, 4294967295, 4294967295}}
> 
> (gdb) p *sa
> $2 = {pool = 0x10b500c4ff0b0a, hostname = 0x503040203030102  Cannot access memory at address 0x503040203030102>,
> servname = 0x17d01040405  0x17d01040405>, port = 770, family = 554829073,
> salen = 319177009, ipaddr_len = 570909009, addr_str_len = -2127424399,
> ipaddr_ptr = 0x24f0d15215c1b142,
> next = 0x17160a0982726233, sa = {sin = {sin_family = 6424, sin_port =
> 9498, sin_addr = {s_addr = 690497318},
> sin_zero = "*456789:"}, sin6 = {sin6_family = 6424, sin6_port =
> 9498, sin6_flowinfo = 690497318, sin6_addr = {__in6_u = {
> __u6_addr8 = "*456789:CDEFGHIJ", __u6_addr16 = {13354, 13877,
> 14391, 14905, 17475, 17989, 18503, 19017}, __u6_addr32 = {
>   909456426, 976828471, 1178944579, 1246316615}}},
> sin6_scope_id = 1448432723}, sas = {ss_family = 6424,
> __ss_align = 4195446337656140842,
> __ss_padding =
> "CDEFGHIJSTUVWXYZcdefghijstuvwxyz\203\204\205\206\207\21

Re: httpd 2.4.25, mpm_event, ssl: segfaults

2017-02-27 Thread William A Rowe Jr
On Mon, Feb 27, 2017 at 12:16 PM, Jacob Champion  wrote:
>
> On 02/23/2017 04:48 PM, Yann Ylavic wrote:
>> On Wed, Feb 22, 2017 at 8:55 PM, Daniel Lescohier wrote:
>>>
>>>
>>> IOW: read():Three copies: copy from filesystem cache to httpd
>>> read() buffer to encrypted-data buffer to kernel socket buffer.
>
>>
>> Not really, "copy from filesystem cache to httpd read() buffer" is
>> likely mapping to userspace, so no copy (on read) here.
>
> Oh, cool. Which kernels do this? It seems like the VM tricks would have to
> be incredibly intricate for this to work; reads typically don't happen in
> page-sized chunks, nor to aligned addresses. Linux in particular has
> comments in the source explaining that they *don't* do it for other syscalls
> (e.g. vmsplice)... but I don't have much experience with non-Linux systems.

I don't understand this claim.

If read() returned an API-provisioned buffer, it could point wherever it liked,
including a 4k page. As things stand the void* (or char*) of the read() buffer
is at an arbitrary offset, no common OS I'm familiar with maps a page to
a non-page-aligned address.

The kernel socket send[v]() call might avoid copy in the direct-send case,
depending on the implementation.


Re: release v1.9.2

2017-02-27 Thread Stefan Priebe - Profihost AG
That one breaks everyting. Multiple crashes per second.

[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/local/apache/bin/httpd -DFOREGROUND'.
Program terminated with signal SIGABRT, Aborted.
#0  0x7f02724ea067 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#0  0x7f02724ea067 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x7f02724eb448 in abort () from /lib/x86_64-linux-gnu/libc.so.6
#2  0x7f02724e3266 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#3  0x7f02724e3312 in __assert_fail () from
/lib/x86_64-linux-gnu/libc.so.6
#4  0x7f027287062f in __pthread_tpp_change_priority ()
   from /lib/x86_64-linux-gnu/libpthread.so.0
#5  0x7f0272865e9f in __pthread_mutex_lock_full ()
   from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x7f0272a98db8 in allocator_alloc (in_size=in_size@entry=8032,
allocator=0x7f025c03f3c0) at memory/unix/apr_pools.c:244
#7  apr_allocator_alloc (allocator=0x7f025c03f3c0, size=size@entry=8032)
at memory/unix/apr_pools.c:438
#8  0x7f0272cbcac4 in apr_bucket_alloc (size=8032, size@entry=8000,
list=0x7f022c0008e8) at buckets/apr_buckets_alloc.c:157
#9  0x7f0272cbdca2 in socket_bucket_read (a=0x7f022c000b08,
str=0x7f02627a87b8, len=0x7f02627a87c0, block=)
at buckets/apr_buckets_socket.c:34
#10 0x565098cf1fb1 in ap_core_input_filter (f=0x7f022c0056b0,
b=0x7f022c005630, mode=, block=APR_NONBLOCK_READ,
readbytes=5) at core_filters.c:235
#11 0x565098d3aaca in logio_in_filter (f=,
bb=0x7f022c005630, mode=, block=,
readbytes=) at mod_logio.c:165
#12 0x565098d45b9a in bio_filter_in_read (bio=0x7f024800b460,
in=0x7f02480492a3 "", inlen=5) at ssl_engine_io.c:483
#13 0x7f02736b014c in BIO_read ()
   from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0
#14 0x7f0273a13c92 in ?? () from
/usr/lib/x86_64-linux-gnu/libssl.so.1.0.0
#15 0x7f0273a1548d in ?? () from
/usr/lib/x86_64-linux-gnu/libssl.so.1.0.0
#16 0x7f0273a12024 in ?? () from
/usr/lib/x86_64-linux-gnu/libssl.so.1.0.0
#17 0x565098d469f3 in ssl_io_input_read (inctx=0x7f022c0035b8,
buf=0x7f022c003600 "GET
/media/image/07/3c/c2/4612_13721_160192280322.jpeg HTTP/1.1\r\nHost:
www.onlinestore.it\r\nUser-Agent: curl/7.15+ (x64-criteo) libcurl/7.15+
OpenSSL zlib libidn\r\nAccept: */*\r\n\r\n", len=0x7f02627a8b38)
at ssl_engine_io.c:614
#18 0x565098d493ec in ssl_io_filter_input (f=0x7f022c005608,
bb=0x7f022c006dd0, mode=, block=,
readbytes=8000) at ssl_engine_io.c:1474
#19 0x565098d5cd85 in h2_filter_core_input (f=0x7f022c005980,
brigade=0x7f022c006dd0, mode=738225624, block=APR_NONBLOCK_READ,
readbytes=8000) at h2_filter.c:149
#20 0x565098d6809b in h2_session_read (session=0x7f022c006920,
block=) at h2_session.c:1440
#21 0x565098d6b97f in h2_session_process (session=0x7f022c006920,
async=3964) at h2_session.c:2074
#22 0x565098d5b84e in h2_conn_run (ctx=,
c=0x7f025c03f7d8)
at h2_conn.c:218
#23 0x565098d5e551 in h2_h2_process_conn (c=0x7f025c03f7d8) at
h2_h2.c:658
#24 0x565098d00fd0 in ap_run_process_connection (c=0x7f025c03f7d8)
at connection.c:42
#25 0x565098d9b6d0 in process_socket (my_thread_num=,
my_child_num=, cs=0x7f025c03f748, sock=,
p=, thd=) at event.c:1134
#26 worker_thread (thd=0xf5e, dummy=0xf7c) at event.c:2137
#27 0x7f02728680a4 in start_thread ()
   from /lib/x86_64-linux-gnu/libpthread.so.0
#28 0x7f027259d62d in clone () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) (gdb) quit
Reading symbols from /usr/local/apache/bin/httpd...Reading symbols from
/usr/lib/debug//usr/local/apache2/bin/httpd...done.
done.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/local/apache/bin/httpd -DFOREGROUND'.
Program terminated with signal SIGABRT, Aborted.
#0  0x7f02724ea067 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#0  0x7f02724ea067 in raise () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x7f02724eb448 in abort () from /lib/x86_64-linux-gnu/libc.so.6
#2  0x7f02724e3266 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#3  0x7f02724e3312 in __assert_fail () from
/lib/x86_64-linux-gnu/libc.so.6
#4  0x7f027287062f in __pthread_tpp_change_priority ()
   from /lib/x86_64-linux-gnu/libpthread.so.0
#5  0x7f0272865e9f in __pthread_mutex_lock_full ()
   from /lib/x86_64-linux-gnu/libpthread.so.0
#6  0x7f0272a98db8 in allocator_alloc (in_size=in_size@entry=8032,
allocator=0x7f025c03d320) at memory/unix/apr_pools.c:244
#7  apr_allocator_alloc (allocator=0x7f025c03d320, size=size@entry=8032)
at memory/unix/apr_pools.c:438
#8  0x7f0272cbcac4 in apr_bucket_alloc (size=8032, size@entry=8000,
list=0x7f02340008e8) at buckets/apr_buckets_alloc.c:157
#9  0x7f0272cbdca2 in socket_bucket_read (a=0x7f0234000b08,
str=0x7f0260fa57b8