When handling new connections in 'accept_connections' the number of active connections got increased before the AnyEvent::Handle registered the callback which would decrement it on error/eof.
Any error/die beforehand would skip the decrement, and leave the process in an endless loop upon exiting in wait_end_loop. This can happen e.g. when the call to getpeername fails, or if the connection is denied by the ALLOW_FROM/DENY_FROM settings in '/etc/default/pveproxy' (which is also a simple reproducer for that). Additionally it can cause a denial of service, by attempting to connect from a denied ip until the connection count exeeds the maximum connections of all child-processes. Should the connection count become negative a warning is logged in both places where we decrement it, in case of a failed AnyEvent::Handle->new(), the count is not decremented if this would happen. Reported via our community-forum: https://forum.proxmox.com/threads/pveproxy-eats-available-ram.79617/ Co-Authored-by: Dominik Csapak <d.csa...@proxmox.com> Signed-off-by: Stoiko Ivanov <s.iva...@proxmox.com> --- PVE/APIServer/AnyEvent.pm | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/PVE/APIServer/AnyEvent.pm b/PVE/APIServer/AnyEvent.pm index 7916bdd..af2fde8 100644 --- a/PVE/APIServer/AnyEvent.pm +++ b/PVE/APIServer/AnyEvent.pm @@ -157,6 +157,11 @@ sub client_do_disconnect { &$shutdown_hdl($hdl); + if ($self->{conn_count} <= 0) { + my $msg = "connection count <= 0!\n"; + syslog('warn', $msg); + $self->dprint("warn: $msg"); + } $self->{conn_count}--; print "$$: CLOSE FH" . $hdl->{fh}->fileno() . " CONN$self->{conn_count}\n" if $self->{debug}; @@ -1489,8 +1494,6 @@ sub accept { fh_nonblocking $clientfh, 1; - $self->{conn_count}++; - return $clientfh; } @@ -1564,6 +1567,7 @@ sub check_host_access { sub accept_connections { my ($self) = @_; + my $hdl_err; eval { while (my $clientfh = $self->accept()) { @@ -1571,7 +1575,7 @@ sub accept_connections { my $reqstate = { keep_alive => $self->{keep_alive} }; # stop keep-alive when there are many open connections - if ($self->{conn_count} >= $self->{max_conn_soft_limit}) { + if ($self->{conn_count} + 1 >= $self->{max_conn_soft_limit}) { $reqstate->{keep_alive} = 0; } @@ -1587,6 +1591,8 @@ sub accept_connections { next; } + $hdl_err = 1; + $self->{conn_count}++; $reqstate->{hdl} = AnyEvent::Handle->new( fh => $clientfh, rbuf_max => 64*1024, @@ -1609,6 +1615,7 @@ sub accept_connections { if (my $err = $@) { syslog('err', "$err"); } }, ($self->{tls_ctx} ? (tls => "accept", tls_ctx => $self->{tls_ctx}) : ())); + $hdl_err = 0; print "$$: ACCEPT FH" . $clientfh->fileno() . " CONN$self->{conn_count}\n" if $self->{debug}; @@ -1618,6 +1625,15 @@ sub accept_connections { if (my $err = $@) { syslog('err', $err); + if ($hdl_err) { + if ($self->{conn_count} <= 0) { + my $msg = "connection count <= 0 not decrementing!\n"; + syslog('warn', $msg); + $self->dprint("warn: $msg"); + } else { + $self->{conn_count}--; + } + } $self->{end_loop} = 1; } -- 2.20.1 _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
