Hello, hackers!
While trying to test a patch that adds a synchronization barrier in
pgbench [1] on Windows, I found that since the commit "Use ppoll(2), if
available, to wait for input in pgbench." [2] I cannot use a large
number of client connections in pgbench on my Windows virtual machines
(Windows Server 2008 R2 and Windows 2019), for example:
bin\pgbench.exe -c 90 -S -T 3 postgres
starting vacuum...end.
too many client connections for select()
The almost same thing happens with reindexdb and vacuumdb (build on
commit [3]):
bin\reindexdb.exe -j 95 postgres
reindexdb: fatal: too many jobs for this platform -- try 90
bin\vacuumdb.exe -j 95 postgres
vacuumdb: vacuuming database "postgres"
vacuumdb: fatal: too many jobs for this platform -- try 90
IIUC the checks below are not correct on Windows, since on this system
sockets can have values equal to or greater than FD_SETSIZE (see Windows
documentation [4] and pgbench debug output in attached
pgbench_debug.txt).
src/bin/pgbench/pgbench.c, the function add_socket_to_set:
if (fd < 0 || fd >= FD_SETSIZE)
{
/*
* Doing a hard exit here is a bit grotty, but it doesn't seem worth
* complicating the API to make it less grotty.
*/
pg_log_fatal("too many client connections for select()");
exit(1);
}
src/bin/scripts/scripts_parallel.c, the function ParallelSlotsSetup:
/*
* Fail and exit immediately if trying to use a socket in an
* unsupported range. POSIX requires open(2) to use the lowest
* unused file descriptor and the hint given relies on that.
*/
if (PQsocket(conn) >= FD_SETSIZE)
{
pg_log_fatal("too many jobs for this platform -- try %d", i);
exit(1);
}
I tried to fix this, see attached fix_max_client_conn_on_Windows.patch
(based on commit [3]). I checked it for reindexdb and vacuumdb, and it
works for simple databases (1025 jobs are not allowed and 1024 jobs is
ok). Unfortunately, pgbench was getting connection errors when it tried
to use 1000 jobs on my virtual machines, although there were no errors
for fewer jobs (500) and the same number of clients (1000)...
Any suggestions are welcome!
[1]
https://www.postgresql.org/message-id/flat/20200227180100.zyvjwzcpiokfsqm2%40alap3.anarazel.de
[2]
https://github.com/postgres/postgres/commit/60e612b602999e670f2d57a01e52799eaa903ca9
[3]
https://github.com/postgres/postgres/commit/48e1291342dd7771cf8c67aa1d7ec1f394b95dd8
[4] From
https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-select
:
Internally, socket handles in an fd_set structure are not represented as
bit flags as in Berkeley Unix. Their data representation is opaque.
--
Marina Polyakova
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Company
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 3057665bbec567331ad5ea03d31af707f5e91b4c..7a54638db191982d538cabf007d82715fa254b6a 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -62,6 +62,7 @@
#include "common/string.h"
#include "fe_utils/cancel.h"
#include "fe_utils/conditional.h"
+#include "fe_utils/socket_utils.h"
#include "getopt_long.h"
#include "libpq-fe.h"
#include "pgbench.h"
@@ -6698,7 +6699,7 @@ clear_socket_set(socket_set *sa)
static void
add_socket_to_set(socket_set *sa, int fd, int idx)
{
- if (fd < 0 || fd >= FD_SETSIZE)
+ if (fd < 0 || !check_fd_set_size(fd, &sa->fds))
{
/*
* Doing a hard exit here is a bit grotty, but it doesn't seem worth
diff --git a/src/bin/scripts/scripts_parallel.c b/src/bin/scripts/scripts_parallel.c
index ec264a269a7d7a506c347112af6be183b2aec9ce..61977741c7c5f76b1966ab8e59792a1d2b53b934 100644
--- a/src/bin/scripts/scripts_parallel.c
+++ b/src/bin/scripts/scripts_parallel.c
@@ -25,6 +25,7 @@
#include "common.h"
#include "common/logging.h"
#include "fe_utils/cancel.h"
+#include "fe_utils/socket_utils.h"
#include "scripts_parallel.h"
static void init_slot(ParallelSlot *slot, PGconn *conn);
@@ -144,6 +145,16 @@ ParallelSlotsGetIdle(ParallelSlot *slots, int numslots)
if (sock < 0)
continue;
+ /*
+ * Fail and exit immediately if trying to use a socket in an
+ * unsupported range.
+ */
+ if (!check_fd_set_size(sock, &slotset))
+ {
+ pg_log_fatal("too many jobs for this platform -- try %d", i);
+ exit(1);
+ }
+
FD_SET(sock, &slotset);
if (sock > maxFd)
maxFd = sock;
@@ -221,18 +232,6 @@ ParallelSlotsSetup(const ConnParams *cparams,
for (i = 1; i < numslots; i++)
{
conn = connectDatabase(cparams, progname, echo, false, true);
-
- /*
- * Fail and exit immediately if trying to use a socket in an
- * unsupported range. POSIX requires open(2) to use the lowest
- * unused file descriptor and the hint given relies on that.
- */
- if (PQsocket(conn) >= FD_SETSIZE)
- {
- pg_log_fatal("too many jobs for this platform -- try %d", i);
- exit(1);
- }
-
init_slot(slots + i, conn);
}
}
diff --git a/src/include/fe_utils/socket_utils.h b/src/include/fe_utils/socket_utils.h
new file mode 100644
index 0000000000000000000000000000000000000000..31f00c869631a8845823d99c74045557d6272e19
--- /dev/null
+++ b/src/include/fe_utils/socket_utils.h
@@ -0,0 +1,38 @@
+/*-------------------------------------------------------------------------
+ *
+ * Socket-processing utility routines for frontend code
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/fe_utils/socket_utils.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef SOCKET_UTILS_H
+#define SOCKET_UTILS_H
+
+/*
+ * Return true if the file descriptor can be added to the set with the size
+ * FD_SETSIZE and false otherwise.
+ */
+static inline bool
+check_fd_set_size(int fd, const fd_set *fds)
+{
+#ifdef WIN32
+ /*
+ * We cannot check the socket value at runtime because on Windows it can be
+ * greater than or equal to FD_SETSIZE.
+ */
+ Assert(fds != NULL);
+ return (fds->fd_count < FD_SETSIZE);
+#else /* !WIN32 */
+ /*
+ * POSIX: the behavior of the macro FD_SET is undefined if the fd argument
+ * is less than 0 or greater than or equal to FD_SETSIZE.
+ */
+ return (fd < FD_SETSIZE);
+#endif /* !WIN32 */
+}
+
+#endif /* SOCKET_UTILS_H */
starting vacuum...end.
add_socket_to_set client 0 fd 328 FD_SETSIZE 1024
add_socket_to_set client 1 fd 340 FD_SETSIZE 1024
add_socket_to_set client 2 fd 348 FD_SETSIZE 1024
add_socket_to_set client 3 fd 356 FD_SETSIZE 1024
add_socket_to_set client 4 fd 364 FD_SETSIZE 1024
add_socket_to_set client 5 fd 372 FD_SETSIZE 1024
add_socket_to_set client 6 fd 380 FD_SETSIZE 1024
add_socket_to_set client 7 fd 388 FD_SETSIZE 1024
add_socket_to_set client 8 fd 396 FD_SETSIZE 1024
add_socket_to_set client 9 fd 404 FD_SETSIZE 1024
add_socket_to_set client 10 fd 412 FD_SETSIZE 1024
add_socket_to_set client 11 fd 420 FD_SETSIZE 1024
add_socket_to_set client 12 fd 428 FD_SETSIZE 1024
add_socket_to_set client 13 fd 436 FD_SETSIZE 1024
add_socket_to_set client 14 fd 444 FD_SETSIZE 1024
add_socket_to_set client 15 fd 452 FD_SETSIZE 1024
add_socket_to_set client 16 fd 460 FD_SETSIZE 1024
add_socket_to_set client 17 fd 468 FD_SETSIZE 1024
add_socket_to_set client 18 fd 476 FD_SETSIZE 1024
add_socket_to_set client 19 fd 484 FD_SETSIZE 1024
add_socket_to_set client 20 fd 492 FD_SETSIZE 1024
add_socket_to_set client 21 fd 500 FD_SETSIZE 1024
add_socket_to_set client 22 fd 508 FD_SETSIZE 1024
add_socket_to_set client 23 fd 516 FD_SETSIZE 1024
add_socket_to_set client 24 fd 524 FD_SETSIZE 1024
add_socket_to_set client 25 fd 532 FD_SETSIZE 1024
add_socket_to_set client 26 fd 540 FD_SETSIZE 1024
add_socket_to_set client 27 fd 548 FD_SETSIZE 1024
add_socket_to_set client 28 fd 556 FD_SETSIZE 1024
add_socket_to_set client 29 fd 564 FD_SETSIZE 1024
add_socket_to_set client 30 fd 572 FD_SETSIZE 1024
add_socket_to_set client 31 fd 580 FD_SETSIZE 1024
add_socket_to_set client 32 fd 588 FD_SETSIZE 1024
add_socket_to_set client 33 fd 596 FD_SETSIZE 1024
add_socket_to_set client 34 fd 604 FD_SETSIZE 1024
add_socket_to_set client 35 fd 612 FD_SETSIZE 1024
add_socket_to_set client 36 fd 620 FD_SETSIZE 1024
add_socket_to_set client 37 fd 628 FD_SETSIZE 1024
add_socket_to_set client 38 fd 636 FD_SETSIZE 1024
add_socket_to_set client 39 fd 644 FD_SETSIZE 1024
add_socket_to_set client 40 fd 652 FD_SETSIZE 1024
add_socket_to_set client 41 fd 660 FD_SETSIZE 1024
add_socket_to_set client 42 fd 668 FD_SETSIZE 1024
add_socket_to_set client 43 fd 676 FD_SETSIZE 1024
add_socket_to_set client 44 fd 684 FD_SETSIZE 1024
add_socket_to_set client 45 fd 692 FD_SETSIZE 1024
add_socket_to_set client 46 fd 700 FD_SETSIZE 1024
add_socket_to_set client 47 fd 708 FD_SETSIZE 1024
add_socket_to_set client 48 fd 716 FD_SETSIZE 1024
add_socket_to_set client 49 fd 724 FD_SETSIZE 1024
add_socket_to_set client 50 fd 732 FD_SETSIZE 1024
add_socket_to_set client 51 fd 740 FD_SETSIZE 1024
add_socket_to_set client 52 fd 748 FD_SETSIZE 1024
add_socket_to_set client 53 fd 756 FD_SETSIZE 1024
add_socket_to_set client 54 fd 764 FD_SETSIZE 1024
add_socket_to_set client 55 fd 772 FD_SETSIZE 1024
add_socket_to_set client 56 fd 780 FD_SETSIZE 1024
add_socket_to_set client 57 fd 788 FD_SETSIZE 1024
add_socket_to_set client 58 fd 796 FD_SETSIZE 1024
add_socket_to_set client 59 fd 804 FD_SETSIZE 1024
add_socket_to_set client 60 fd 812 FD_SETSIZE 1024
add_socket_to_set client 61 fd 820 FD_SETSIZE 1024
add_socket_to_set client 62 fd 828 FD_SETSIZE 1024
add_socket_to_set client 63 fd 836 FD_SETSIZE 1024
add_socket_to_set client 64 fd 844 FD_SETSIZE 1024
add_socket_to_set client 65 fd 852 FD_SETSIZE 1024
add_socket_to_set client 66 fd 860 FD_SETSIZE 1024
add_socket_to_set client 67 fd 868 FD_SETSIZE 1024
add_socket_to_set client 68 fd 876 FD_SETSIZE 1024
add_socket_to_set client 69 fd 884 FD_SETSIZE 1024
add_socket_to_set client 70 fd 892 FD_SETSIZE 1024
add_socket_to_set client 71 fd 900 FD_SETSIZE 1024
add_socket_to_set client 72 fd 908 FD_SETSIZE 1024
add_socket_to_set client 73 fd 916 FD_SETSIZE 1024
add_socket_to_set client 74 fd 924 FD_SETSIZE 1024
add_socket_to_set client 75 fd 932 FD_SETSIZE 1024
add_socket_to_set client 76 fd 940 FD_SETSIZE 1024
add_socket_to_set client 77 fd 948 FD_SETSIZE 1024
add_socket_to_set client 78 fd 956 FD_SETSIZE 1024
add_socket_to_set client 79 fd 964 FD_SETSIZE 1024
add_socket_to_set client 80 fd 972 FD_SETSIZE 1024
add_socket_to_set client 81 fd 980 FD_SETSIZE 1024
add_socket_to_set client 82 fd 988 FD_SETSIZE 1024
add_socket_to_set client 83 fd 996 FD_SETSIZE 1024
add_socket_to_set client 84 fd 1004 FD_SETSIZE 1024
add_socket_to_set client 85 fd 1012 FD_SETSIZE 1024
add_socket_to_set client 86 fd 1020 FD_SETSIZE 1024
add_socket_to_set client 87 fd 1032 FD_SETSIZE 1024
too many client connections for select()
