From bf4fcb14c19d222a32053e7098f53e8aed7229ef Mon Sep 17 00:00:00 2001 From: "houzj.fnst" Date: Fri, 21 May 2021 15:39:07 +0800 Subject: [PATCH] limit-the-fdw-batch-size --- contrib/postgres_fdw/postgres_fdw.c | 12 +++++++++++- doc/src/sgml/postgres-fdw.sgml | 11 +++++++++-- src/interfaces/libpq/fe-exec.c | 21 ++++++++++++--------- src/interfaces/libpq/libpq-fe.h | 1 + 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index c48a421e88..9215c10bf3 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -1994,7 +1994,17 @@ postgresGetForeignModifyBatchSize(ResultRelInfo *resultRelInfo) resultRelInfo->ri_TrigDesc->trig_insert_after_row)) return 1; - /* Otherwise use the batch size specified for server/table. */ + /* + * Adjust the batch_size to make sure the number of parameters in a batch + * does not exceed maximum number of parameters supported by the FE/BE + * protocol. + */ + Assert(fmstate->p_nums > 0); + if (batch_size * fmstate->p_nums > PQ_QUERY_PARAM_MAX_LIMIT) + { + batch_size = PQ_QUERY_PARAM_MAX_LIMIT / fmstate->p_nums; + } + return batch_size; } diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml index fb87372bde..d77cf1c6d7 100644 --- a/doc/src/sgml/postgres-fdw.sgml +++ b/doc/src/sgml/postgres-fdw.sgml @@ -367,9 +367,16 @@ OPTIONS (ADD password_required 'false'); This option specifies the number of rows postgres_fdw - should insert in each insert operation. It can be specified for a + inserts in each insert operation. It can be specified for a foreign table or a foreign server. The option specified on a table - overrides an option specified for the server. + overrides an option specified for the server. Note the final number + of rows postgres_fdw inserts in a batch actually depends on the + number of columns and the provided batch_size value. This is because + of the limit the libpq protocol (which postgres_fdw uses to connect + to a remote server) has on the number of query parameters that can + be specified per query. For instance, if the number of columns * batch_size + is more than the limit, then the libpq emits an error. But postgres_fdw + adjusts the batch_size to avoid this error. The default is 1. diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c index 03592bdce9..3876093250 100644 --- a/src/interfaces/libpq/fe-exec.c +++ b/src/interfaces/libpq/fe-exec.c @@ -1403,10 +1403,11 @@ PQsendQueryParams(PGconn *conn, libpq_gettext("command string is a null pointer\n")); return 0; } - if (nParams < 0 || nParams > 65535) + if (nParams < 0 || nParams > PQ_QUERY_PARAM_MAX_LIMIT) { - appendPQExpBufferStr(&conn->errorMessage, - libpq_gettext("number of parameters must be between 0 and 65535\n")); + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("number of parameters must be between 0 and %d\n"), + PQ_QUERY_PARAM_MAX_LIMIT); return 0; } @@ -1451,10 +1452,11 @@ PQsendPrepare(PGconn *conn, libpq_gettext("command string is a null pointer\n")); return 0; } - if (nParams < 0 || nParams > 65535) + if (nParams < 0 || nParams > PQ_QUERY_PARAM_MAX_LIMIT) { - appendPQExpBufferStr(&conn->errorMessage, - libpq_gettext("number of parameters must be between 0 and 65535\n")); + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("number of parameters must be between 0 and %d\n"), + PQ_QUERY_PARAM_MAX_LIMIT); return 0; } @@ -1548,10 +1550,11 @@ PQsendQueryPrepared(PGconn *conn, libpq_gettext("statement name is a null pointer\n")); return 0; } - if (nParams < 0 || nParams > 65535) + if (nParams < 0 || nParams > PQ_QUERY_PARAM_MAX_LIMIT) { - appendPQExpBufferStr(&conn->errorMessage, - libpq_gettext("number of parameters must be between 0 and 65535\n")); + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("number of parameters must be between 0 and %d\n"), + PQ_QUERY_PARAM_MAX_LIMIT); return 0; } diff --git a/src/interfaces/libpq/libpq-fe.h b/src/interfaces/libpq/libpq-fe.h index 227adde5a5..13cf9a9e26 100644 --- a/src/interfaces/libpq/libpq-fe.h +++ b/src/interfaces/libpq/libpq-fe.h @@ -429,6 +429,7 @@ extern PGresult *PQexecPrepared(PGconn *conn, int resultFormat); /* Interface for multiple-result or asynchronous queries */ +#define PQ_QUERY_PARAM_MAX_LIMIT 65535 extern int PQsendQuery(PGconn *conn, const char *query); extern int PQsendQueryParams(PGconn *conn, const char *command, -- 2.18.4