From 19180b6a7c2e17ec8d7ad44385f2cab28cd62c5f Mon Sep 17 00:00:00 2001 From: Mingli Zhang Date: Mon, 1 Aug 2022 21:13:32 +0800 Subject: [PATCH vn] COPY FROM enable FORCE_NULL/FORCE_NOT_NULL on all columns We already have FORCE_NULL/FORCE_NOT_NULL options to force null string to be converted to NULL or empty string. But users must set the columns one by one. CREATE TABLE forcetest ( a INT NOT NULL, b TEXT NOT NULL, c TEXT, d TEXT, e TEXT ); \pset null NULL BEGIN; COPY forcetest (a, b, c, d) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL(c,d), FORCE_NULL(c,d)); 1,'a',,"" \. COMMIT; SELECT c, d FROM forcetest WHERE a = 1; c | d ---+------ | NULL (1 row) This commit enables FORCE_NULL/FORCE_NOT_NULL options to select all columns of a table to be converted. Like FORCE_QUOTE * , the sql format are FORCE_NULL * and FORCE_NOT_NULL *. FORCE_NULL * and FORCE_NOT_NULL * can be used simultaneously as before. BEGIN COPY forcetest (a, b, c, d) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NULL *); 2,'b',,"" \. COMMIT; SELECT c, d FROM forcetest WHERE a = 2; c | d ---+------ | NULL (1 row) --- doc/src/sgml/ref/copy.sgml | 6 ++-- src/backend/commands/copy.c | 12 +++++--- src/backend/commands/copyfrom.c | 16 +++++++++-- src/backend/parser/gram.y | 8 ++++++ src/include/commands/copy.h | 2 ++ src/test/regress/expected/copy2.out | 44 +++++++++++++++++++++++++++++ src/test/regress/sql/copy2.sql | 30 ++++++++++++++++++++ 7 files changed, 110 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 8aae711b3b..d20f4494d6 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -40,8 +40,8 @@ COPY { table_name [ ( quote_character' ESCAPE 'escape_character' FORCE_QUOTE { ( column_name [, ...] ) | * } - FORCE_NOT_NULL ( column_name [, ...] ) - FORCE_NULL ( column_name [, ...] ) + FORCE_NOT_NULL { ( column_name [, ...] ) | * } + FORCE_NULL { ( column_name [, ...] ) | * } ENCODING 'encoding_name' @@ -336,6 +336,7 @@ COPY { table_name [ ( * is specified, it will be applied in all columns. This option is allowed only in COPY FROM, and only when using CSV format. @@ -350,6 +351,7 @@ COPY { table_name [ ( NULL. In the default case where the null string is empty, this converts a quoted empty string into NULL. + If * is specified, it will be applied in all columns. This option is allowed only in COPY FROM, and only when using CSV format. diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 3ac731803b..30bdcc9702 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -490,9 +490,11 @@ ProcessCopyOptions(ParseState *pstate, } else if (strcmp(defel->defname, "force_not_null") == 0) { - if (opts_out->force_notnull) + if (opts_out->force_notnull || opts_out->force_notnull_all) errorConflictingDefElem(defel, pstate); - if (defel->arg && IsA(defel->arg, List)) + if (defel->arg && IsA(defel->arg, A_Star)) + opts_out->force_notnull_all = true; + else if (defel->arg && IsA(defel->arg, List)) opts_out->force_notnull = castNode(List, defel->arg); else ereport(ERROR, @@ -503,9 +505,11 @@ ProcessCopyOptions(ParseState *pstate, } else if (strcmp(defel->defname, "force_null") == 0) { - if (opts_out->force_null) + if (opts_out->force_null || opts_out->force_null_all) errorConflictingDefElem(defel, pstate); - if (defel->arg && IsA(defel->arg, List)) + if (defel->arg && IsA(defel->arg, A_Star)) + opts_out->force_null_all = true; + else if (defel->arg && IsA(defel->arg, List)) opts_out->force_null = castNode(List, defel->arg); else ereport(ERROR, diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index a976008b3d..49ba40a8dd 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1249,7 +1249,13 @@ BeginCopyFrom(ParseState *pstate, /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */ cstate->opts.force_notnull_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool)); - if (cstate->opts.force_notnull) + if (cstate->opts.force_notnull_all) + { + int i; + for(i = 0; i < num_phys_attrs; i++) + cstate->opts.force_notnull_flags[i] = true; + } + else if (cstate->opts.force_notnull) { List *attnums; ListCell *cur; @@ -1272,7 +1278,13 @@ BeginCopyFrom(ParseState *pstate, /* Convert FORCE_NULL name list to per-column flags, check validity */ cstate->opts.force_null_flags = (bool *) palloc0(num_phys_attrs * sizeof(bool)); - if (cstate->opts.force_null) + if (cstate->opts.force_null_all) + { + int i; + for(i = 0; i < num_phys_attrs; i++) + cstate->opts.force_null_flags[i] = true; + } + else if (cstate->opts.force_null) { List *attnums; ListCell *cur; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index f9037761f9..8b417b6c4f 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -3510,10 +3510,18 @@ copy_opt_item: { $$ = makeDefElem("force_not_null", (Node *) $4, @1); } + | FORCE NOT NULL_P '*' + { + $$ = makeDefElem("force_not_null", (Node *) makeNode(A_Star), @1); + } | FORCE NULL_P columnList { $$ = makeDefElem("force_null", (Node *) $3, @1); } + | FORCE NULL_P '*' + { + $$ = makeDefElem("force_null", (Node *) makeNode(A_Star), @1); + } | ENCODING Sconst { $$ = makeDefElem("encoding", (Node *) makeString($2), @1); diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index cb0096aeb6..de5edf7584 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -54,8 +54,10 @@ typedef struct CopyFormatOptions bool force_quote_all; /* FORCE_QUOTE *? */ bool *force_quote_flags; /* per-column CSV FQ flags */ List *force_notnull; /* list of column names */ + bool force_notnull_all; /* FORCE_NOT_NULL * */ bool *force_notnull_flags; /* per-column CSV FNN flags */ List *force_null; /* list of column names */ + bool force_null_all; /* FORCE_NULL * */ bool *force_null_flags; /* per-column CSV FN flags */ bool convert_selectively; /* do selective binary conversion? */ List *convert_select; /* list of column names (can be NIL) */ diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index 5f3685e9ef..b4ea80a268 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -503,6 +503,50 @@ BEGIN; COPY forcetest (d, e) FROM STDIN WITH (FORMAT csv, FORCE_NULL(b)); ERROR: FORCE_NULL column "b" not referenced by COPY ROLLBACK; +-- should succeed with no effect ("b" remains an empty string, "c" remains NULL) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NULL *); +COMMIT; +SELECT b, c FROM forcetest WHERE a = 4; + b | c +---+------ + | NULL +(1 row) + +-- should succeed with effect ("b" remains an empty string) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *); +COMMIT; +SELECT b, c FROM forcetest WHERE a = 5; + b | c +---+--- + | +(1 row) + +-- should succeed with effect ("c" remains NULL) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *); +COMMIT; +SELECT b, c FROM forcetest WHERE a = 6; + b | c +---+------ + b | NULL +(1 row) + +-- should fail with "conflicting or redundant options" error +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NOT_NULL(b)); +ERROR: conflicting or redundant options +LINE 1: ...c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NOT_... + ^ +ROLLBACK; +-- should fail with "conflicting or redundant options" error +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *, FORCE_NULL(b)); +ERROR: conflicting or redundant options +LINE 1: ... b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *, FORCE_NULL... + ^ +ROLLBACK; \pset null '' -- test case with whole-row Var in a check constraint create table check_con_tbl (f1 int); diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql index b3c16af48e..528c36f92c 100644 --- a/src/test/regress/sql/copy2.sql +++ b/src/test/regress/sql/copy2.sql @@ -334,6 +334,36 @@ ROLLBACK; BEGIN; COPY forcetest (d, e) FROM STDIN WITH (FORMAT csv, FORCE_NULL(b)); ROLLBACK; +-- should succeed with no effect ("b" remains an empty string, "c" remains NULL) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NULL *); +4,,"" +\. +COMMIT; +SELECT b, c FROM forcetest WHERE a = 4; +-- should succeed with effect ("b" remains an empty string) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *); +5,,"" +\. +COMMIT; +SELECT b, c FROM forcetest WHERE a = 5; +-- should succeed with effect ("c" remains NULL) +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *); +6,"b","" +\. +COMMIT; +SELECT b, c FROM forcetest WHERE a = 6; +-- should fail with "conflicting or redundant options" error +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NOT_NULL *, FORCE_NOT_NULL(b)); +ROLLBACK; +-- should fail with "conflicting or redundant options" error +BEGIN; +COPY forcetest (a, b, c) FROM STDIN WITH (FORMAT csv, FORCE_NULL *, FORCE_NULL(b)); +ROLLBACK; + \pset null '' -- test case with whole-row Var in a check constraint -- 2.34.1