From bc9c06da1b84b535193f5f56c81c4a33b4254d87 Mon Sep 17 00:00:00 2001 From: Joel Jacobson Date: Sat, 12 Oct 2024 08:02:49 +0200 Subject: [PATCH 3/3] Add "raw" COPY format support for unstructured text data. This commit introduces a new format option to the COPY command, enabling the import and export of unstructured text data where each line is treated as a single field without any delimiters. A new typedef enum CopyFormat is introduced, and the new CopyFormatOptions field "format" replaces the "binary" and "csv_mode" fields. This commit also refactors ProcessCopyOptions(), separating the validation of all options into their own sections. It moves the binary format validations under respective option checks. For clarity, explicitly check `on_error != COPY_ON_ERROR_IGNORE` instead of `!on_error`. Also update comment for the section of code at the end of ProcessCopyOptions() that now is dedicated to additional checks for interdependent options. --- doc/src/sgml/ref/copy.sgml | 98 +++- src/backend/commands/copy.c | 485 +++++++++++------- src/backend/commands/copyfrom.c | 17 +- src/backend/commands/copyfromparse.c | 238 ++++++++- src/backend/commands/copyto.c | 88 +++- src/backend/parser/gram.y | 8 +- src/include/commands/copy.h | 14 +- src/include/parser/kwlist.h | 1 + src/test/regress/data/newlines_cr.data | 1 + src/test/regress/data/newlines_cr_lr.data | 2 + .../regress/data/newlines_cr_lr_nolast.data | 2 + src/test/regress/data/newlines_cr_nolast.data | 1 + src/test/regress/data/newlines_lr.data | 2 + src/test/regress/data/newlines_lr_nolast.data | 2 + src/test/regress/data/newlines_mixed_1.data | 1 + src/test/regress/data/newlines_mixed_2.data | 2 + src/test/regress/data/newlines_mixed_3.data | 2 + src/test/regress/data/newlines_mixed_4.data | 2 + src/test/regress/data/newlines_mixed_5.data | 2 + src/test/regress/expected/copy.out | 92 ++++ src/test/regress/expected/copy2.out | 57 +- src/test/regress/sql/copy.sql | 43 ++ src/test/regress/sql/copy2.sql | 43 +- src/tools/pgindent/typedefs.list | 1 + 24 files changed, 958 insertions(+), 246 deletions(-) create mode 100644 src/test/regress/data/newlines_cr.data create mode 100644 src/test/regress/data/newlines_cr_lr.data create mode 100644 src/test/regress/data/newlines_cr_lr_nolast.data create mode 100644 src/test/regress/data/newlines_cr_nolast.data create mode 100644 src/test/regress/data/newlines_lr.data create mode 100644 src/test/regress/data/newlines_lr_nolast.data create mode 100644 src/test/regress/data/newlines_mixed_1.data create mode 100644 src/test/regress/data/newlines_mixed_2.data create mode 100644 src/test/regress/data/newlines_mixed_3.data create mode 100644 src/test/regress/data/newlines_mixed_4.data create mode 100644 src/test/regress/data/newlines_mixed_5.data diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 8394402f09..06ca632ee3 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -218,8 +218,9 @@ COPY { table_name [ ( Selects the data format to be read or written: text, - csv (Comma Separated Values), - or binary. + CSV (Comma Separated Values), + binary, + or raw The default is text. See below for details. @@ -257,7 +258,8 @@ COPY { table_name [ ( CSV format. This must be a single one-byte character. - This option is not allowed when using binary format. + This option is allowed only when using text or + CSV format. @@ -271,7 +273,8 @@ COPY { table_name [ ( CSV format. You might prefer an empty string even in text format for cases where you don't want to distinguish nulls from empty strings. - This option is not allowed when using binary format. + This option is allowed only when using text or + CSV format. @@ -294,7 +297,7 @@ COPY { table_name [ ( COPY FROM, and only when - not using binary format. + using text or CSV format. @@ -400,7 +403,8 @@ COPY { table_name [ ( The ignore option is applicable only for COPY FROM - when the FORMAT is text or csv. + when the FORMAT is text, + CSV or raw. A NOTICE message containing the ignored row count is @@ -893,6 +897,88 @@ COPY count + + Raw Format + + + This format option is used for importing and exporting files containing + unstructured text, where each line is treated as a single field. It is + ideal for data that does not conform to a structured, tabular format and + lacks delimiters. + + + + In the raw format, each line of the input or output is + considered a complete value without any field separation. There are no + field delimiters, and all characters are taken literally. There is no + special handling for quotes, backslashes, or escape sequences. All + characters, including whitespace and special characters, are preserved + exactly as they appear in the file. However, it's important to note that + the text is still interpreted according to the specified ENCODING + option or the current client encoding for input, and encoded using the + specified ENCODING or the current client encoding for output. + + + + When using this format, the COPY command must specify + exactly one column. Specifying multiple columns will result in an error. + If the table has multiple columns and no column list is provided, an error + will occur. + + + + The raw format does not distinguish a NULL + value from an empty string. Empty lines are imported as empty strings, not + as NULL values. + + + + Encoding works the same as in the text and CSV formats. + + + + + + Raw Format + + + This format option is used for importing and exporting files containing + unstructured text, where each line is treated as a single field. It is + ideal for data that does not conform to a structured, tabular format and + lacks delimiters. + + + + In the raw format, each line of the input or output is + considered a complete value without any field separation. There are no + field delimiters, and all characters are taken literally. There is no + special handling for quotes, backslashes, or escape sequences. All + characters, including whitespace and special characters, are preserved + exactly as they appear in the file. However, it's important to note that + the text is still interpreted according to the specified ENCODING + option or the current client encoding for input, and encoded using the + specified ENCODING or the current client encoding for output. + + + + When using this format, the COPY command must specify + exactly one column. Specifying multiple columns will result in an error. + If the table has multiple columns and no column list is provided, an error + will occur. + + + + The raw format does not distinguish a NULL + value from an empty string. Empty lines are imported as empty strings, not + as NULL values. + + + + Encoding works the same as in the text and CSV formats. + + + + Binary Format diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index e93ea3d627..74d6ebb78d 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -511,11 +511,13 @@ ProcessCopyOptions(ParseState *pstate, errorConflictingDefElem(defel, pstate); format_specified = true; if (strcmp(fmt, "text") == 0) - /* default format */ ; + opts_out->format = COPY_FORMAT_TEXT; else if (strcmp(fmt, "csv") == 0) - opts_out->csv_mode = true; + opts_out->format = COPY_FORMAT_CSV; else if (strcmp(fmt, "binary") == 0) - opts_out->binary = true; + opts_out->format = COPY_FORMAT_BINARY; + else if (strcmp(fmt, "raw") == 0) + opts_out->format = COPY_FORMAT_RAW; else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -672,62 +674,150 @@ ProcessCopyOptions(ParseState *pstate, } /* - * Check for incompatible options (must do these three before inserting - * defaults) + * Set default format if not specified. + * This isn't strictly necessary since COPY_FORMAT_TEXT is 0 and + * opts_out is palloc0'd, but do it for clarity. */ - if (opts_out->binary && opts_out->delim) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ - errmsg("cannot specify %s in BINARY mode", "DELIMITER"))); - - if (opts_out->binary && opts_out->null_print) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("cannot specify %s in BINARY mode", "NULL"))); - - if (opts_out->binary && opts_out->default_print) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("cannot specify %s in BINARY mode", "DEFAULT"))); - - /* Set defaults for omitted options */ - if (!opts_out->delim) - opts_out->delim = opts_out->csv_mode ? "," : "\t"; - - if (!opts_out->null_print) - opts_out->null_print = opts_out->csv_mode ? "" : "\\N"; - opts_out->null_print_len = strlen(opts_out->null_print); - - if (opts_out->csv_mode) + if (!format_specified) + opts_out->format = COPY_FORMAT_TEXT; + + /* --- DELIMITER option --- */ + if (opts_out->delim) + { + if (opts_out->format == COPY_FORMAT_BINARY) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ + errmsg("cannot specify %s in BINARY mode", "DELIMITER"))); + + if (opts_out->format == COPY_FORMAT_RAW) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ + errmsg("cannot specify %s in RAW mode", "DELIMITER"))); + + /* Only single-byte delimiter strings are supported. */ + if (strlen(opts_out->delim) != 1) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY delimiter must be a single one-byte character"))); + + /* Disallow end-of-line characters */ + if (strchr(opts_out->delim, '\r') != NULL || + strchr(opts_out->delim, '\n') != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY delimiter cannot be newline or carriage return"))); + + /* + * Disallow unsafe delimiter characters in non-CSV mode. We can't allow + * backslash because it would be ambiguous. We can't allow the other + * cases because data characters matching the delimiter must be + * backslashed, and certain backslash combinations are interpreted + * non-literally by COPY IN. Disallowing all lower case ASCII letters is + * more than strictly necessary, but seems best for consistency and + * future-proofing. Likewise we disallow all digits though only octal + * digits are actually dangerous. + */ + if (opts_out->format != COPY_FORMAT_CSV && + strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789", + opts_out->delim[0]) != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY delimiter cannot be \"%s\"", opts_out->delim))); + } + /* Set default delimiter */ + else if (opts_out->format == COPY_FORMAT_CSV) + opts_out->delim = ","; + else if (opts_out->format == COPY_FORMAT_TEXT) + opts_out->delim = "\t"; + + /* --- NULL option --- */ + if (opts_out->null_print) + { + if (opts_out->format == COPY_FORMAT_BINARY) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot specify %s in BINARY mode", "NULL"))); + + if (opts_out->format == COPY_FORMAT_RAW) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot specify %s in RAW mode", "NULL"))); + + /* Disallow end-of-line characters */ + if (strchr(opts_out->null_print, '\r') != NULL || + strchr(opts_out->null_print, '\n') != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY null representation cannot use newline or carriage return"))); + } + /* Set default null_print */ + else if (opts_out->format == COPY_FORMAT_CSV) + opts_out->null_print = ""; + else if (opts_out->format == COPY_FORMAT_TEXT) + opts_out->null_print = "\\N"; + + if (opts_out->null_print) + opts_out->null_print_len = strlen(opts_out->null_print); + + /* --- QUOTE option --- */ + if (opts_out->quote) + { + if (opts_out->format != COPY_FORMAT_CSV) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ + errmsg("COPY %s requires CSV mode", "QUOTE"))); + + if (strlen(opts_out->quote) != 1) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY quote must be a single one-byte character"))); + } + else if (opts_out->format == COPY_FORMAT_CSV) { - if (!opts_out->quote) - opts_out->quote = "\""; - if (!opts_out->escape) - opts_out->escape = opts_out->quote; + /* Set default quote */ + opts_out->quote = "\""; } - /* Only single-byte delimiter strings are supported. */ - if (strlen(opts_out->delim) != 1) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("COPY delimiter must be a single one-byte character"))); - - /* Disallow end-of-line characters */ - if (strchr(opts_out->delim, '\r') != NULL || - strchr(opts_out->delim, '\n') != NULL) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY delimiter cannot be newline or carriage return"))); - - if (strchr(opts_out->null_print, '\r') != NULL || - strchr(opts_out->null_print, '\n') != NULL) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY null representation cannot use newline or carriage return"))); + /* --- ESCAPE option --- */ + if (opts_out->escape) + { + if (opts_out->format != COPY_FORMAT_CSV) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ + errmsg("COPY %s requires CSV mode", "ESCAPE"))); + + if (strlen(opts_out->escape) != 1) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY escape must be a single one-byte character"))); + } + else if (opts_out->format == COPY_FORMAT_CSV) + { + /* Set default escape to quote character */ + opts_out->escape = opts_out->quote; + } + /* --- DEFAULT option --- */ if (opts_out->default_print) { + if (opts_out->format == COPY_FORMAT_BINARY) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot specify %s in BINARY mode", "DEFAULT"))); + + if (opts_out->format == COPY_FORMAT_RAW) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot specify %s in RAW mode", "DEFAULT"))); + + /* Assert options have been set (defaults applied if not specified) */ + Assert(opts_out->delim); + Assert(opts_out->null_print); + opts_out->default_print_len = strlen(opts_out->default_print); if (strchr(opts_out->default_print, '\r') != NULL || @@ -735,135 +825,7 @@ ProcessCopyOptions(ParseState *pstate, ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY default representation cannot use newline or carriage return"))); - } - /* - * Disallow unsafe delimiter characters in non-CSV mode. We can't allow - * backslash because it would be ambiguous. We can't allow the other - * cases because data characters matching the delimiter must be - * backslashed, and certain backslash combinations are interpreted - * non-literally by COPY IN. Disallowing all lower case ASCII letters is - * more than strictly necessary, but seems best for consistency and - * future-proofing. Likewise we disallow all digits though only octal - * digits are actually dangerous. - */ - if (!opts_out->csv_mode && - strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789", - opts_out->delim[0]) != NULL) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY delimiter cannot be \"%s\"", opts_out->delim))); - - /* Check header */ - if (opts_out->binary && opts_out->header_line) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ - errmsg("cannot specify %s in BINARY mode", "HEADER"))); - - /* Check quote */ - if (!opts_out->csv_mode && opts_out->quote != NULL) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ - errmsg("COPY %s requires CSV mode", "QUOTE"))); - - if (opts_out->csv_mode && strlen(opts_out->quote) != 1) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("COPY quote must be a single one-byte character"))); - - if (opts_out->csv_mode && opts_out->delim[0] == opts_out->quote[0]) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY delimiter and quote must be different"))); - - /* Check escape */ - if (!opts_out->csv_mode && opts_out->escape != NULL) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ - errmsg("COPY %s requires CSV mode", "ESCAPE"))); - - if (opts_out->csv_mode && strlen(opts_out->escape) != 1) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("COPY escape must be a single one-byte character"))); - - /* Check force_quote */ - if (!opts_out->csv_mode && (opts_out->force_quote || opts_out->force_quote_all)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ - errmsg("COPY %s requires CSV mode", "FORCE_QUOTE"))); - if ((opts_out->force_quote || opts_out->force_quote_all) && is_from) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - /*- translator: first %s is the name of a COPY option, e.g. ON_ERROR, - second %s is a COPY with direction, e.g. COPY TO */ - errmsg("COPY %s cannot be used with %s", "FORCE_QUOTE", - "COPY FROM"))); - - /* Check force_notnull */ - if (!opts_out->csv_mode && (opts_out->force_notnull != NIL || - opts_out->force_notnull_all)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ - errmsg("COPY %s requires CSV mode", "FORCE_NOT_NULL"))); - if ((opts_out->force_notnull != NIL || opts_out->force_notnull_all) && - !is_from) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - /*- translator: first %s is the name of a COPY option, e.g. ON_ERROR, - second %s is a COPY with direction, e.g. COPY TO */ - errmsg("COPY %s cannot be used with %s", "FORCE_NOT_NULL", - "COPY TO"))); - - /* Check force_null */ - if (!opts_out->csv_mode && (opts_out->force_null != NIL || - opts_out->force_null_all)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ - errmsg("COPY %s requires CSV mode", "FORCE_NULL"))); - - if ((opts_out->force_null != NIL || opts_out->force_null_all) && !is_from) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - /*- translator: first %s is the name of a COPY option, e.g. ON_ERROR, - second %s is a COPY with direction, e.g. COPY TO */ - errmsg("COPY %s cannot be used with %s", "FORCE_NULL", - "COPY TO"))); - - /* Don't allow the delimiter to appear in the null string. */ - if (strchr(opts_out->null_print, opts_out->delim[0]) != NULL) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - /*- translator: %s is the name of a COPY option, e.g. NULL */ - errmsg("COPY delimiter character must not appear in the %s specification", - "NULL"))); - - /* Don't allow the CSV quote char to appear in the null string. */ - if (opts_out->csv_mode && - strchr(opts_out->null_print, opts_out->quote[0]) != NULL) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - /*- translator: %s is the name of a COPY option, e.g. NULL */ - errmsg("CSV quote character must not appear in the %s specification", - "NULL"))); - - /* Check freeze */ - if (opts_out->freeze && !is_from) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - /*- translator: first %s is the name of a COPY option, e.g. ON_ERROR, - second %s is a COPY with direction, e.g. COPY TO */ - errmsg("COPY %s cannot be used with %s", "FREEZE", - "COPY TO"))); - - if (opts_out->default_print) - { if (!is_from) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -881,7 +843,7 @@ ProcessCopyOptions(ParseState *pstate, "DEFAULT"))); /* Don't allow the CSV quote char to appear in the default string. */ - if (opts_out->csv_mode && + if (opts_out->format == COPY_FORMAT_CSV && strchr(opts_out->default_print, opts_out->quote[0]) != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -897,19 +859,154 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("NULL specification and DEFAULT specification cannot be the same"))); } - /* Check on_error */ - if (opts_out->binary && opts_out->on_error != COPY_ON_ERROR_STOP) - ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("only ON_ERROR STOP is allowed in BINARY mode"))); - - if (opts_out->reject_limit && !opts_out->on_error) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - /*- translator: first and second %s are the names of COPY option, e.g. - * ON_ERROR, third is the value of the COPY option, e.g. IGNORE */ - errmsg("COPY %s requires %s to be set to %s", - "REJECT_LIMIT", "ON_ERROR", "IGNORE"))); + else + { + /* No default for default_print; remains NULL */ + } + + /* --- HEADER option --- */ + if (header_specified) + { + if (opts_out->format == COPY_FORMAT_BINARY) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ + errmsg("cannot specify %s in BINARY mode", "HEADER"))); + } + else + { + /* Default is no header; no action needed */ + } + + /* --- FORCE_QUOTE option --- */ + if (opts_out->force_quote || opts_out->force_quote_all) + { + if (opts_out->format != COPY_FORMAT_CSV) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ + errmsg("COPY %s requires CSV mode", "FORCE_QUOTE"))); + + if (is_from) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + /*- translator: first %s is the name of a COPY option, e.g. ON_ERROR, + second %s is a COPY with direction, e.g. COPY TO */ + errmsg("COPY %s cannot be used with %s", "FORCE_QUOTE", + "COPY FROM"))); + } + + /* --- FORCE_NOT_NULL option --- */ + if (opts_out->force_notnull != NIL || opts_out->force_notnull_all) + { + if (opts_out->format != COPY_FORMAT_CSV) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ + errmsg("COPY %s requires CSV mode", "FORCE_NOT_NULL"))); + + if (!is_from) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + /*- translator: first %s is the name of a COPY option, e.g. ON_ERROR, + second %s is a COPY with direction, e.g. COPY TO */ + errmsg("COPY %s cannot be used with %s", "FORCE_NOT_NULL", + "COPY TO"))); + } + + /* --- FORCE_NULL option --- */ + if (opts_out->force_null != NIL || opts_out->force_null_all) + { + if (opts_out->format != COPY_FORMAT_CSV) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ + errmsg("COPY %s requires CSV mode", "FORCE_NULL"))); + + if (!is_from) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + /*- translator: first %s is the name of a COPY option, e.g. ON_ERROR, + second %s is a COPY with direction, e.g. COPY TO */ + errmsg("COPY %s cannot be used with %s", "FORCE_NULL", + "COPY TO"))); + } + + /* --- FREEZE option --- */ + if (opts_out->freeze) + { + if (!is_from) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + /*- translator: first %s is the name of a COPY option, e.g. ON_ERROR, + second %s is a COPY with direction, e.g. COPY TO */ + errmsg("COPY %s cannot be used with %s", "FREEZE", + "COPY TO"))); + } + + /* --- ON_ERROR option --- */ + if (opts_out->on_error != COPY_ON_ERROR_STOP) + { + if (opts_out->format == COPY_FORMAT_BINARY) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("only ON_ERROR STOP is allowed in BINARY mode"))); + } + + /* --- REJECT_LIMIT option --- */ + if (opts_out->reject_limit) + { + if (opts_out->on_error != COPY_ON_ERROR_IGNORE) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + /*- translator: first and second %s are the names of COPY option, e.g. + * ON_ERROR, third is the value of the COPY option, e.g. IGNORE */ + errmsg("COPY %s requires %s to be set to %s", + "REJECT_LIMIT", "ON_ERROR", "IGNORE"))); + } + + /* + * Additional checks for interdependent options + */ + + /* Checks specific to the CSV and TEXT formats */ + if (opts_out->format == COPY_FORMAT_TEXT || + opts_out->format == COPY_FORMAT_CSV) + { + /* Assert options have been set (defaults applied if not specified) */ + Assert(opts_out->delim); + Assert(opts_out->null_print); + + /* Don't allow the delimiter to appear in the null string. */ + if (strchr(opts_out->null_print, opts_out->delim[0]) != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + /*- translator: %s is the name of a COPY option, e.g. NULL */ + errmsg("COPY delimiter character must not appear in the %s specification", + "NULL"))); + } + + /* Checks specific to the CSV format */ + if (opts_out->format == COPY_FORMAT_CSV) + { + /* Assert options have been set (defaults applied if not specified) */ + Assert(opts_out->delim); + Assert(opts_out->quote); + Assert(opts_out->null_print); + + if (opts_out->delim[0] == opts_out->quote[0]) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY delimiter and quote must be different"))); + + /* Don't allow the CSV quote char to appear in the null string. */ + if (strchr(opts_out->null_print, opts_out->quote[0]) != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + /*- translator: %s is the name of a COPY option, e.g. NULL */ + errmsg("CSV quote character must not appear in the %s specification", + "NULL"))); + } } /* diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index 07cbd5d22b..99dcb00f8a 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -122,7 +122,7 @@ CopyFromErrorCallback(void *arg) cstate->cur_relname); return; } - if (cstate->opts.binary) + if (cstate->opts.format == COPY_FORMAT_BINARY) { /* can't usefully display the data */ if (cstate->cur_attname) @@ -1438,6 +1438,13 @@ BeginCopyFrom(ParseState *pstate, /* Generate or convert list of attributes to process */ cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist); + /* Enforce single column requirement for RAW format */ + if (cstate->opts.format == COPY_FORMAT_RAW && + list_length(cstate->attnumlist) != 1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY with format 'raw' must specify exactly one column"))); + num_phys_attrs = tupDesc->natts; /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */ @@ -1583,7 +1590,7 @@ BeginCopyFrom(ParseState *pstate, cstate->raw_buf_index = cstate->raw_buf_len = 0; cstate->raw_reached_eof = false; - if (!cstate->opts.binary) + if (cstate->opts.format != COPY_FORMAT_BINARY) { /* * If encoding conversion is needed, we need another buffer to hold @@ -1634,7 +1641,7 @@ BeginCopyFrom(ParseState *pstate, continue; /* Fetch the input function and typioparam info */ - if (cstate->opts.binary) + if (cstate->opts.format == COPY_FORMAT_BINARY) getTypeBinaryInputInfo(att->atttypid, &in_func_oid, &typioparams[attnum - 1]); else @@ -1775,14 +1782,14 @@ BeginCopyFrom(ParseState *pstate, pgstat_progress_update_multi_param(3, progress_cols, progress_vals); - if (cstate->opts.binary) + if (cstate->opts.format == COPY_FORMAT_BINARY) { /* Read and verify binary header */ ReceiveCopyBinaryHeader(cstate); } /* create workspace for CopyReadAttributes results */ - if (!cstate->opts.binary) + if (cstate->opts.format != COPY_FORMAT_BINARY) { AttrNumber attr_count = list_length(cstate->attnumlist); diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 654fecb1b1..2528c6f111 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -7,7 +7,7 @@ * formats. The main entry point is NextCopyFrom(), which parses the * next input line and returns it as Datums. * - * In text/CSV mode, the parsing happens in multiple stages: + * In text/CSV/raw mode, the parsing happens in multiple stages: * * [data source] --> raw_buf --> input_buf --> line_buf --> attribute_buf * 1. 2. 3. 4. @@ -25,7 +25,7 @@ * is copied into 'line_buf', with quotes and escape characters still * intact. * - * 4. CopyReadAttributesText/CSV() function takes the input line from + * 4. CopyReadAttributesText/CSV/Raw() function takes the input line from * 'line_buf', and splits it into fields, unescaping the data as required. * The fields are stored in 'attribute_buf', and 'raw_fields' array holds * pointers to each field. @@ -143,8 +143,10 @@ static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0"; /* non-export function prototypes */ static bool CopyReadLine(CopyFromState cstate); static bool CopyReadLineText(CopyFromState cstate); +static bool CopyReadLineRawText(CopyFromState cstate); static int CopyReadAttributesText(CopyFromState cstate); static int CopyReadAttributesCSV(CopyFromState cstate); +static int CopyReadAttributesRaw(CopyFromState cstate); static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull); @@ -163,7 +165,7 @@ ReceiveCopyBegin(CopyFromState cstate) { StringInfoData buf; int natts = list_length(cstate->attnumlist); - int16 format = (cstate->opts.binary ? 1 : 0); + int16 format = (cstate->opts.format == COPY_FORMAT_BINARY ? 1 : 0); int i; pq_beginmessage(&buf, PqMsg_CopyInResponse); @@ -732,7 +734,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) } /* - * Read raw fields in the next line for COPY FROM in text or csv mode. + * Read raw fields in the next line for COPY FROM in text, csv, or raw mode. * Return false if no more lines. * * An internal temporary buffer is returned via 'fields'. It is valid until @@ -748,8 +750,8 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) int fldct; bool done; - /* only available for text or csv input */ - Assert(!cstate->opts.binary); + /* only available for text, csv, or raw input */ + Assert(cstate->opts.format != COPY_FORMAT_BINARY); /* on input check that the header line is correct if needed */ if (cstate->cur_lineno == 0 && cstate->opts.header_line) @@ -766,10 +768,17 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) { int fldnum; - if (cstate->opts.csv_mode) + if (cstate->opts.format == COPY_FORMAT_CSV) fldct = CopyReadAttributesCSV(cstate); - else + else if (cstate->opts.format == COPY_FORMAT_TEXT) fldct = CopyReadAttributesText(cstate); + else if (cstate->opts.format == COPY_FORMAT_RAW) + fldct = CopyReadAttributesRaw(cstate); + else + { + elog(ERROR, "unexpected COPY format: %d", cstate->opts.format); + pg_unreachable(); + } if (fldct != list_length(cstate->attnumlist)) ereport(ERROR, @@ -821,10 +830,17 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) return false; /* Parse the line into de-escaped field values */ - if (cstate->opts.csv_mode) + if (cstate->opts.format == COPY_FORMAT_CSV) fldct = CopyReadAttributesCSV(cstate); - else + else if (cstate->opts.format == COPY_FORMAT_TEXT) fldct = CopyReadAttributesText(cstate); + else if (cstate->opts.format == COPY_FORMAT_RAW) + fldct = CopyReadAttributesRaw(cstate); + else + { + elog(ERROR, "unexpected COPY format: %d", cstate->opts.format); + pg_unreachable(); + } *fields = cstate->raw_fields; *nfields = fldct; @@ -865,7 +881,7 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, MemSet(nulls, true, num_phys_attrs * sizeof(bool)); MemSet(cstate->defaults, false, num_phys_attrs * sizeof(bool)); - if (!cstate->opts.binary) + if (cstate->opts.format != COPY_FORMAT_BINARY) { char **field_strings; ListCell *cur; @@ -906,7 +922,7 @@ NextCopyFrom(CopyFromState cstate, ExprContext *econtext, continue; } - if (cstate->opts.csv_mode) + if (cstate->opts.format == COPY_FORMAT_CSV) { if (string == NULL && cstate->opts.force_notnull_flags[m]) @@ -1096,7 +1112,10 @@ CopyReadLine(CopyFromState cstate) cstate->line_buf_valid = false; /* Parse data and transfer into line_buf */ - result = CopyReadLineText(cstate); + if (cstate->opts.format == COPY_FORMAT_RAW) + result = CopyReadLineRawText(cstate); + else + result = CopyReadLineText(cstate); if (result) { @@ -1179,7 +1198,7 @@ CopyReadLineText(CopyFromState cstate) char quotec = '\0'; char escapec = '\0'; - if (cstate->opts.csv_mode) + if (cstate->opts.format == COPY_FORMAT_CSV) { quotec = cstate->opts.quote[0]; escapec = cstate->opts.escape[0]; @@ -1256,7 +1275,7 @@ CopyReadLineText(CopyFromState cstate) prev_raw_ptr = input_buf_ptr; c = copy_input_buf[input_buf_ptr++]; - if (cstate->opts.csv_mode) + if (cstate->opts.format == COPY_FORMAT_CSV) { /* * If character is '\r', we may need to look ahead below. Force @@ -1295,7 +1314,7 @@ CopyReadLineText(CopyFromState cstate) } /* Process \r */ - if (c == '\r' && (!cstate->opts.csv_mode || !in_quote)) + if (c == '\r' && (cstate->opts.format != COPY_FORMAT_CSV || !in_quote)) { /* Check for \r\n on first line, _and_ handle \r\n. */ if (cstate->eol_type == EOL_UNKNOWN || @@ -1323,10 +1342,10 @@ CopyReadLineText(CopyFromState cstate) if (cstate->eol_type == EOL_CRNL) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - !cstate->opts.csv_mode ? + cstate->opts.format != COPY_FORMAT_CSV ? errmsg("literal carriage return found in data") : errmsg("unquoted carriage return found in data"), - !cstate->opts.csv_mode ? + cstate->opts.format != COPY_FORMAT_CSV ? errhint("Use \"\\r\" to represent carriage return.") : errhint("Use quoted CSV field to represent carriage return."))); @@ -1340,10 +1359,10 @@ CopyReadLineText(CopyFromState cstate) else if (cstate->eol_type == EOL_NL) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - !cstate->opts.csv_mode ? + cstate->opts.format != COPY_FORMAT_CSV ? errmsg("literal carriage return found in data") : errmsg("unquoted carriage return found in data"), - !cstate->opts.csv_mode ? + cstate->opts.format != COPY_FORMAT_CSV ? errhint("Use \"\\r\" to represent carriage return.") : errhint("Use quoted CSV field to represent carriage return."))); /* If reach here, we have found the line terminator */ @@ -1351,15 +1370,15 @@ CopyReadLineText(CopyFromState cstate) } /* Process \n */ - if (c == '\n' && (!cstate->opts.csv_mode || !in_quote)) + if (c == '\n' && (cstate->opts.format != COPY_FORMAT_CSV || !in_quote)) { if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), - !cstate->opts.csv_mode ? + cstate->opts.format != COPY_FORMAT_CSV ? errmsg("literal newline found in data") : errmsg("unquoted newline found in data"), - !cstate->opts.csv_mode ? + cstate->opts.format != COPY_FORMAT_CSV ? errhint("Use \"\\n\" to represent newline.") : errhint("Use quoted CSV field to represent newline."))); cstate->eol_type = EOL_NL; /* in case not set yet */ @@ -1371,7 +1390,7 @@ CopyReadLineText(CopyFromState cstate) * Process backslash, except in CSV mode where backslash is a normal * character. */ - if (c == '\\' && !cstate->opts.csv_mode) + if (c == '\\' && cstate->opts.format != COPY_FORMAT_CSV) { char c2; @@ -1462,6 +1481,138 @@ CopyReadLineText(CopyFromState cstate) return result; } +/* + * CopyReadLineRawText - inner loop of CopyReadLine for raw text mode + */ +static bool +CopyReadLineRawText(CopyFromState cstate) +{ + char *copy_input_buf; + int input_buf_ptr; + int copy_buf_len; + bool need_data = false; + bool hit_eof = false; + bool result = false; + + /* + * The objective of this loop is to transfer the entire next input line + * into line_buf. We only care for detecting newlines (\r and/or \n). + * All other characters are treated as regular data. + * + * For speed, we try to move data from input_buf to line_buf in chunks + * rather than one character at a time. input_buf_ptr points to the next + * character to examine; any characters from input_buf_index to + * input_buf_ptr have been determined to be part of the line, but not yet + * transferred to line_buf. + * + * For a little extra speed within the loop, we copy input_buf and + * input_buf_len into local variables. + */ + copy_input_buf = cstate->input_buf; + input_buf_ptr = cstate->input_buf_index; + copy_buf_len = cstate->input_buf_len; + + for (;;) + { + int prev_raw_ptr; + char c; + + /* + * Load more data if needed. + */ + if (input_buf_ptr >= copy_buf_len || need_data) + { + REFILL_LINEBUF; + + CopyLoadInputBuf(cstate); + /* update our local variables */ + hit_eof = cstate->input_reached_eof; + input_buf_ptr = cstate->input_buf_index; + copy_buf_len = cstate->input_buf_len; + + /* + * If we are completely out of data, break out of the loop, + * reporting EOF. + */ + if (INPUT_BUF_BYTES(cstate) <= 0) + { + result = true; + break; + } + need_data = false; + } + + /* OK to fetch a character */ + prev_raw_ptr = input_buf_ptr; + c = copy_input_buf[input_buf_ptr++]; + + /* Process \r */ + if (c == '\r') + { + /* Check for \r\n on first line, _and_ handle \r\n. */ + if (cstate->eol_type == EOL_UNKNOWN || + cstate->eol_type == EOL_CRNL) + { + /* + * If need more data, go back to loop top to load it. + * + * Note that if we are at EOF, c will wind up as '\0' because + * of the guaranteed pad of input_buf. + */ + IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0); + + /* get next char */ + c = copy_input_buf[input_buf_ptr]; + + if (c == '\n') + { + input_buf_ptr++; /* eat newline */ + cstate->eol_type = EOL_CRNL; /* in case not set yet */ + } + else + { + if (cstate->eol_type == EOL_CRNL) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("inconsistent newline style"))); + /* + * if we got here, it is the first line and we didn't find + * \n, so don't consume the peeked character + */ + cstate->eol_type = EOL_CR; + } + } + else if (cstate->eol_type == EOL_NL) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("inconsistent newline style"))); + /* If reach here, we have found the line terminator */ + break; + } + + /* Process \n */ + if (c == '\n') + { + if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("inconsistent newline style"))); + cstate->eol_type = EOL_NL; /* in case not set yet */ + /* If reach here, we have found the line terminator */ + break; + } + + /* All other characters are treated as regular data */ + } /* end of outer loop */ + + /* + * Transfer any still-uncopied data to line_buf. + */ + REFILL_LINEBUF; + + return result; +} + /* * Return decimal value for a hexadecimal digit */ @@ -1938,6 +2089,45 @@ endfield: return fieldno; } +/* + * Parse the current line as a single attribute for the "raw" COPY format. + * No parsing, quoting, or escaping is performed. + * Empty lines are treated as empty strings, not NULL. + */ +static int +CopyReadAttributesRaw(CopyFromState cstate) +{ + /* Enforce single column requirement */ + if (cstate->max_fields != 1) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY with format 'raw' requires exactly one column"))); + } + + resetStringInfo(&cstate->attribute_buf); + + /* + * The attribute will certainly not be longer than the input + * data line, so we can just force attribute_buf to be large enough and + * then transfer data without any checks for enough space. We need to do + * it this way because enlarging attribute_buf mid-stream would invalidate + * pointers already stored into cstate->raw_fields[]. + */ + if (cstate->attribute_buf.maxlen <= cstate->line_buf.len) + enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len); + + /* Copy the entire line into attribute_buf */ + memcpy(cstate->attribute_buf.data, cstate->line_buf.data, + cstate->line_buf.len); + cstate->attribute_buf.data[cstate->line_buf.len] = '\0'; + cstate->attribute_buf.len = cstate->line_buf.len; + + /* Assign the single field to raw_fields[0] */ + cstate->raw_fields[0] = cstate->attribute_buf.data; + + return 1; +} /* * Read a binary attribute diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 463083e645..99fd68a483 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -113,6 +113,7 @@ static void CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot); static void CopyAttributeOutText(CopyToState cstate, const char *string); static void CopyAttributeOutCSV(CopyToState cstate, const char *string, bool use_quote); +static void CopyAttributeOutRaw(CopyToState cstate, const char *string); /* Low-level communications functions */ static void SendCopyBegin(CopyToState cstate); @@ -134,7 +135,7 @@ SendCopyBegin(CopyToState cstate) { StringInfoData buf; int natts = list_length(cstate->attnumlist); - int16 format = (cstate->opts.binary ? 1 : 0); + int16 format = (cstate->opts.format == COPY_FORMAT_BINARY ? 1 : 0); int i; pq_beginmessage(&buf, PqMsg_CopyOutResponse); @@ -191,7 +192,7 @@ CopySendEndOfRow(CopyToState cstate) switch (cstate->copy_dest) { case COPY_FILE: - if (!cstate->opts.binary) + if (cstate->opts.format != COPY_FORMAT_BINARY) { /* Default line termination depends on platform */ #ifndef WIN32 @@ -236,7 +237,7 @@ CopySendEndOfRow(CopyToState cstate) break; case COPY_FRONTEND: /* The FE/BE protocol uses \n as newline for all platforms */ - if (!cstate->opts.binary) + if (cstate->opts.format != COPY_FORMAT_BINARY) CopySendChar(cstate, '\n'); /* Dump the accumulated row as one CopyData message */ @@ -570,6 +571,13 @@ BeginCopyTo(ParseState *pstate, /* Generate or convert list of attributes to process */ cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist); + /* Enforce single column requirement for RAW format */ + if (cstate->opts.format == COPY_FORMAT_RAW && + list_length(cstate->attnumlist) != 1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY with format 'raw' must specify exactly one column"))); + num_phys_attrs = tupDesc->natts; /* Convert FORCE_QUOTE name list to per-column flags, check validity */ @@ -771,7 +779,7 @@ DoCopyTo(CopyToState cstate) bool isvarlena; Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); - if (cstate->opts.binary) + if (cstate->opts.format == COPY_FORMAT_BINARY) getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); @@ -792,7 +800,7 @@ DoCopyTo(CopyToState cstate) "COPY TO", ALLOCSET_DEFAULT_SIZES); - if (cstate->opts.binary) + if (cstate->opts.format == COPY_FORMAT_BINARY) { /* Generate header for a binary copy */ int32 tmp; @@ -833,10 +841,12 @@ DoCopyTo(CopyToState cstate) colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); - if (cstate->opts.csv_mode) + if (cstate->opts.format == COPY_FORMAT_CSV) CopyAttributeOutCSV(cstate, colname, false); - else + else if (cstate->opts.format == COPY_FORMAT_TEXT) CopyAttributeOutText(cstate, colname); + else if (cstate->opts.format == COPY_FORMAT_RAW) + CopyAttributeOutRaw(cstate, colname); } CopySendEndOfRow(cstate); @@ -880,7 +890,7 @@ DoCopyTo(CopyToState cstate) processed = ((DR_copy *) cstate->queryDesc->dest)->processed; } - if (cstate->opts.binary) + if (cstate->opts.format == COPY_FORMAT_BINARY) { /* Generate trailer for a binary copy */ CopySendInt16(cstate, -1); @@ -908,7 +918,7 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) MemoryContextReset(cstate->rowcontext); oldcontext = MemoryContextSwitchTo(cstate->rowcontext); - if (cstate->opts.binary) + if (cstate->opts.format == COPY_FORMAT_BINARY) { /* Binary per-tuple header */ CopySendInt16(cstate, list_length(cstate->attnumlist)); @@ -917,7 +927,8 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) /* Make sure the tuple is fully deconstructed */ slot_getallattrs(slot); - if (!cstate->opts.binary) + if (cstate->opts.format == COPY_FORMAT_TEXT || + cstate->opts.format == COPY_FORMAT_CSV) { bool need_delim = false; @@ -937,7 +948,7 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) { string = OutputFunctionCall(&out_functions[attnum - 1], value); - if (cstate->opts.csv_mode) + if (cstate->opts.format == COPY_FORMAT_CSV) CopyAttributeOutCSV(cstate, string, cstate->opts.force_quote_flags[attnum - 1]); else @@ -945,7 +956,7 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) } } } - else + else if (cstate->opts.format == COPY_FORMAT_BINARY) { foreach_int(attnum, cstate->attnumlist) { @@ -965,6 +976,37 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) } } } + else if (cstate->opts.format == COPY_FORMAT_RAW) + { + int attnum; + Datum value; + bool isnull; + + /* Ensure only one column is being copied */ + if (list_length(cstate->attnumlist) != 1) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY with format 'raw' must specify exactly one column"))); + + attnum = linitial_int(cstate->attnumlist); + value = slot->tts_values[attnum - 1]; + isnull = slot->tts_isnull[attnum - 1]; + + if (!isnull) + { + char *string = OutputFunctionCall(&out_functions[attnum - 1], + value); + CopyAttributeOutRaw(cstate, string); + } + /* For RAW format, we don't send anything for NULL values */ + } + else + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("Unsupported COPY format"))); + } + CopySendEndOfRow(cstate); @@ -1219,6 +1261,28 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string, } } +/* + * Send text representation of one attribute for RAW format. + */ +static void +CopyAttributeOutRaw(CopyToState cstate, const char *string) +{ + const char *ptr; + + /* Ensure the format is RAW */ + Assert(cstate->opts.format == COPY_FORMAT_RAW); + + /* Ensure exactly one column is being processed */ + Assert(list_length(cstate->attnumlist) == 1); + + if (cstate->need_transcoding) + ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding); + else + ptr = string; + + CopySendString(cstate, ptr); +} + /* * copy_dest_startup --- executor startup */ diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 4aa8646af7..0d0a3ad7ff 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -768,7 +768,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); QUOTE QUOTES - RANGE READ REAL REASSIGN RECURSIVE REF_P REFERENCES REFERENCING + RANGE RAW READ REAL REASSIGN RECURSIVE REF_P REFERENCES REFERENCING REFRESH REINDEX RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP ROUTINE ROUTINES ROW ROWS RULE @@ -3513,6 +3513,10 @@ copy_opt_item: { $$ = makeDefElem("encoding", (Node *) makeString($2), @1); } + | RAW + { + $$ = makeDefElem("format", (Node *) makeString("raw"), @1); + } ; /* The following exist for backward compatibility with very old versions */ @@ -17771,6 +17775,7 @@ unreserved_keyword: | QUOTE | QUOTES | RANGE + | RAW | READ | REASSIGN | RECURSIVE @@ -18398,6 +18403,7 @@ bare_label_keyword: | QUOTE | QUOTES | RANGE + | RAW | READ | REAL | REASSIGN diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index 4002a7f538..37c44fa1bc 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -51,6 +51,17 @@ typedef enum CopyLogVerbosityChoice COPY_LOG_VERBOSITY_VERBOSE, /* logs additional messages */ } CopyLogVerbosityChoice; +/* + * Represents the format of the COPY operation. + */ +typedef enum CopyFormat +{ + COPY_FORMAT_TEXT = 0, + COPY_FORMAT_BINARY, + COPY_FORMAT_CSV, + COPY_FORMAT_RAW, +} CopyFormat; + /* * A struct to hold COPY options, in a parsed form. All of these are related * to formatting, except for 'freeze', which doesn't really belong here, but @@ -61,9 +72,8 @@ typedef struct CopyFormatOptions /* parameters from the COPY command */ int file_encoding; /* file or remote side's character encoding, * -1 if not specified */ - bool binary; /* binary format? */ + CopyFormat format; /* format of the COPY operation */ bool freeze; /* freeze rows on loading? */ - bool csv_mode; /* Comma Separated Value format? */ CopyHeaderChoice header_line; /* header line? */ char *null_print; /* NULL marker string (server encoding!) */ int null_print_len; /* length of same */ diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 899d64ad55..02cd28c750 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -360,6 +360,7 @@ PG_KEYWORD("publication", PUBLICATION, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("quote", QUOTE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("quotes", QUOTES, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("range", RANGE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("raw", RAW, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("read", READ, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("real", REAL, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("reassign", REASSIGN, UNRESERVED_KEYWORD, BARE_LABEL) diff --git a/src/test/regress/data/newlines_cr.data b/src/test/regress/data/newlines_cr.data new file mode 100644 index 0000000000..5397a14fca --- /dev/null +++ b/src/test/regress/data/newlines_cr.data @@ -0,0 +1 @@ +line1 line2 \ No newline at end of file diff --git a/src/test/regress/data/newlines_cr_lr.data b/src/test/regress/data/newlines_cr_lr.data new file mode 100644 index 0000000000..8561d5d6dc --- /dev/null +++ b/src/test/regress/data/newlines_cr_lr.data @@ -0,0 +1,2 @@ +line1 +line2 diff --git a/src/test/regress/data/newlines_cr_lr_nolast.data b/src/test/regress/data/newlines_cr_lr_nolast.data new file mode 100644 index 0000000000..3a1bd7a527 --- /dev/null +++ b/src/test/regress/data/newlines_cr_lr_nolast.data @@ -0,0 +1,2 @@ +line1 +line2 \ No newline at end of file diff --git a/src/test/regress/data/newlines_cr_nolast.data b/src/test/regress/data/newlines_cr_nolast.data new file mode 100644 index 0000000000..d9dce6c5ea --- /dev/null +++ b/src/test/regress/data/newlines_cr_nolast.data @@ -0,0 +1 @@ +line1 line2 \ No newline at end of file diff --git a/src/test/regress/data/newlines_lr.data b/src/test/regress/data/newlines_lr.data new file mode 100644 index 0000000000..c0d0fb45c3 --- /dev/null +++ b/src/test/regress/data/newlines_lr.data @@ -0,0 +1,2 @@ +line1 +line2 diff --git a/src/test/regress/data/newlines_lr_nolast.data b/src/test/regress/data/newlines_lr_nolast.data new file mode 100644 index 0000000000..f8be7bb828 --- /dev/null +++ b/src/test/regress/data/newlines_lr_nolast.data @@ -0,0 +1,2 @@ +line1 +line2 \ No newline at end of file diff --git a/src/test/regress/data/newlines_mixed_1.data b/src/test/regress/data/newlines_mixed_1.data new file mode 100644 index 0000000000..d20e511549 --- /dev/null +++ b/src/test/regress/data/newlines_mixed_1.data @@ -0,0 +1 @@ +line1 line2 diff --git a/src/test/regress/data/newlines_mixed_2.data b/src/test/regress/data/newlines_mixed_2.data new file mode 100644 index 0000000000..fe03b64cc3 --- /dev/null +++ b/src/test/regress/data/newlines_mixed_2.data @@ -0,0 +1,2 @@ +line1 +line2 diff --git a/src/test/regress/data/newlines_mixed_3.data b/src/test/regress/data/newlines_mixed_3.data new file mode 100644 index 0000000000..d2772944d6 --- /dev/null +++ b/src/test/regress/data/newlines_mixed_3.data @@ -0,0 +1,2 @@ +line1 +line2 \ No newline at end of file diff --git a/src/test/regress/data/newlines_mixed_4.data b/src/test/regress/data/newlines_mixed_4.data new file mode 100644 index 0000000000..7afb2406f0 --- /dev/null +++ b/src/test/regress/data/newlines_mixed_4.data @@ -0,0 +1,2 @@ +line1 +line2 line3 \ No newline at end of file diff --git a/src/test/regress/data/newlines_mixed_5.data b/src/test/regress/data/newlines_mixed_5.data new file mode 100644 index 0000000000..658b3593ea --- /dev/null +++ b/src/test/regress/data/newlines_mixed_5.data @@ -0,0 +1,2 @@ +line1 +line2 diff --git a/src/test/regress/expected/copy.out b/src/test/regress/expected/copy.out index f554d42c84..d7ec9dd736 100644 --- a/src/test/regress/expected/copy.out +++ b/src/test/regress/expected/copy.out @@ -325,3 +325,95 @@ SELECT tableoid::regclass, id % 2 = 0 is_even, count(*) from parted_si GROUP BY (2 rows) DROP TABLE parted_si; +-- Test COPY FORMAT raw +\set filename :abs_builddir '/results/copy_raw_test.data' +CREATE TABLE copy_raw_test (id SERIAL PRIMARY KEY, col text); +INSERT INTO copy_raw_test (col) VALUES +(E'",\\'), (E'\\.'), (NULL), (''), (' '), ('test'); +COPY copy_raw_test (col) TO :'filename' (FORMAT raw); +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + col | ?column? +------+---------- + ",\ | f + \. | f + | f + | f + | f + test | f +(6 rows) + +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' RAW; +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + col | ?column? +------+---------- + ",\ | f + \. | f + | f + | f + | f + test | f +(6 rows) + +\set filename :abs_srcdir '/data/newlines_lr.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + col | ?column? +-------+---------- + line1 | f + line2 | f +(2 rows) + +\set filename :abs_srcdir '/data/newlines_lr_nolast.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + col | ?column? +-------+---------- + line1 | f + line2 | f +(2 rows) + +\set filename :abs_srcdir '/data/newlines_cr_lr.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + col | ?column? +-------+---------- + line1 | f + line2 | f +(2 rows) + +\set filename :abs_srcdir '/data/newlines_cr_lr_nolast.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + col | ?column? +-------+---------- + line1 | f + line2 | f +(2 rows) + +\set filename :abs_srcdir '/data/newlines_cr.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + col | ?column? +-------+---------- + line1 | f + line2 | f +(2 rows) + +\set filename :abs_srcdir '/data/newlines_cr_nolast.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + col | ?column? +-------+---------- + line1 | f + line2 | f +(2 rows) + diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index 626a437d40..34bf06390b 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -88,8 +88,12 @@ LINE 1: COPY x from stdin (log_verbosity default, log_verbosity verb... -- incorrect options COPY x to stdout (format BINARY, delimiter ','); ERROR: cannot specify DELIMITER in BINARY mode +COPY x to stdout (format RAW, delimiter ','); +ERROR: cannot specify DELIMITER in RAW mode COPY x to stdout (format BINARY, null 'x'); ERROR: cannot specify NULL in BINARY mode +COPY x to stdout (format RAW, null 'x'); +ERROR: cannot specify NULL in RAW mode COPY x from stdin (format BINARY, on_error ignore); ERROR: only ON_ERROR STOP is allowed in BINARY mode COPY x from stdin (on_error unsupported); @@ -100,6 +104,10 @@ COPY x to stdout (format TEXT, force_quote(a)); ERROR: COPY FORCE_QUOTE requires CSV mode COPY x to stdout (format TEXT, force_quote *); ERROR: COPY FORCE_QUOTE requires CSV mode +COPY x to stdout (format RAW, force_quote(a)); +ERROR: COPY FORCE_QUOTE requires CSV mode +COPY x to stdout (format RAW, force_quote *); +ERROR: COPY FORCE_QUOTE requires CSV mode COPY x from stdin (format CSV, force_quote(a)); ERROR: COPY FORCE_QUOTE cannot be used with COPY FROM COPY x from stdin (format CSV, force_quote *); @@ -108,6 +116,10 @@ COPY x from stdin (format TEXT, force_not_null(a)); ERROR: COPY FORCE_NOT_NULL requires CSV mode COPY x from stdin (format TEXT, force_not_null *); ERROR: COPY FORCE_NOT_NULL requires CSV mode +COPY x from stdin (format RAW, force_not_null(a)); +ERROR: COPY FORCE_NOT_NULL requires CSV mode +COPY x from stdin (format RAW, force_not_null *); +ERROR: COPY FORCE_NOT_NULL requires CSV mode COPY x to stdout (format CSV, force_not_null(a)); ERROR: COPY FORCE_NOT_NULL cannot be used with COPY TO COPY x to stdout (format CSV, force_not_null *); @@ -116,6 +128,10 @@ COPY x from stdin (format TEXT, force_null(a)); ERROR: COPY FORCE_NULL requires CSV mode COPY x from stdin (format TEXT, force_null *); ERROR: COPY FORCE_NULL requires CSV mode +COPY x from stdin (format RAW, force_null(a)); +ERROR: COPY FORCE_NULL requires CSV mode +COPY x from stdin (format RAW, force_null *); +ERROR: COPY FORCE_NULL requires CSV mode COPY x to stdout (format CSV, force_null(a)); ERROR: COPY FORCE_NULL cannot be used with COPY TO COPY x to stdout (format CSV, force_null *); @@ -858,9 +874,11 @@ select id, text_value, ts_value from copy_default; (2 rows) truncate copy_default; --- DEFAULT cannot be used in binary mode +-- DEFAULT cannot be used in binary or raw mode copy copy_default from stdin with (format binary, default '\D'); ERROR: cannot specify DEFAULT in BINARY mode +copy copy_default from stdin with (format raw, default '\D'); +ERROR: cannot specify DEFAULT in RAW mode -- DEFAULT cannot be new line nor carriage return copy copy_default from stdin with (default E'\n'); ERROR: COPY default representation cannot use newline or carriage return @@ -929,3 +947,40 @@ truncate copy_default; -- DEFAULT cannot be used in COPY TO copy (select 1 as test) TO stdout with (default '\D'); ERROR: COPY DEFAULT cannot be used with COPY TO +-- +-- Test COPY FORMAT errors +-- +\getenv abs_srcdir PG_ABS_SRCDIR +\getenv abs_builddir PG_ABS_BUILDDIR +\set filename :abs_builddir '/results/copy_raw_test_errors.data' +-- Test single column requirement +CREATE TABLE copy_raw_test_errors (col1 text, col2 text); +COPY copy_raw_test_errors TO :'filename' (FORMAT raw); +ERROR: COPY with format 'raw' must specify exactly one column +COPY copy_raw_test_errors (col1, col2) TO :'filename' (FORMAT raw); +ERROR: COPY with format 'raw' must specify exactly one column +COPY copy_raw_test_errors FROM :'filename' (FORMAT raw); +ERROR: COPY with format 'raw' must specify exactly one column +COPY copy_raw_test_errors (col1, col2) FROM :'filename' (FORMAT raw); +ERROR: COPY with format 'raw' must specify exactly one column +-- Test inconsistent newline style +\set filename :abs_srcdir '/data/newlines_mixed_1.data' +COPY copy_raw_test_errors (col1) FROM :'filename' (FORMAT raw); +ERROR: inconsistent newline style +CONTEXT: COPY copy_raw_test_errors, line 2 +\set filename :abs_srcdir '/data/newlines_mixed_2.data' +COPY copy_raw_test_errors (col1) FROM :'filename' (FORMAT raw); +ERROR: inconsistent newline style +CONTEXT: COPY copy_raw_test_errors, line 2 +\set filename :abs_srcdir '/data/newlines_mixed_3.data' +COPY copy_raw_test_errors (col1) FROM :'filename' (FORMAT raw); +ERROR: inconsistent newline style +CONTEXT: COPY copy_raw_test_errors, line 2 +\set filename :abs_srcdir '/data/newlines_mixed_4.data' +COPY copy_raw_test_errors (col1) FROM :'filename' (FORMAT raw); +ERROR: inconsistent newline style +CONTEXT: COPY copy_raw_test_errors, line 2 +\set filename :abs_srcdir '/data/newlines_mixed_5.data' +COPY copy_raw_test_errors (col1) FROM :'filename' (FORMAT raw); +ERROR: inconsistent newline style +CONTEXT: COPY copy_raw_test_errors, line 2 diff --git a/src/test/regress/sql/copy.sql b/src/test/regress/sql/copy.sql index f1699b66b0..c106bd74ec 100644 --- a/src/test/regress/sql/copy.sql +++ b/src/test/regress/sql/copy.sql @@ -348,3 +348,46 @@ COPY parted_si(id, data) FROM :'filename'; SELECT tableoid::regclass, id % 2 = 0 is_even, count(*) from parted_si GROUP BY 1, 2 ORDER BY 1; DROP TABLE parted_si; + +-- Test COPY FORMAT raw +\set filename :abs_builddir '/results/copy_raw_test.data' +CREATE TABLE copy_raw_test (id SERIAL PRIMARY KEY, col text); +INSERT INTO copy_raw_test (col) VALUES +(E'",\\'), (E'\\.'), (NULL), (''), (' '), ('test'); +COPY copy_raw_test (col) TO :'filename' (FORMAT raw); +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' RAW; +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + +\set filename :abs_srcdir '/data/newlines_lr.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + +\set filename :abs_srcdir '/data/newlines_lr_nolast.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + +\set filename :abs_srcdir '/data/newlines_cr_lr.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + +\set filename :abs_srcdir '/data/newlines_cr_lr_nolast.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + +\set filename :abs_srcdir '/data/newlines_cr.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; + +\set filename :abs_srcdir '/data/newlines_cr_nolast.data' +TRUNCATE copy_raw_test; +COPY copy_raw_test (col) FROM :'filename' (FORMAT raw); +SELECT col, col IS NULL FROM copy_raw_test ORDER BY id; diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql index 3458d287f2..56367234bf 100644 --- a/src/test/regress/sql/copy2.sql +++ b/src/test/regress/sql/copy2.sql @@ -71,19 +71,27 @@ COPY x from stdin (log_verbosity default, log_verbosity verbose); -- incorrect options COPY x to stdout (format BINARY, delimiter ','); +COPY x to stdout (format RAW, delimiter ','); COPY x to stdout (format BINARY, null 'x'); +COPY x to stdout (format RAW, null 'x'); COPY x from stdin (format BINARY, on_error ignore); COPY x from stdin (on_error unsupported); COPY x to stdout (format TEXT, force_quote(a)); COPY x to stdout (format TEXT, force_quote *); +COPY x to stdout (format RAW, force_quote(a)); +COPY x to stdout (format RAW, force_quote *); COPY x from stdin (format CSV, force_quote(a)); COPY x from stdin (format CSV, force_quote *); COPY x from stdin (format TEXT, force_not_null(a)); COPY x from stdin (format TEXT, force_not_null *); +COPY x from stdin (format RAW, force_not_null(a)); +COPY x from stdin (format RAW, force_not_null *); COPY x to stdout (format CSV, force_not_null(a)); COPY x to stdout (format CSV, force_not_null *); COPY x from stdin (format TEXT, force_null(a)); COPY x from stdin (format TEXT, force_null *); +COPY x from stdin (format RAW, force_null(a)); +COPY x from stdin (format RAW, force_null *); COPY x to stdout (format CSV, force_null(a)); COPY x to stdout (format CSV, force_null *); COPY x to stdout (format BINARY, on_error unsupported); @@ -636,8 +644,9 @@ select id, text_value, ts_value from copy_default; truncate copy_default; --- DEFAULT cannot be used in binary mode +-- DEFAULT cannot be used in binary or raw mode copy copy_default from stdin with (format binary, default '\D'); +copy copy_default from stdin with (format raw, default '\D'); -- DEFAULT cannot be new line nor carriage return copy copy_default from stdin with (default E'\n'); @@ -707,3 +716,35 @@ truncate copy_default; -- DEFAULT cannot be used in COPY TO copy (select 1 as test) TO stdout with (default '\D'); + +-- +-- Test COPY FORMAT errors +-- + +\getenv abs_srcdir PG_ABS_SRCDIR +\getenv abs_builddir PG_ABS_BUILDDIR + +\set filename :abs_builddir '/results/copy_raw_test_errors.data' + +-- Test single column requirement +CREATE TABLE copy_raw_test_errors (col1 text, col2 text); +COPY copy_raw_test_errors TO :'filename' (FORMAT raw); +COPY copy_raw_test_errors (col1, col2) TO :'filename' (FORMAT raw); +COPY copy_raw_test_errors FROM :'filename' (FORMAT raw); +COPY copy_raw_test_errors (col1, col2) FROM :'filename' (FORMAT raw); + +-- Test inconsistent newline style +\set filename :abs_srcdir '/data/newlines_mixed_1.data' +COPY copy_raw_test_errors (col1) FROM :'filename' (FORMAT raw); + +\set filename :abs_srcdir '/data/newlines_mixed_2.data' +COPY copy_raw_test_errors (col1) FROM :'filename' (FORMAT raw); + +\set filename :abs_srcdir '/data/newlines_mixed_3.data' +COPY copy_raw_test_errors (col1) FROM :'filename' (FORMAT raw); + +\set filename :abs_srcdir '/data/newlines_mixed_4.data' +COPY copy_raw_test_errors (col1) FROM :'filename' (FORMAT raw); + +\set filename :abs_srcdir '/data/newlines_mixed_5.data' +COPY copy_raw_test_errors (col1) FROM :'filename' (FORMAT raw); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 57de1acff3..59433d120e 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -491,6 +491,7 @@ ConversionLocation ConvertRowtypeExpr CookedConstraint CopyDest +CopyFormat CopyFormatOptions CopyFromState CopyFromStateData -- 2.45.1