From 160af36a258a6125a4213e3a267ce5d239a93c29 Mon Sep 17 00:00:00 2001 From: Joel Jacobson Date: Tue, 15 Oct 2024 03:03:09 +0200 Subject: [PATCH 2/2] Add raw format to COPY command. This commit introduces a new raw format to the COPY command, enabling efficient bulk data transfer of a single text column without any parsing, quoting, or escaping. In raw format, data is copied exactly as it appears in the file or table, adhering to the specified ENCODING option or the current client encoding. The raw format enforces a single column requirement, ensuring that exactly one column is specified in the column list. Attempts to specify multiple columns or omit the column list when the table has multiple columns will result in an error. Additionally, the DELIMITER option in raw format accepts any string, including multi-byte characters, providing greater flexibility in defining data separators. If no DELIMITER is specified, the entire input or output is treated as a single data value. Furthermore, the raw format does not support format-specific options such as NULL, HEADER, QUOTE, ESCAPE, FORCE_QUOTE, FORCE_NOT_NULL, and FORCE_NULL. Using these options with the raw format will trigger errors, ensuring that data remains unaltered during the transfer process. This enhancement is particularly useful when handling text blobs, JSON files, or other text-based formats where preserving the data "as is" is crucial. --- doc/src/sgml/ref/copy.sgml | 134 ++++++++++++++-- src/backend/commands/copy.c | 105 ++++++++----- src/backend/commands/copyfrom.c | 7 + src/backend/commands/copyfromparse.c | 188 ++++++++++++++++++++++- src/backend/commands/copyto.c | 92 ++++++++++- src/bin/psql/tab-complete.in.c | 2 +- src/include/commands/copy.h | 3 +- src/include/commands/copyfrom_internal.h | 1 + src/test/regress/expected/copy.out | 52 +++++++ src/test/regress/expected/copy2.out | 52 ++++++- src/test/regress/sql/copy.sql | 24 +++ src/test/regress/sql/copy2.sql | 37 ++++- 12 files changed, 625 insertions(+), 72 deletions(-) diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 8394402f09..f17d606537 100644 --- a/doc/src/sgml/ref/copy.sgml +++ b/doc/src/sgml/ref/copy.sgml @@ -218,8 +218,9 @@ COPY { table_name [ ( Selects the data format to be read or written: text, - csv (Comma Separated Values), - or binary. + CSV (Comma Separated Values), + binary, + or raw The default is text. See below for details. @@ -253,11 +254,27 @@ COPY { table_name [ ( DELIMITER - Specifies the character that separates columns within each row - (line) of the file. The default is a tab character in text format, - a comma in CSV format. - This must be a single one-byte character. - This option is not allowed when using binary format. + Specifies the delimiter used in the file. Its usage depends on the + FORMAT specified: + + + In text and CSV formats, + the delimiter separates columns within each row + (line) of the file. + The default is a tab character in text format and + a comma in CSV format. This must be a single + one-byte character. + + + In raw format, the delimiter separates + rows in the file. The default is no delimiter, + which means that for COPY FROM, the entire input is + read as a single field, and for COPY TO, the output + is concatenated without any delimiter. If a delimiter is specified, + it can be a multi-byte string; for example, E'\r\n' + can be used when dealing with text files on Windows platforms. + + @@ -271,7 +288,8 @@ COPY { table_name [ ( CSV format. You might prefer an empty string even in text format for cases where you don't want to distinguish nulls from empty strings. - This option is not allowed when using binary format. + This option is allowed only when using text or + CSV format. @@ -294,7 +312,7 @@ COPY { table_name [ ( COPY FROM, and only when - not using binary format. + using text or CSV format. @@ -310,7 +328,8 @@ COPY { table_name [ ( MATCH, the number and names of the columns in the header line must match the actual column names of the table, in order; otherwise an error is raised. - This option is not allowed when using binary format. + This option is allowed only when using text or + CSV format. The MATCH option is only valid for COPY FROM commands. @@ -400,7 +419,8 @@ COPY { table_name [ ( The ignore option is applicable only for COPY FROM - when the FORMAT is text or csv. + when the FORMAT is text, + CSV or raw. A NOTICE message containing the ignored row count is @@ -893,6 +913,98 @@ COPY count + + Raw Format + + + The raw format is designed for efficient bulk data + transfer of a single text column without any parsing, quoting, or + escaping. In this format, data is copied exactly as it appears in the file + or table, interpreted according to the specified ENCODING + option or the current client encoding. + + + + When using the raw format, each data value corresponds + to a single field with no additional formatting or processing. The + DELIMITER option specifies the string that separates + data values. Unlike in other formats, the delimiter in + raw format can be any string, including multi-byte + characters. If no DELIMITER is specified, the entire + input or output is treated as a single data value. + + + + The raw format requires that exactly one column be + specified in the column list. An error is raised if more than one column + is specified or if no column list is specified when the table has multiple + columns. + + + + The raw format does not support any of the + format-specific options of other formats, such as NULL, + HEADER, QUOTE, + ESCAPE, FORCE_QUOTE, + FORCE_NOT_NULL, and FORCE_NULL. + Attempting to use these options with raw format will + result in an error. + + + + Since the raw format deals with text, the data is + interpreted according to the specified ENCODING option + or the current client encoding for input, and encoded using the specified + ENCODING or the current client encoding for output. + + + + + Empty lines in the input are treated as empty strings, not as + NULL values. There is no way to represent a + NULL value in raw format. + + + + + + The raw format is particularly useful when you need to + import or export data exactly as it appears. This can be + helpful when dealing with large text blobs, JSON files, or other + text-based formats. + + + + + + The raw format can only be used when copying exactly + one column. If the table has multiple columns, you must specify the + column list containing only one column. + + + + + + Unlike other formats, the delimiter in raw format can + be any string, and there are no restrictions on the characters used in + the delimiter, including newline or carriage return characters. + + + + + + When using COPY TO with raw format + and a specified DELIMITER, there is no check to prevent + data values from containing the delimiter string, which could be + problematic if it would be needed to import the data preserved using + COPY FROM, since a data value containing the delimiter + would then be split into two values. If this is a concern, a different + format should be used instead. + + + + + Binary Format diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index a5cde15724..6bff50127c 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -516,6 +516,8 @@ ProcessCopyOptions(ParseState *pstate, opts_out->format = COPY_FORMAT_CSV; else if (strcmp(fmt, "binary") == 0) opts_out->format = COPY_FORMAT_BINARY; + else if (strcmp(fmt, "raw") == 0) + opts_out->format = COPY_FORMAT_RAW; else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), @@ -680,41 +682,47 @@ ProcessCopyOptions(ParseState *pstate, /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ errmsg("cannot specify %s in BINARY mode", "DELIMITER"))); - /* Only single-byte delimiter strings are supported. */ - if (strlen(opts_out->delim) != 1) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("COPY delimiter must be a single one-byte character"))); + if (opts_out->format == COPY_FORMAT_TEXT || + opts_out->format == COPY_FORMAT_CSV) + { + /* Only single-byte delimiter strings are supported. */ + if (strlen(opts_out->delim) != 1) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY delimiter must be a single one-byte character"))); - /* Disallow end-of-line characters */ - if (strchr(opts_out->delim, '\r') != NULL || - strchr(opts_out->delim, '\n') != NULL) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY delimiter cannot be newline or carriage return"))); + /* Disallow end-of-line characters */ + if (strchr(opts_out->delim, '\r') != NULL || + strchr(opts_out->delim, '\n') != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY delimiter cannot be newline or carriage return"))); + } - /* - * Disallow unsafe delimiter characters in non-CSV mode. We can't allow - * backslash because it would be ambiguous. We can't allow the other - * cases because data characters matching the delimiter must be - * backslashed, and certain backslash combinations are interpreted - * non-literally by COPY IN. Disallowing all lower case ASCII letters is - * more than strictly necessary, but seems best for consistency and - * future-proofing. Likewise we disallow all digits though only octal - * digits are actually dangerous. - */ - if (opts_out->format != COPY_FORMAT_CSV && - strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789", - opts_out->delim[0]) != NULL) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("COPY delimiter cannot be \"%s\"", opts_out->delim))); - } - else if (opts_out->format != COPY_FORMAT_BINARY) - { - /* Set default delimiter */ - opts_out->delim = opts_out->format == COPY_FORMAT_CSV ? "," : "\t"; + if (opts_out->format == COPY_FORMAT_TEXT) + { + /* + * Disallow unsafe delimiter characters in text mode. We can't allow + * backslash because it would be ambiguous. We can't allow the other + * cases because data characters matching the delimiter must be + * backslashed, and certain backslash combinations are interpreted + * non-literally by COPY IN. Disallowing all lower case ASCII letters is + * more than strictly necessary, but seems best for consistency and + * future-proofing. Likewise we disallow all digits though only octal + * digits are actually dangerous. + */ + if (strchr("\\.abcdefghijklmnopqrstuvwxyz0123456789", + opts_out->delim[0]) != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY delimiter cannot be \"%s\"", opts_out->delim))); + } } + /* Set default delimiter */ + else if (opts_out->format == COPY_FORMAT_CSV) + opts_out->delim = ","; + else if (opts_out->format == COPY_FORMAT_TEXT) + opts_out->delim = "\t"; /* --- NULL option --- */ if (opts_out->null_print) @@ -724,6 +732,11 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("cannot specify %s in BINARY mode", "NULL"))); + if (opts_out->format == COPY_FORMAT_RAW) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot specify %s in RAW mode", "NULL"))); + /* Disallow end-of-line characters */ if (strchr(opts_out->null_print, '\r') != NULL || strchr(opts_out->null_print, '\n') != NULL) @@ -731,11 +744,12 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("COPY null representation cannot use newline or carriage return"))); } - else if (opts_out->format != COPY_FORMAT_BINARY) - { - /* Set default null_print */ - opts_out->null_print = opts_out->format == COPY_FORMAT_CSV ? "" : "\\N"; - } + /* Set default null_print */ + else if (opts_out->format == COPY_FORMAT_CSV) + opts_out->null_print = ""; + else if (opts_out->format == COPY_FORMAT_TEXT) + opts_out->null_print = "\\N"; + if (opts_out->null_print) opts_out->null_print_len = strlen(opts_out->null_print); @@ -787,6 +801,11 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("cannot specify %s in BINARY mode", "DEFAULT"))); + if (opts_out->format == COPY_FORMAT_RAW) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("cannot specify %s in RAW mode", "DEFAULT"))); + /* Assert options have been set (defaults applied if not specified) */ Assert(opts_out->delim); Assert(opts_out->null_print); @@ -845,6 +864,12 @@ ProcessCopyOptions(ParseState *pstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ errmsg("cannot specify %s in BINARY mode", "HEADER"))); + + if (opts_out->format == COPY_FORMAT_RAW) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + /*- translator: %s is the name of a COPY option, e.g. ON_ERROR */ + errmsg("cannot specify %s in RAW mode", "HEADER"))); } else { @@ -933,8 +958,8 @@ ProcessCopyOptions(ParseState *pstate, ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), /*- translator: first and second %s are the names of COPY option, e.g. - * ON_ERROR, third is the value of the COPY option, e.g. IGNORE */ - errmsg("COPY %s requires %s to be set to %s", + * ON_ERROR, third is the value of the COPY option, e.g. IGNORE */ + errmsg("COPY %s requires %s to be set to %s", "REJECT_LIMIT", "ON_ERROR", "IGNORE"))); } @@ -977,7 +1002,7 @@ ProcessCopyOptions(ParseState *pstate, ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), /*- translator: %s is the name of a COPY option, e.g. NULL */ - errmsg("CSV quote character must not appear in the %s specification", + errmsg("CSV quote character must not appear in the %s specification", "NULL"))); } } diff --git a/src/backend/commands/copyfrom.c b/src/backend/commands/copyfrom.c index f350a4ff97..99dcb00f8a 100644 --- a/src/backend/commands/copyfrom.c +++ b/src/backend/commands/copyfrom.c @@ -1438,6 +1438,13 @@ BeginCopyFrom(ParseState *pstate, /* Generate or convert list of attributes to process */ cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist); + /* Enforce single column requirement for RAW format */ + if (cstate->opts.format == COPY_FORMAT_RAW && + list_length(cstate->attnumlist) != 1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY with format 'raw' must specify exactly one column"))); + num_phys_attrs = tupDesc->natts; /* Convert FORCE_NOT_NULL name list to per-column flags, check validity */ diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 50bb4b7750..d898fce2c2 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -7,7 +7,7 @@ * formats. The main entry point is NextCopyFrom(), which parses the * next input line and returns it as Datums. * - * In text/CSV mode, the parsing happens in multiple stages: + * In text/CSV/raw mode, the parsing happens in multiple stages: * * [data source] --> raw_buf --> input_buf --> line_buf --> attribute_buf * 1. 2. 3. 4. @@ -25,7 +25,7 @@ * is copied into 'line_buf', with quotes and escape characters still * intact. * - * 4. CopyReadAttributesText/CSV() function takes the input line from + * 4. CopyReadAttributesText/CSV/Raw() function takes the input line from * 'line_buf', and splits it into fields, unescaping the data as required. * The fields are stored in 'attribute_buf', and 'raw_fields' array holds * pointers to each field. @@ -143,8 +143,10 @@ static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0"; /* non-export function prototypes */ static bool CopyReadLine(CopyFromState cstate); static bool CopyReadLineText(CopyFromState cstate); +static bool CopyReadLineRawText(CopyFromState cstate); static int CopyReadAttributesText(CopyFromState cstate); static int CopyReadAttributesCSV(CopyFromState cstate); +static int CopyReadAttributesRaw(CopyFromState cstate); static Datum CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, Oid typioparam, int32 typmod, bool *isnull); @@ -732,7 +734,7 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) } /* - * Read raw fields in the next line for COPY FROM in text or csv mode. + * Read raw fields in the next line for COPY FROM in text, csv, or raw mode. * Return false if no more lines. * * An internal temporary buffer is returned via 'fields'. It is valid until @@ -748,7 +750,7 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) int fldct; bool done; - /* only available for text or csv input */ + /* only available for text, csv, or raw input */ Assert(cstate->opts.format != COPY_FORMAT_BINARY); /* on input check that the header line is correct if needed */ @@ -768,8 +770,13 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) if (cstate->opts.format == COPY_FORMAT_CSV) fldct = CopyReadAttributesCSV(cstate); - else + else if (cstate->opts.format == COPY_FORMAT_TEXT) fldct = CopyReadAttributesText(cstate); + else + { + elog(ERROR, "unexpected COPY format: %d", cstate->opts.format); + pg_unreachable(); + } if (fldct != list_length(cstate->attnumlist)) ereport(ERROR, @@ -823,8 +830,15 @@ NextCopyFromRawFields(CopyFromState cstate, char ***fields, int *nfields) /* Parse the line into de-escaped field values */ if (cstate->opts.format == COPY_FORMAT_CSV) fldct = CopyReadAttributesCSV(cstate); - else + else if (cstate->opts.format == COPY_FORMAT_TEXT) fldct = CopyReadAttributesText(cstate); + else if (cstate->opts.format == COPY_FORMAT_RAW) + fldct = CopyReadAttributesRaw(cstate); + else + { + elog(ERROR, "unexpected COPY format: %d", cstate->opts.format); + pg_unreachable(); + } *fields = cstate->raw_fields; *nfields = fldct; @@ -1096,7 +1110,10 @@ CopyReadLine(CopyFromState cstate) cstate->line_buf_valid = false; /* Parse data and transfer into line_buf */ - result = CopyReadLineText(cstate); + if (cstate->opts.format == COPY_FORMAT_RAW) + result = CopyReadLineRawText(cstate); + else + result = CopyReadLineText(cstate); if (result) { @@ -1147,6 +1164,21 @@ CopyReadLine(CopyFromState cstate) cstate->line_buf.len -= 2; cstate->line_buf.data[cstate->line_buf.len] = '\0'; break; + case EOL_CUSTOM: + { + int delim_len; + Assert(cstate->opts.format == COPY_FORMAT_RAW); + Assert(cstate->opts.delim); + delim_len = strlen(cstate->opts.delim); + Assert(delim_len > 0); + Assert(cstate->line_buf.len >= delim_len); + Assert(memcmp(cstate->line_buf.data + cstate->line_buf.len - delim_len, + cstate->opts.delim, + delim_len) == 0); + cstate->line_buf.len -= delim_len; + cstate->line_buf.data[cstate->line_buf.len] = '\0'; + } + break; case EOL_UNKNOWN: /* shouldn't get here */ Assert(false); @@ -1462,6 +1494,109 @@ CopyReadLineText(CopyFromState cstate) return result; } +/* + * CopyReadLineRawText - inner loop of CopyReadLine for raw text mode + */ +static bool +CopyReadLineRawText(CopyFromState cstate) +{ + char *copy_input_buf; + int input_buf_ptr; + int copy_buf_len; + bool need_data = false; + bool hit_eof = false; + bool result = false; + bool read_entire_file = (cstate->opts.delim == NULL); + int delim_len = cstate->opts.delim ? strlen(cstate->opts.delim) : 0; + + /* + * The objective of this loop is to transfer data into line_buf until we + * find the specified delimiter or reach EOF. In raw format, we treat the + * input data as-is, without any parsing, quoting, or escaping. We are + * only interested in locating the delimiter to determine the boundaries + * of each data value. + * + * If a delimiter is specified, we read data until we encounter the + * delimiter string. If no delimiter is specified, we read the entire + * input as a single data value. Unlike text or CSV modes, we do not need + * to handle line endings, escape sequences, or special characters. + * + * The input has already been converted to the database encoding. All + * supported server encodings have the property that all bytes in a + * multi-byte sequence have the high bit set, so a multibyte character + * cannot contain any newline or escape characters embedded in the + * multibyte sequence. Therefore, we can process the input byte-by-byte, + * regardless of the encoding. + * + * For speed, we try to move data from input_buf to line_buf in chunks + * rather than one character at a time. input_buf_ptr points to the next + * character to examine; any characters from input_buf_index to + * input_buf_ptr have been determined to be part of the line, but not yet + * transferred to line_buf. + * + * We handle both single-byte and multi-byte delimiters. For multi-byte + * delimiters, we ensure that we have enough data in the buffer to compare + * the delimiter string. + */ + copy_input_buf = cstate->input_buf; + input_buf_ptr = cstate->input_buf_index; + copy_buf_len = cstate->input_buf_len; + + for (;;) + { + int prev_raw_ptr; + + /* Load more data if needed */ + if (input_buf_ptr >= copy_buf_len || need_data) + { + REFILL_LINEBUF; + + CopyLoadInputBuf(cstate); + /* Update local variables */ + hit_eof = cstate->input_reached_eof; + input_buf_ptr = cstate->input_buf_index; + copy_buf_len = cstate->input_buf_len; + + /* If no more data, break out of the loop */ + if (INPUT_BUF_BYTES(cstate) <= 0) + { + result = true; + break; + } + need_data = false; + } + + /* Fetch a character */ + prev_raw_ptr = input_buf_ptr; + + if (read_entire_file) + { + /* Continue until EOF if reading entire file */ + input_buf_ptr++; + continue; + } + else + { + /* Check for delimiter, possibly multi-byte */ + IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(delim_len - 1); + if (strncmp(©_input_buf[input_buf_ptr], cstate->opts.delim, + delim_len) == 0) + { + cstate->eol_type = EOL_CUSTOM; + input_buf_ptr += delim_len; + break; + } + input_buf_ptr++; + } + } + + /* Transfer data to line_buf, including the delimiter if found */ + REFILL_LINEBUF; + + return result; +} + + /* * Return decimal value for a hexadecimal digit */ @@ -1938,6 +2073,45 @@ endfield: return fieldno; } +/* + * Parse the current line as a single attribute for the "raw" COPY format. + * No parsing, quoting, or escaping is performed. + * Empty lines are treated as empty strings, not NULL. + */ +static int +CopyReadAttributesRaw(CopyFromState cstate) +{ + /* Enforce single column requirement */ + if (cstate->max_fields != 1) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY with format 'raw' must specify exactly one column"))); + } + + resetStringInfo(&cstate->attribute_buf); + + /* + * The attribute will certainly not be longer than the input + * data line, so we can just force attribute_buf to be large enough and + * then transfer data without any checks for enough space. We need to do + * it this way because enlarging attribute_buf mid-stream would invalidate + * pointers already stored into cstate->raw_fields[]. + */ + if (cstate->attribute_buf.maxlen <= cstate->line_buf.len) + enlargeStringInfo(&cstate->attribute_buf, cstate->line_buf.len); + + /* Copy the entire line into attribute_buf */ + memcpy(cstate->attribute_buf.data, cstate->line_buf.data, + cstate->line_buf.len); + cstate->attribute_buf.data[cstate->line_buf.len] = '\0'; + cstate->attribute_buf.len = cstate->line_buf.len; + + /* Assign the single field to raw_fields[0] */ + cstate->raw_fields[0] = cstate->attribute_buf.data; + + return 1; +} /* * Read a binary attribute diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 78531ae846..ea277b66b1 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -113,6 +113,7 @@ static void CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot); static void CopyAttributeOutText(CopyToState cstate, const char *string); static void CopyAttributeOutCSV(CopyToState cstate, const char *string, bool use_quote); +static void CopyAttributeOutRaw(CopyToState cstate, const char *string); /* Low-level communications functions */ static void SendCopyBegin(CopyToState cstate); @@ -191,7 +192,14 @@ CopySendEndOfRow(CopyToState cstate) switch (cstate->copy_dest) { case COPY_FILE: - if (cstate->opts.format != COPY_FORMAT_BINARY) + if (cstate->opts.format == COPY_FORMAT_RAW && + cstate->opts.delim != NULL) + { + /* Output the user-specified delimiter between rows */ + CopySendString(cstate, cstate->opts.delim); + } + else if (cstate->opts.format == COPY_FORMAT_TEXT || + cstate->opts.format == COPY_FORMAT_CSV) { /* Default line termination depends on platform */ #ifndef WIN32 @@ -235,9 +243,18 @@ CopySendEndOfRow(CopyToState cstate) } break; case COPY_FRONTEND: - /* The FE/BE protocol uses \n as newline for all platforms */ - if (cstate->opts.format != COPY_FORMAT_BINARY) + if (cstate->opts.format == COPY_FORMAT_RAW && + cstate->opts.delim != NULL) + { + /* Output the user-specified delimiter between rows */ + CopySendString(cstate, cstate->opts.delim); + } + else if (cstate->opts.format == COPY_FORMAT_TEXT || + cstate->opts.format == COPY_FORMAT_CSV) + { + /* The FE/BE protocol uses \n as newline for all platforms */ CopySendChar(cstate, '\n'); + } /* Dump the accumulated row as one CopyData message */ (void) pq_putmessage(PqMsg_CopyData, fe_msgbuf->data, fe_msgbuf->len); @@ -570,6 +587,13 @@ BeginCopyTo(ParseState *pstate, /* Generate or convert list of attributes to process */ cstate->attnumlist = CopyGetAttnums(tupDesc, cstate->rel, attnamelist); + /* Enforce single column requirement for RAW format */ + if (cstate->opts.format == COPY_FORMAT_RAW && + list_length(cstate->attnumlist) != 1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("COPY with format 'raw' must specify exactly one column"))); + num_phys_attrs = tupDesc->natts; /* Convert FORCE_QUOTE name list to per-column flags, check validity */ @@ -835,8 +859,10 @@ DoCopyTo(CopyToState cstate) if (cstate->opts.format == COPY_FORMAT_CSV) CopyAttributeOutCSV(cstate, colname, false); - else + else if (cstate->opts.format == COPY_FORMAT_TEXT) CopyAttributeOutText(cstate, colname); + else if (cstate->opts.format == COPY_FORMAT_RAW) + CopyAttributeOutRaw(cstate, colname); } CopySendEndOfRow(cstate); @@ -917,7 +943,8 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) /* Make sure the tuple is fully deconstructed */ slot_getallattrs(slot); - if (cstate->opts.format != COPY_FORMAT_BINARY) + if (cstate->opts.format == COPY_FORMAT_TEXT || + cstate->opts.format == COPY_FORMAT_CSV) { bool need_delim = false; @@ -945,7 +972,7 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) } } } - else + else if (cstate->opts.format == COPY_FORMAT_BINARY) { foreach_int(attnum, cstate->attnumlist) { @@ -965,6 +992,37 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) } } } + else if (cstate->opts.format == COPY_FORMAT_RAW) + { + int attnum; + Datum value; + bool isnull; + + /* Ensure only one column is being copied */ + if (list_length(cstate->attnumlist) != 1) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("COPY with format 'raw' must specify exactly one column"))); + + attnum = linitial_int(cstate->attnumlist); + value = slot->tts_values[attnum - 1]; + isnull = slot->tts_isnull[attnum - 1]; + + if (!isnull) + { + char *string = OutputFunctionCall(&out_functions[attnum - 1], + value); + CopyAttributeOutRaw(cstate, string); + } + /* For RAW format, we don't send anything for NULL values */ + } + else + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("Unsupported COPY format"))); + } + CopySendEndOfRow(cstate); @@ -1219,6 +1277,28 @@ CopyAttributeOutCSV(CopyToState cstate, const char *string, } } +/* + * Send text representation of one attribute for RAW format. + */ +static void +CopyAttributeOutRaw(CopyToState cstate, const char *string) +{ + const char *ptr; + + /* Ensure the format is RAW */ + Assert(cstate->opts.format == COPY_FORMAT_RAW); + + /* Ensure exactly one column is being processed */ + Assert(list_length(cstate->attnumlist) == 1); + + if (cstate->need_transcoding) + ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding); + else + ptr = string; + + CopySendString(cstate, ptr); +} + /* * copy_dest_startup --- executor startup */ diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index 1be0056af7..7f8d6f4f94 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -3239,7 +3239,7 @@ match_previous_words(int pattern_id, /* Complete COPY FROM|TO filename WITH (FORMAT */ else if (Matches("COPY|\\copy", MatchAny, "FROM|TO", MatchAny, "WITH", "(", "FORMAT")) - COMPLETE_WITH("binary", "csv", "text"); + COMPLETE_WITH("binary", "csv", "text", "raw"); /* Complete COPY FROM filename WITH (ON_ERROR */ else if (Matches("COPY|\\copy", MatchAny, "FROM|TO", MatchAny, "WITH", "(", "ON_ERROR")) diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index c3d1df267f..8996bc89e5 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -59,6 +59,7 @@ typedef enum CopyFormat COPY_FORMAT_TEXT = 0, COPY_FORMAT_BINARY, COPY_FORMAT_CSV, + COPY_FORMAT_RAW, } CopyFormat; /* @@ -79,7 +80,7 @@ typedef struct CopyFormatOptions char *null_print_client; /* same converted to file encoding */ char *default_print; /* DEFAULT marker string */ int default_print_len; /* length of same */ - char *delim; /* column delimiter (must be 1 byte) */ + char *delim; /* delimiter (1 byte, except for raw format) */ char *quote; /* CSV quote char (must be 1 byte) */ char *escape; /* CSV escape char (must be 1 byte) */ List *force_quote; /* list of column names */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index cad52fcc78..b8693ae59e 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -38,6 +38,7 @@ typedef enum EolType EOL_NL, EOL_CR, EOL_CRNL, + EOL_CUSTOM, } EolType; /* diff --git a/src/test/regress/expected/copy.out b/src/test/regress/expected/copy.out index f554d42c84..2825d833ea 100644 --- a/src/test/regress/expected/copy.out +++ b/src/test/regress/expected/copy.out @@ -325,3 +325,55 @@ SELECT tableoid::regclass, id % 2 = 0 is_even, count(*) from parted_si GROUP BY (2 rows) DROP TABLE parted_si; +-- Test COPY FORMAT raw +\set filename :abs_srcdir '/data/emp.data' +CREATE TABLE copy_raw_test (col text); +COPY copy_raw_test FROM :'filename' (FORMAT raw); +SELECT col FROM copy_raw_test; + col +---------------------------------------- + sharon 25 (15,12) 1000 sam + + sam 30 (10,5) 2000 bill + + bill 20 (11,10) 1000 sharon+ + +(1 row) + +TRUNCATE copy_raw_test; +COPY copy_raw_test FROM :'filename' (FORMAT raw, DELIMITER E'\n'); +SELECT col FROM copy_raw_test ORDER BY col COLLATE "C"; + col +---------------------------------------- + bill 20 (11,10) 1000 sharon + sam 30 (10,5) 2000 bill + sharon 25 (15,12) 1000 sam +(3 rows) + +COPY copy_raw_test TO stdout (FORMAT raw, DELIMITER E'\n***\n'); +sharon 25 (15,12) 1000 sam +*** +sam 30 (10,5) 2000 bill +*** +bill 20 (11,10) 1000 sharon +*** +\qecho + +TRUNCATE copy_raw_test; +COPY copy_raw_test FROM stdin (FORMAT raw, DELIMITER E'\n***\n'); +SELECT col FROM copy_raw_test ORDER BY col COLLATE "C"; + col +-------- + + "def", + abc\. + ghi +(4 rows) + +COPY copy_raw_test TO stdout (FORMAT raw, DELIMITER E'\n***\n'); +abc\. +*** +"def", +*** + +*** +ghi +*** diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index 64ea33aeae..f31bd6a322 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -90,15 +90,35 @@ COPY x from stdin (format BINARY, delimiter ','); ERROR: cannot specify DELIMITER in BINARY mode COPY x from stdin (format BINARY, null 'x'); ERROR: cannot specify NULL in BINARY mode +COPY x from stdin (format RAW, null 'x'); +ERROR: cannot specify NULL in RAW mode +COPY x from stdin (format TEXT, escape 'x'); +ERROR: COPY ESCAPE requires CSV mode +COPY x from stdin (format BINARY, escape 'x'); +ERROR: COPY ESCAPE requires CSV mode +COPY x from stdin (format RAW, escape 'x'); +ERROR: COPY ESCAPE requires CSV mode +COPY x from stdin (format TEXT, quote 'x'); +ERROR: COPY QUOTE requires CSV mode +COPY x from stdin (format BINARY, quote 'x'); +ERROR: COPY QUOTE requires CSV mode +COPY x from stdin (format RAW, quote 'x'); +ERROR: COPY QUOTE requires CSV mode +COPY x from stdin (format RAW, header); +ERROR: cannot specify HEADER in RAW mode COPY x from stdin (format BINARY, on_error ignore); ERROR: only ON_ERROR STOP is allowed in BINARY mode COPY x from stdin (on_error unsupported); ERROR: COPY ON_ERROR "unsupported" not recognized LINE 1: COPY x from stdin (on_error unsupported); ^ -COPY x from stdin (format TEXT, force_quote(a)); +COPY x to stdout (format TEXT, force_quote(a)); ERROR: COPY FORCE_QUOTE requires CSV mode -COPY x from stdin (format TEXT, force_quote *); +COPY x to stdout (format TEXT, force_quote *); +ERROR: COPY FORCE_QUOTE requires CSV mode +COPY x to stdout (format RAW, force_quote(a)); +ERROR: COPY FORCE_QUOTE requires CSV mode +COPY x to stdout (format RAW, force_quote *); ERROR: COPY FORCE_QUOTE requires CSV mode COPY x from stdin (format CSV, force_quote(a)); ERROR: COPY FORCE_QUOTE cannot be used with COPY FROM @@ -108,6 +128,10 @@ COPY x from stdin (format TEXT, force_not_null(a)); ERROR: COPY FORCE_NOT_NULL requires CSV mode COPY x from stdin (format TEXT, force_not_null *); ERROR: COPY FORCE_NOT_NULL requires CSV mode +COPY x from stdin (format RAW, force_not_null(a)); +ERROR: COPY FORCE_NOT_NULL requires CSV mode +COPY x from stdin (format RAW, force_not_null *); +ERROR: COPY FORCE_NOT_NULL requires CSV mode COPY x to stdout (format CSV, force_not_null(a)); ERROR: COPY FORCE_NOT_NULL cannot be used with COPY TO COPY x to stdout (format CSV, force_not_null *); @@ -116,6 +140,10 @@ COPY x from stdin (format TEXT, force_null(a)); ERROR: COPY FORCE_NULL requires CSV mode COPY x from stdin (format TEXT, force_null *); ERROR: COPY FORCE_NULL requires CSV mode +COPY x from stdin (format RAW, force_null(a)); +ERROR: COPY FORCE_NULL requires CSV mode +COPY x from stdin (format RAW, force_null *); +ERROR: COPY FORCE_NULL requires CSV mode COPY x to stdout (format CSV, force_null(a)); ERROR: COPY FORCE_NULL cannot be used with COPY TO COPY x to stdout (format CSV, force_null *); @@ -858,9 +886,11 @@ select id, text_value, ts_value from copy_default; (2 rows) truncate copy_default; --- DEFAULT cannot be used in binary mode +-- DEFAULT cannot be used in binary or raw mode copy copy_default from stdin with (format binary, default '\D'); ERROR: cannot specify DEFAULT in BINARY mode +copy copy_default from stdin with (format raw, default '\D'); +ERROR: cannot specify DEFAULT in RAW mode -- DEFAULT cannot be new line nor carriage return copy copy_default from stdin with (default E'\n'); ERROR: COPY default representation cannot use newline or carriage return @@ -929,3 +959,19 @@ truncate copy_default; -- DEFAULT cannot be used in COPY TO copy (select 1 as test) TO stdout with (default '\D'); ERROR: COPY DEFAULT cannot be used with COPY TO +-- +-- Test COPY FORMAT errors +-- +\getenv abs_srcdir PG_ABS_SRCDIR +\getenv abs_builddir PG_ABS_BUILDDIR +\set filename :abs_builddir '/results/copy_raw_test_errors.data' +-- Test single column requirement +CREATE TABLE copy_raw_test_errors (col1 text, col2 text); +COPY copy_raw_test_errors TO :'filename' (FORMAT raw); +ERROR: COPY with format 'raw' must specify exactly one column +COPY copy_raw_test_errors (col1, col2) TO :'filename' (FORMAT raw); +ERROR: COPY with format 'raw' must specify exactly one column +COPY copy_raw_test_errors FROM :'filename' (FORMAT raw); +ERROR: COPY with format 'raw' must specify exactly one column +COPY copy_raw_test_errors (col1, col2) FROM :'filename' (FORMAT raw); +ERROR: COPY with format 'raw' must specify exactly one column diff --git a/src/test/regress/sql/copy.sql b/src/test/regress/sql/copy.sql index f1699b66b0..93595037dc 100644 --- a/src/test/regress/sql/copy.sql +++ b/src/test/regress/sql/copy.sql @@ -348,3 +348,27 @@ COPY parted_si(id, data) FROM :'filename'; SELECT tableoid::regclass, id % 2 = 0 is_even, count(*) from parted_si GROUP BY 1, 2 ORDER BY 1; DROP TABLE parted_si; + +-- Test COPY FORMAT raw +\set filename :abs_srcdir '/data/emp.data' +CREATE TABLE copy_raw_test (col text); +COPY copy_raw_test FROM :'filename' (FORMAT raw); +SELECT col FROM copy_raw_test; +TRUNCATE copy_raw_test; +COPY copy_raw_test FROM :'filename' (FORMAT raw, DELIMITER E'\n'); +SELECT col FROM copy_raw_test ORDER BY col COLLATE "C"; +COPY copy_raw_test TO stdout (FORMAT raw, DELIMITER E'\n***\n'); +\qecho +TRUNCATE copy_raw_test; +COPY copy_raw_test FROM stdin (FORMAT raw, DELIMITER E'\n***\n'); +abc\. +*** +"def", +*** + +*** +ghi +*** +\. +SELECT col FROM copy_raw_test ORDER BY col COLLATE "C"; +COPY copy_raw_test TO stdout (FORMAT raw, DELIMITER E'\n***\n'); diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql index 45273557ce..7aee4ca8ea 100644 --- a/src/test/regress/sql/copy2.sql +++ b/src/test/regress/sql/copy2.sql @@ -72,18 +72,32 @@ COPY x from stdin (log_verbosity default, log_verbosity verbose); -- incorrect options COPY x from stdin (format BINARY, delimiter ','); COPY x from stdin (format BINARY, null 'x'); +COPY x from stdin (format RAW, null 'x'); +COPY x from stdin (format TEXT, escape 'x'); +COPY x from stdin (format BINARY, escape 'x'); +COPY x from stdin (format RAW, escape 'x'); +COPY x from stdin (format TEXT, quote 'x'); +COPY x from stdin (format BINARY, quote 'x'); +COPY x from stdin (format RAW, quote 'x'); +COPY x from stdin (format RAW, header); COPY x from stdin (format BINARY, on_error ignore); COPY x from stdin (on_error unsupported); -COPY x from stdin (format TEXT, force_quote(a)); -COPY x from stdin (format TEXT, force_quote *); +COPY x to stdout (format TEXT, force_quote(a)); +COPY x to stdout (format TEXT, force_quote *); +COPY x to stdout (format RAW, force_quote(a)); +COPY x to stdout (format RAW, force_quote *); COPY x from stdin (format CSV, force_quote(a)); COPY x from stdin (format CSV, force_quote *); COPY x from stdin (format TEXT, force_not_null(a)); COPY x from stdin (format TEXT, force_not_null *); +COPY x from stdin (format RAW, force_not_null(a)); +COPY x from stdin (format RAW, force_not_null *); COPY x to stdout (format CSV, force_not_null(a)); COPY x to stdout (format CSV, force_not_null *); COPY x from stdin (format TEXT, force_null(a)); COPY x from stdin (format TEXT, force_null *); +COPY x from stdin (format RAW, force_null(a)); +COPY x from stdin (format RAW, force_null *); COPY x to stdout (format CSV, force_null(a)); COPY x to stdout (format CSV, force_null *); COPY x to stdout (format BINARY, on_error unsupported); @@ -636,8 +650,9 @@ select id, text_value, ts_value from copy_default; truncate copy_default; --- DEFAULT cannot be used in binary mode +-- DEFAULT cannot be used in binary or raw mode copy copy_default from stdin with (format binary, default '\D'); +copy copy_default from stdin with (format raw, default '\D'); -- DEFAULT cannot be new line nor carriage return copy copy_default from stdin with (default E'\n'); @@ -707,3 +722,19 @@ truncate copy_default; -- DEFAULT cannot be used in COPY TO copy (select 1 as test) TO stdout with (default '\D'); + +-- +-- Test COPY FORMAT errors +-- + +\getenv abs_srcdir PG_ABS_SRCDIR +\getenv abs_builddir PG_ABS_BUILDDIR + +\set filename :abs_builddir '/results/copy_raw_test_errors.data' + +-- Test single column requirement +CREATE TABLE copy_raw_test_errors (col1 text, col2 text); +COPY copy_raw_test_errors TO :'filename' (FORMAT raw); +COPY copy_raw_test_errors (col1, col2) TO :'filename' (FORMAT raw); +COPY copy_raw_test_errors FROM :'filename' (FORMAT raw); +COPY copy_raw_test_errors (col1, col2) FROM :'filename' (FORMAT raw); -- 2.45.1