[GitHub] incubator-quickstep pull request #291: Add "COPY TO" operator for exporting ...

[jianqiao]

Github user jianqiao commented on a diff in the pull request:

    https://github.com/apache/incubator-quickstep/pull/291#discussion_r136199029
  
    --- Diff: query_optimizer/resolver/Resolver.cpp ---
    @@ -418,27 +455,157 @@ L::LogicalPtr Resolver::resolve(const ParseStatement 
&parse_query) {
     }
     
     L::LogicalPtr Resolver::resolveCopyFrom(
    -    const ParseStatementCopyFrom &copy_from_statement) {
    -  // Default parameters.
    -  std::string column_delimiter_ = "\t";
    -  bool escape_strings_ = true;
    +    const ParseStatementCopy &copy_from_statement) {
    +  DCHECK(copy_from_statement.getCopyDirection() == 
ParseStatementCopy::kFrom);
    +  const PtrList<ParseKeyValue> *params = copy_from_statement.params();
     
    -  const ParseCopyFromParams *params = copy_from_statement.params();
    +  BulkIOFormat file_format = BulkIOFormat::kText;
       if (params != nullptr) {
    -    if (params->delimiter != nullptr) {
    -      column_delimiter_ = params->delimiter->value();
    -      if (column_delimiter_.size() != 1) {
    -        THROW_SQL_ERROR_AT(params->delimiter)
    -            << "DELIMITER is not a single character";
    +    for (const ParseKeyValue &param : *params) {
    +      const std::string &key = ToLower(param.key()->value());
    +      if (key == "format") {
    +        const ParseString *parse_format = GetKeyValueString(param);
    +        const std::string format = ToLower(parse_format->value());
    +        // TODO(jianqiao): Support other bulk load formats such as CSV.
    +        if (format != "text") {
    +          THROW_SQL_ERROR_AT(parse_format) << "Unsupported file format: " 
<< format;
    +        }
    +        // Update file_format when other formats get supported.
    +        break;
    +      }
    +    }
    +  }
    +
    +  std::unique_ptr<BulkIOConfiguration> options =
    +      std::make_unique<BulkIOConfiguration>(file_format);
    +  if (params != nullptr) {
    +    for (const ParseKeyValue &param : *params) {
    +      const std::string &key = ToLower(param.key()->value());
    +      if (key == "delimiter") {
    +        const ParseString *parse_delimiter = GetKeyValueString(param);
    +        const std::string &delimiter = parse_delimiter->value();
    +        if (delimiter.size() != 1) {
    +          THROW_SQL_ERROR_AT(parse_delimiter)
    +              << "DELIMITER is not a single character";
    +        }
    +        options->setDelimiter(delimiter.front());
    +      } else if (key == "escape_strings") {
    +        options->setEscapeStrings(GetKeyValueBool(param));
    +      } else if (key != "format") {
    +        THROW_SQL_ERROR_AT(&param) << "Unsupported copy option: " << key;
           }
         }
    -    escape_strings_ = params->escape_strings;
       }
     
       return 
L::CopyFrom::Create(resolveRelationName(copy_from_statement.relation_name()),
    -                             
copy_from_statement.source_filename()->value(),
    -                             column_delimiter_[0],
    -                             escape_strings_);
    +                             copy_from_statement.file_name()->value(),
    +                             BulkIOConfigurationPtr(options.release()));
    +}
    +
    +L::LogicalPtr Resolver::resolveCopyTo(
    +    const ParseStatementCopy &copy_to_statement) {
    +  DCHECK(copy_to_statement.getCopyDirection() == ParseStatementCopy::kTo);
    +  const PtrList<ParseKeyValue> *params = copy_to_statement.params();
    +
    +  // Check if copy format is explicitly specified.
    +  BulkIOFormat file_format = BulkIOFormat::kText;
    +  bool format_specified = false;
    +  if (params != nullptr) {
    +    for (const ParseKeyValue &param : *params) {
    +      const std::string &key = ToLower(param.key()->value());
    +      if (key == "format") {
    +        const ParseString *parse_format = GetKeyValueString(param);
    +        const std::string format = ToLower(parse_format->value());
    +        if (format == "csv") {
    +          file_format = BulkIOFormat::kCSV;
    +        } else if (format == "text") {
    +          file_format = BulkIOFormat::kText;
    +        } else {
    +          THROW_SQL_ERROR_AT(parse_format) << "Unsupported file format: " 
<< format;
    +        }
    +        format_specified = true;
    +        break;
    +      }
    +    }
    +  }
    +
    +  const std::string &file_name = copy_to_statement.file_name()->value();
    +  if (file_name.length() <= 1) {
    +    THROW_SQL_ERROR_AT(copy_to_statement.file_name())
    +        << "File name can not be empty";
    +  }
    +
    +  // Infer copy format from file name extension.
    +  if (!format_specified) {
    +    if (file_name.length() > 4) {
    +      if (ToLower(file_name.substr(file_name.length() - 4)) == ".csv") {
    +        file_format = BulkIOFormat::kCSV;
    +      }
    +    }
    +  }
    +
    +  // Resolve the copy options.
    +  std::unique_ptr<BulkIOConfiguration> options =
    +      std::make_unique<BulkIOConfiguration>(file_format);
    +  if (params != nullptr) {
    +    for (const ParseKeyValue &param : *params) {
    +      const std::string &key = ToLower(param.key()->value());
    +      if (key == "delimiter") {
    +        const ParseString *parse_delimiter = GetKeyValueString(param);
    +        const std::string &delimiter = parse_delimiter->value();
    +        if (delimiter.size() != 1) {
    +          THROW_SQL_ERROR_AT(parse_delimiter)
    +              << "DELIMITER is not a single character";
    +        }
    +        options->setDelimiter(delimiter.front());
    +      } else if (file_format == BulkIOFormat::kText && key == 
"escape_strings") {
    +        options->setEscapeStrings(GetKeyValueBool(param));
    +      } else if (file_format == BulkIOFormat::kCSV && key == "header") {
    +        options->setHeader(GetKeyValueBool(param));
    +      } else if (file_format == BulkIOFormat::kCSV && key == "quote") {
    +        const ParseString *parse_quote = GetKeyValueString(param);
    +        const std::string &quote = parse_quote->value();
    +        if (quote.size() != 1) {
    +          THROW_SQL_ERROR_AT(parse_quote)
    +              << "QUOTE is not a single character";
    +        }
    +        options->setQuoteCharacter(quote.front());
    +      } else if (key == "null_string") {
    +        const ParseString *parse_null_string = GetKeyValueString(param);
    +        options->setNullString(parse_null_string->value());
    +      } else if (key != "format") {
    +        THROW_SQL_ERROR_AT(&param)
    +            << "Unsupported copy option \"" << key
    +            << "\" for file format " << options->getFormatName();
    +      }
    +    }
    +  }
    +
    +  // Resolve the source relation.
    +  L::LogicalPtr input;
    +  if (copy_to_statement.set_operation_query() != nullptr) {
    +    input = resolveSetOperation(*copy_to_statement.set_operation_query(),
    +                                "" /* set_operation_name */,
    +                                nullptr /* type_hints */,
    +                                nullptr /* parent_resolver */);
    +  } else {
    +    const ParseString *relation_name = copy_to_statement.relation_name();
    +    DCHECK(relation_name != nullptr);
    +    std::unique_ptr<ParseTableReference> table_reference =
    +        std::make_unique<ParseSimpleTableReference>(
    --- End diff --
    
    Updated to stack variable.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---