Gabriel39 commented on code in PR #67784:
URL: https://github.com/apache/doris/pull/67784#discussion_r3986559239
##########
be/src/exec/operator/file_scan_operator.cpp:
##########
@@ -147,9 +148,18 @@ bool FileScanLocalState::_should_use_file_scanner_v2(const
TQueryOptions& query_
const bool is_transactional_hive =
scan_params.__isset.table_format_params &&
scan_params.table_format_params.table_format_type ==
"transactional_hive";
- return query_options.__isset.enable_file_scanner_v2 &&
query_options.enable_file_scanner_v2 &&
- !is_load && scan_params.format_type !=
TFileFormatType::FORMAT_ES_HTTP &&
- !is_transactional_hive;
+ const bool has_versioned_parquet_semantics =
+ scan_params.format_type == TFileFormatType::FORMAT_PARQUET &&
Review Comment:
Fixed in db858711d3. Versioned Paimon plans with native history schemas now
force FileScannerV2 even though their scan-level format remains FORMAT_JNI. The
unit test covers the scan-level shape, and the Paimon regression now runs
native scans with enable_file_scanner_v2=false.
##########
be/src/core/data_type_serde/data_type_serde.cpp:
##########
@@ -496,8 +524,28 @@ Status decode_timestamp_orc_values(IColumn& nested_column,
const OrcDecodedColum
}
auto& value =
reinterpret_cast<DateV2Value<DateTimeV2ValueType>&>(data[old_data_size + row]);
+ RoundedOrcTimestamp timestamp;
+ auto status = round_orc_timestamp_to_microseconds(
+ orc_batch->data[source_row],
orc_batch->nanoseconds[source_row], ×tamp);
+ if (!status.ok()) {
+ data.resize(old_data_size);
+ return status;
+ }
value.from_unixtime(orc_batch->data[source_row], timezone);
-
value.set_microsecond(cast_set<uint64_t>(orc_batch->nanoseconds[source_row] /
1000));
+ if (!value.is_valid_date()) {
+ data.resize(old_data_size);
+ return Status::DataQualityError(
+ "Decoded ORC timestamp is outside the target timezone
range");
+ }
+ value.set_microsecond(timestamp.microseconds);
+ // Plain ORC TIMESTAMP is a civil value. Carry after timezone
conversion so a fractional
+ // round does not jump backward or skip an hour at a daylight-saving
transition.
+ if (timestamp.carry &&
Review Comment:
Fixed in db858711d3. TIMESTAMP_INSTANT now applies the rounded epoch carry
before timezone conversion when mapped to DATETIMEV2, while plain TIMESTAMP
retains civil-time carry. Added DST fallback coverage for the unmapped path.
##########
be/src/format/transformer/vparquet_transformer.h:
##########
@@ -88,18 +89,20 @@ struct ParquetFileOptions {
// a wrapper of parquet output stream
class VParquetTransformer final : public VFileFormatTransformer {
public:
- VParquetTransformer(RuntimeState* state, doris::io::FileWriter*
file_writer,
- const VExprContextSPtrs& output_vexpr_ctxs,
- std::vector<std::string> column_names, bool
output_object_data,
- const ParquetFileOptions& parquet_options,
- const std::string* iceberg_schema_json = nullptr,
- const iceberg::Schema* iceberg_schema = nullptr);
-
- VParquetTransformer(RuntimeState* state, doris::io::FileWriter*
file_writer,
- const VExprContextSPtrs& output_vexpr_ctxs,
- std::vector<TParquetSchema> parquet_schemas, bool
output_object_data,
- const ParquetFileOptions& parquet_options,
- const std::string* iceberg_schema_json = nullptr);
+ VParquetTransformer(
+ RuntimeState* state, doris::io::FileWriter* file_writer,
+ const VExprContextSPtrs& output_vexpr_ctxs,
std::vector<std::string> column_names,
+ bool output_object_data, const ParquetFileOptions& parquet_options,
+ const std::string* iceberg_schema_json = nullptr,
+ const iceberg::Schema* iceberg_schema = nullptr,
+ const ArrowWriteConverter& arrow_write_converter =
plain_arrow_write_converter());
Review Comment:
Fixed in db858711d3. Both Iceberg Variant transformer tests now pass
iceberg_arrow_write_converter() together with the Iceberg schema, matching the
production writer setup.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
