eldenmoon commented on code in PR #66310:
URL: https://github.com/apache/doris/pull/66310#discussion_r3686201270
##########
be/src/format_v2/table_reader.h:
##########
@@ -1339,32 +1401,107 @@ class TableReader {
return Status::OK();
}
if (mapping.default_expr != nullptr) {
- if (current_block->rows() == rows) {
- ColumnWithTypeAndName result;
- RETURN_IF_ERROR(_execute_default_expr_without_root_type_check(
- mapping.default_expr, current_block, &result));
- ColumnPtr result_column = result.column;
- RETURN_IF_ERROR(_align_column_nullability(&result_column,
mapping.table_type));
- *column = _detach_column(std::move(result_column));
- } else {
- DORIS_CHECK(mapping.constant_index.has_value());
- Block eval_block;
-
eval_block.insert({mapping.table_type->create_column_const_with_default_value(rows),
- mapping.table_type,
"__table_reader_const_rows"});
- ColumnWithTypeAndName result;
- RETURN_IF_ERROR(_execute_default_expr_without_root_type_check(
- mapping.default_expr, &eval_block, &result));
- ColumnPtr result_column = result.column;
- RETURN_IF_ERROR(_align_column_nullability(&result_column,
mapping.table_type));
- *column = _detach_column(std::move(result_column));
- }
- return Status::OK();
+ return _materialize_default_mapping_column(mapping, current_block,
rows, column);
}
ColumnPtr result_column =
mapping.table_type->create_column_const_with_default_value(rows);
*column = _detach_column(std::move(result_column));
return Status::OK();
}
+ Status _materialize_default_mapping_column(const ColumnMapping& mapping,
Block* current_block,
+ size_t rows, ColumnPtr* column)
{
+ DORIS_CHECK(mapping.default_expr != nullptr);
+ DORIS_CHECK(mapping.table_type != nullptr);
+ DORIS_CHECK(current_block != nullptr);
+ DORIS_CHECK(column != nullptr);
+ if (current_block->rows() == rows) {
+ ColumnWithTypeAndName result;
+
RETURN_IF_ERROR(_execute_default_expr_without_root_type_check(mapping.default_expr,
+
current_block, &result));
+ ColumnPtr result_column = result.column;
+ RETURN_IF_ERROR(_align_column_nullability(&result_column,
mapping.table_type));
+ *column = _detach_column(std::move(result_column));
+ return Status::OK();
+ }
+
+ DORIS_CHECK(mapping.constant_index.has_value());
+ Block eval_block;
+
eval_block.insert({mapping.table_type->create_column_const_with_default_value(rows),
+ mapping.table_type, "__table_reader_const_rows"});
+ ColumnWithTypeAndName result;
+
RETURN_IF_ERROR(_execute_default_expr_without_root_type_check(mapping.default_expr,
+
&eval_block, &result));
+ ColumnPtr result_column = result.column;
+ RETURN_IF_ERROR(_align_column_nullability(&result_column,
mapping.table_type));
+ *column = _detach_column(std::move(result_column));
+ return Status::OK();
+ }
+
+ Status _materialize_variant_path_mapping_column(const ColumnMapping&
mapping,
+ Block* current_block,
size_t rows,
+ ColumnPtr* column,
+ bool
take_projection_result) {
+ DORIS_CHECK(!mapping.column_paths.empty());
+ DORIS_CHECK(mapping.resolved_variant_path != nullptr);
+ DORIS_CHECK(mapping.table_type != nullptr);
+ DORIS_CHECK(current_block != nullptr);
+ DORIS_CHECK(column != nullptr);
+
+ ColumnPtr root_column;
+ if (mapping.projection != nullptr) {
+ int result_id;
+ auto status = mapping.projection->execute(current_block,
&result_id);
+ if (!status.ok()) {
+ return Status::InternalError(
+ "Failed to read Variant root carrier for Path Slot
'{}' "
+ "(global_index={}, rows={}): {}, mapping={}",
+ mapping.table_column_name,
mapping.global_index.value(), rows,
+ status.to_string(), mapping.debug_string());
+ }
+ root_column = take_projection_result
+ ?
_take_and_detach_block_column(current_block, result_id)
+ :
current_block->get_by_position(result_id).column;
+ } else if (mapping.default_expr != nullptr) {
+ // Schema-evolution defaults describe the missing root column.
Apply the same path
+ // extraction after evaluating that root value instead of exposing
the whole default.
+ RETURN_IF_ERROR(_materialize_default_mapping_column(mapping,
current_block, rows,
+ &root_column));
+ } else {
+ // A missing root without a default makes every requested subpath
SQL NULL.
+ *column = _detach_column(
+
mapping.table_type->create_column_const_with_default_value(rows));
+ return Status::OK();
+ }
+
+ const ColumnPtr materialized =
root_column->convert_to_full_column_if_const();
+ const IColumn* physical = materialized.get();
+ std::span<const uint8_t> outer_nulls;
+ if (const auto* nullable =
check_and_get_column<ColumnNullable>(physical)) {
+ outer_nulls = nullable->get_null_map_data();
+ physical = &nullable->get_nested_column();
+ }
+ const auto* variant = check_and_get_column<ColumnVariantV2>(physical);
+ if (variant == nullptr) {
+ return Status::NotSupported(
+ "Variant Path Slot '{}' requires a ColumnVariantV2 root
carrier, got {}. "
+ "Legacy ColumnVariant path extraction is not equivalent
because it also needs "
+ "sparse/document fallback semantics",
+ mapping.table_column_name, materialized->get_name());
+ }
+ if (variant->size() != rows) {
+ return Status::InternalError(
+ "Variant root carrier for Path Slot '{}' has {} rows,
expected {}",
+ mapping.table_column_name, variant->size(), rows);
+ }
+
+ ColumnPtr result;
+ RETURN_IF_ERROR(extract_variant_element_v2(*variant,
*mapping.resolved_variant_path,
Review Comment:
Fixed in 171f33ed2a46. TableReader now records the full pre-extraction
Variant root/default carrier bytes, and FileScannerV2 feeds max(final output
bytes, materialization input bytes) into the existing adaptive predictor
through a small explicit-bytes overload. This keeps the current full-root
Parquet fallback from teaching the predictor only the tiny extracted leaf
width. Covered at the TableReader, FileScannerV2, and
AdaptiveBlockSizePredictor layers by
VariantPathTracksFullCarrierBytesForAdaptiveBatching,
MissingVariantPathRootExtractsFromDefaultCarrier,
AdaptiveBatchSizeAccountsForPreExtractionVariantCarrier, and
ExplicitBytesAccountForPreMaterializationCarrier.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
