github-actions[bot] commented on code in PR #63192: URL: https://github.com/apache/doris/pull/63192#discussion_r3253401198
########## be/src/format/parquet/parquet_nested_column_utils.cpp: ########## @@ -0,0 +1,529 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format/parquet/parquet_nested_column_utils.h" + +#include <algorithm> +#include <cctype> +#include <string_view> +#include <unordered_map> +#include <utility> + +#include "core/data_type/data_type_nullable.h" +#include "format/parquet/schema_desc.h" + +namespace doris { +namespace { + +enum class NestedPathMode { + NAME, + FIELD_ID, +}; + +void add_column_id_range(const FieldSchema& field_schema, std::set<uint64_t>& column_ids) { + const uint64_t start_id = field_schema.get_column_id(); + const uint64_t max_column_id = field_schema.get_max_column_id(); + for (uint64_t id = start_id; id <= max_column_id; ++id) { + column_ids.insert(id); + } +} + +const FieldSchema* find_child_by_structural_name(const FieldSchema& field_schema, + std::string_view name) { + std::string lower_name(name); + std::transform(lower_name.begin(), lower_name.end(), lower_name.begin(), + [](unsigned char c) { return static_cast<char>(std::tolower(c)); }); + for (const auto& child : field_schema.children) { + if (child.name == name || child.lower_case_name == lower_name) { + return &child; + } + } + return nullptr; +} + +const FieldSchema* find_child_by_exact_name(const FieldSchema& field_schema, + std::string_view name) { + for (const auto& child : field_schema.children) { + if (child.name == name) { + return &child; + } + } + return nullptr; +} + +const FieldSchema* find_variant_typed_child_by_key(const FieldSchema& field_schema, + std::string_view key, NestedPathMode mode) { + if (const auto* child = find_child_by_exact_name(field_schema, key)) { + return child; + } + if (mode == NestedPathMode::NAME) { + return nullptr; + } + for (const auto& child : field_schema.children) { + if (child.field_id >= 0 && key == std::to_string(child.field_id)) { + return &child; + } + } + return nullptr; +} + +void add_variant_metadata(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) { + if (const auto* metadata = find_child_by_structural_name(variant_field, "metadata")) { + add_column_id_range(*metadata, column_ids); + } +} + +void add_variant_value(const FieldSchema& variant_field, std::set<uint64_t>& column_ids) { + add_variant_metadata(variant_field, column_ids); + if (const auto* value = find_child_by_structural_name(variant_field, "value")) { + add_column_id_range(*value, column_ids); + } +} + +struct VariantColumnIdExtractionResult { + bool has_child_columns = false; + bool needs_metadata = false; +}; + +using VariantPathMap = std::unordered_map<std::string, std::vector<std::vector<std::string>>>; + +bool is_shredded_variant_field(const FieldSchema& field_schema) { + bool has_value = false; + const FieldSchema* typed_value = nullptr; + for (const auto& child : field_schema.children) { + if (child.lower_case_name == "value") { + if (child.physical_type != tparquet::Type::BYTE_ARRAY) { + return false; + } + has_value = true; + continue; + } + if (child.lower_case_name == "typed_value") { + typed_value = &child; + continue; + } + return false; + } + if (has_value) { + return true; + } + if (typed_value == nullptr) { + return false; + } + const auto type = remove_nullable(typed_value->data_type); + return type->get_primitive_type() == TYPE_STRUCT || type->get_primitive_type() == TYPE_ARRAY; +} + +bool add_shredded_variant_field_value(const FieldSchema& shredded_field, + std::set<uint64_t>& column_ids) { + if (const auto* value = find_child_by_structural_name(shredded_field, "value")) { + add_column_id_range(*value, column_ids); + return true; + } + return false; +} + +bool is_variant_array_subscript(std::string_view path) { + return !path.empty() && + std::all_of(path.begin(), path.end(), [](unsigned char c) { return std::isdigit(c); }); +} + +bool is_terminal_variant_meta_component(std::string_view path) { + return path == "NULL" || path == "OFFSET"; +} + +const std::vector<std::string>& effective_variant_path(const std::vector<std::string>& raw_path, + std::vector<std::string>& stripped_path) { + if (!raw_path.empty() && is_terminal_variant_meta_component(raw_path.back())) { + stripped_path.assign(raw_path.begin(), raw_path.end() - 1); + return stripped_path; + } + return raw_path; +} + +bool contains_inherited_metadata_value(const FieldSchema& field_schema) { + if (is_shredded_variant_field(field_schema) && + find_child_by_structural_name(field_schema, "value") != nullptr) { + return true; + } + return std::any_of( + field_schema.children.begin(), field_schema.children.end(), + [](const FieldSchema& child) { return contains_inherited_metadata_value(child); }); +} + +VariantColumnIdExtractionResult extract_variant_typed_nested_column_ids( + const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths, + std::set<uint64_t>& column_ids, NestedPathMode mode); + +VariantColumnIdExtractionResult extract_typed_value_path(const FieldSchema& typed_value, + const std::vector<std::string>& path, + std::set<uint64_t>& column_ids, + NestedPathMode mode) { + VariantColumnIdExtractionResult result; + const auto typed_value_type = remove_nullable(typed_value.data_type); + if (typed_value_type->get_primitive_type() != TYPE_STRUCT) { + result = extract_variant_typed_nested_column_ids(typed_value, {path}, column_ids, mode); + } else if (const auto* typed_child = + find_variant_typed_child_by_key(typed_value, path[0], mode)) { + if (path.size() == 1) { + add_column_id_range(*typed_child, column_ids); + result.has_child_columns = true; + result.needs_metadata = contains_inherited_metadata_value(*typed_child); + } else { + std::vector<std::vector<std::string>> child_paths { + std::vector<std::string>(path.begin() + 1, path.end())}; + result = extract_variant_typed_nested_column_ids(*typed_child, child_paths, column_ids, + mode); + } + } + + if (result.has_child_columns) { + column_ids.insert(typed_value.get_column_id()); + } + return result; +} + +void add_variant_typed_path(PrimitiveType field_type, const FieldSchema& field_schema, + const std::vector<std::string>& path, + VariantColumnIdExtractionResult* result, std::set<uint64_t>& column_ids, + VariantPathMap* child_paths) { + if (path.empty()) { + add_column_id_range(field_schema, column_ids); + result->has_child_columns = true; + result->needs_metadata |= contains_inherited_metadata_value(field_schema); + return; + } + + const bool is_list = field_type == PrimitiveType::TYPE_ARRAY; + const bool is_map = field_type == PrimitiveType::TYPE_MAP; + std::vector<std::string> remaining; + std::string child_key; + if (is_list) { + child_key = "*"; + if (!is_variant_array_subscript(path[0])) { + remaining.assign(path.begin(), path.end()); + } else if (path.size() > 1) { + remaining.assign(path.begin() + 1, path.end()); + } + } else if (is_map) { + (*child_paths)["KEYS"].emplace_back(); + child_key = "VALUES"; + if (path.size() > 1) { + remaining.assign(path.begin() + 1, path.end()); + } + } else { + child_key = path[0]; + if (path.size() > 1) { + remaining.assign(path.begin() + 1, path.end()); + } + } + (*child_paths)[child_key].push_back(std::move(remaining)); +} + +std::string variant_typed_child_key(PrimitiveType field_type, const FieldSchema& field_schema, + uint64_t child_index) { + if (field_type == PrimitiveType::TYPE_ARRAY) { + return "*"; + } + if (field_type == PrimitiveType::TYPE_MAP) { + if (child_index == 0) { + return "KEYS"; + } + return child_index == 1 ? "VALUES" : ""; + } + return field_schema.children[child_index].name; +} + +void append_variant_child_paths(const VariantPathMap& paths_by_name, const std::string& key, + std::vector<std::vector<std::string>>& child_paths) { + auto child_paths_it = paths_by_name.find(key); + if (child_paths_it != paths_by_name.end()) { + child_paths.insert(child_paths.end(), child_paths_it->second.begin(), + child_paths_it->second.end()); + } +} + +std::vector<std::vector<std::string>> collect_variant_typed_child_paths( + const VariantPathMap& paths_by_name, const FieldSchema& child, const std::string& child_key, + bool is_list, bool is_map, NestedPathMode mode) { + std::vector<std::vector<std::string>> child_paths; + append_variant_child_paths(paths_by_name, child_key, child_paths); + if (!is_list && !is_map && mode == NestedPathMode::FIELD_ID && child.field_id >= 0) { + const std::string field_id_key = std::to_string(child.field_id); + if (field_id_key != child_key) { + append_variant_child_paths(paths_by_name, field_id_key, child_paths); + } + } + return child_paths; +} + +void extract_variant_typed_child_column_ids( + const FieldSchema& child, const std::vector<std::vector<std::string>>& child_paths, + std::set<uint64_t>& column_ids, NestedPathMode mode, + VariantColumnIdExtractionResult* result) { + const bool needs_full_child = + std::any_of(child_paths.begin(), child_paths.end(), + [](const std::vector<std::string>& path) { return path.empty(); }); + if (needs_full_child) { + add_column_id_range(child, column_ids); + result->has_child_columns = true; + result->needs_metadata |= contains_inherited_metadata_value(child); + return; + } + + auto child_result = + extract_variant_typed_nested_column_ids(child, child_paths, column_ids, mode); + result->has_child_columns |= child_result.has_child_columns; + result->needs_metadata |= child_result.needs_metadata; +} + +VariantColumnIdExtractionResult extract_shredded_variant_field_ids( + const FieldSchema& shredded_field, const std::vector<std::vector<std::string>>& paths, + std::set<uint64_t>& column_ids, NestedPathMode mode) { + const auto* typed_value = find_child_by_structural_name(shredded_field, "typed_value"); + VariantColumnIdExtractionResult result; + + for (const auto& raw_path : paths) { + std::vector<std::string> stripped_path; + const auto& path = effective_variant_path(raw_path, stripped_path); + if (path.empty()) { + add_column_id_range(shredded_field, column_ids); + result.has_child_columns = true; + result.needs_metadata |= contains_inherited_metadata_value(shredded_field); + continue; + } + + VariantColumnIdExtractionResult typed_result; + if (typed_value != nullptr) { + typed_result = extract_typed_value_path(*typed_value, path, column_ids, mode); + result.needs_metadata |= typed_result.needs_metadata; + } + if (!typed_result.has_child_columns) { Review Comment: This only keeps the field-level residual `value` when no typed child was selected at all. For a shredded field that has both `value` and a nested typed subpath, e.g. `typed_value.metric { value, typed_value: struct<x:int> }`, an access to `v['metric']['x']` selects `metric.typed_value.x` and then skips `metric.value`. But rows whose `metric` value is stored in the field-level residual object can still contain `x`; those rows need `metric.value` plus inherited metadata to evaluate the same subpath. The new regression currently asserts that `v.typed_value.metric.value` is not read for `metric.x`, so this mixed typed/residual case would silently return NULL/missing for residual rows. This is distinct from the existing deeper-path thread where the typed child was absent and the fallback selected the wrong top-level `v.value`; here the typed child exists, but the required field-level residual is pruned away. Please keep the shredded field residual whenever a deeper subpath is requested under a field that has a residual `value`, and add coverage where some rows satisfy `metric.x` through `metric.typed_value.x` and others through `metric.value`. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
