github-actions[bot] commented on code in PR #66910:
URL: https://github.com/apache/doris/pull/66910#discussion_r3819557305
##########
be/src/format_v2/parquet/reader/variant_column_reader.cpp:
##########
@@ -717,6 +815,380 @@ bool find_materialized_path(VariantRef current,
std::span<const VariantShreddedP
return true;
}
+bool find_materialized_path(VariantRef current, std::span<const
VariantShreddedPathSegment> path,
+ VariantRef* output) {
+ return find_materialized_path_impl(
+ current, path,
+ [](VariantRef object, StringRef key, size_t, VariantRef* found) {
+ return object.object_find(key, found);
+ },
+ output);
+}
+
+bool find_materialized_path_with_index(VariantRef current, uint32_t
dictionary_id,
+ const UnshreddedMetadataIndex&
metadata_index,
+ std::span<const
VariantShreddedPathSegment> path,
+ DorisVector<int64_t>&
resolved_field_ids,
+ VariantRef* output) {
+ DCHECK_LT(dictionary_id, metadata_index.dictionaries.size());
+ DCHECK_EQ(resolved_field_ids.size(), metadata_index.dictionaries.size() *
path.size());
+ constexpr int64_t UNRESOLVED_FIELD_ID = -2;
+ return find_materialized_path_impl(
+ current, path,
+ [&](VariantRef object, StringRef key, size_t position, VariantRef*
found) {
+ int64_t& field_id = resolved_field_ids[dictionary_id *
path.size() + position];
+ bool layout_validated = false;
+ if (field_id == UNRESOLVED_FIELD_ID) {
+ // object_find() validates the object layout before
consulting metadata.
+ static_cast<void>(object.num_elements());
+ layout_validated = true;
+ field_id =
metadata_index.dictionaries[dictionary_id].find_key(key);
+ }
+ if (field_id < 0) {
+ // A cached metadata miss must not hide a corrupt object
in a later row.
+ if (!layout_validated) {
+ static_cast<void>(object.num_elements());
+ }
+ return false;
+ }
+ return
object.object_find_by_id(static_cast<uint32_t>(field_id), found);
+ },
+ output);
+}
+
+struct UnshreddedPathScan {
+ MutableColumnPtr outer_nulls;
+ MutableColumnPtr typed_values;
+ DataTypePtr typed_type;
+ std::shared_ptr<const UnshreddedPathCache> path_cache;
+ int64_t copied_bytes = 0;
+};
+
+enum class UnshreddedTypedKind : uint8_t { UNKNOWN, STRING, INTEGER,
UNSUPPORTED };
+
+class UnshreddedTypedValueBuilder {
+public:
+ UnshreddedTypedValueBuilder(size_t rows, const UnshreddedMetadataIndex&
metadata_index)
+ : _rows(rows),
+ _metadata_index(metadata_index),
+ _inner_nulls(ColumnUInt8::create()),
+ _result_nulls(ColumnUInt8::create()),
+ _validated_metadata(metadata_index.dictionaries.size(), 0) {
+ _inner_nulls->reserve(rows);
+ _result_nulls->reserve(rows);
+ }
+
+ void append_outer_null() { append_null(1); }
+
+ void append_json_null(uint32_t dictionary_id) {
+ if (_typed_kind == UnshreddedTypedKind::UNKNOWN) {
+ _pending_json_null_dictionaries.push_back(dictionary_id);
+ } else if (_typed_kind == UnshreddedTypedKind::INTEGER &&
+ !validate_integer_metadata(dictionary_id)) {
+ mark_unsupported();
+ }
+ append_null(0);
+ }
+
+ void append_scalar(const VariantRef& found, uint32_t dictionary_id, size_t
row) {
+ if (_typed_kind == UnshreddedTypedKind::UNSUPPORTED) {
+ append_null(0);
+ return;
+ }
+
+ const VariantBasicType basic_type = found.basic_type();
+ const bool is_string = basic_type == VariantBasicType::SHORT_STRING ||
+ (basic_type == VariantBasicType::PRIMITIVE &&
+ found.primitive_id() ==
VariantPrimitiveId::STRING);
+ if (is_string) {
+ if (!prepare(UnshreddedTypedKind::STRING, row)) {
+ append_null(0);
+ return;
+ }
+ const StringRef string = found.get_string();
Review Comment:
[P1] Preserve corruption checks before publishing direct leaves
This direct STRING branch copies the selected bytes after `get_string()` has
checked only their framing; it never runs the canonical payload validator, so
invalid UTF-8 in the requested string is returned successfully. The same
shortcut can return a valid requested field (or a missing-path NULL) while
another child or trailing region in the same root is malformed. Before this
optimization, unshredded extraction materialized through
`EncodedRowsAppender::append()`, which validates every metadata dictionary and
recursively validates every complete root, so those rows fail instead of losing
the corrupt source once a typed result is published. Please preserve/cache
equivalent metadata and full-root validation before accepting a direct result,
and add differential cases for a selected invalid-UTF-8 string, an absent path,
and an unvisited truncated child/trailing region.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
