eldenmoon commented on code in PR #66910:
URL: https://github.com/apache/doris/pull/66910#discussion_r3819716613
##########
be/src/format_v2/parquet/reader/variant_column_reader.cpp:
##########
@@ -717,6 +815,380 @@ bool find_materialized_path(VariantRef current,
std::span<const VariantShreddedP
return true;
}
+bool find_materialized_path(VariantRef current, std::span<const
VariantShreddedPathSegment> path,
+ VariantRef* output) {
+ return find_materialized_path_impl(
+ current, path,
+ [](VariantRef object, StringRef key, size_t, VariantRef* found) {
+ return object.object_find(key, found);
+ },
+ output);
+}
+
+bool find_materialized_path_with_index(VariantRef current, uint32_t
dictionary_id,
+ const UnshreddedMetadataIndex&
metadata_index,
+ std::span<const
VariantShreddedPathSegment> path,
+ DorisVector<int64_t>&
resolved_field_ids,
+ VariantRef* output) {
+ DCHECK_LT(dictionary_id, metadata_index.dictionaries.size());
+ DCHECK_EQ(resolved_field_ids.size(), metadata_index.dictionaries.size() *
path.size());
+ constexpr int64_t UNRESOLVED_FIELD_ID = -2;
+ return find_materialized_path_impl(
+ current, path,
+ [&](VariantRef object, StringRef key, size_t position, VariantRef*
found) {
+ int64_t& field_id = resolved_field_ids[dictionary_id *
path.size() + position];
+ bool layout_validated = false;
+ if (field_id == UNRESOLVED_FIELD_ID) {
+ // object_find() validates the object layout before
consulting metadata.
+ static_cast<void>(object.num_elements());
+ layout_validated = true;
+ field_id =
metadata_index.dictionaries[dictionary_id].find_key(key);
+ }
+ if (field_id < 0) {
+ // A cached metadata miss must not hide a corrupt object
in a later row.
+ if (!layout_validated) {
+ static_cast<void>(object.num_elements());
+ }
+ return false;
+ }
+ return
object.object_find_by_id(static_cast<uint32_t>(field_id), found);
+ },
+ output);
+}
+
+struct UnshreddedPathScan {
+ MutableColumnPtr outer_nulls;
+ MutableColumnPtr typed_values;
+ DataTypePtr typed_type;
+ std::shared_ptr<const UnshreddedPathCache> path_cache;
+ int64_t copied_bytes = 0;
+};
+
+enum class UnshreddedTypedKind : uint8_t { UNKNOWN, STRING, INTEGER,
UNSUPPORTED };
+
+class UnshreddedTypedValueBuilder {
+public:
+ UnshreddedTypedValueBuilder(size_t rows, const UnshreddedMetadataIndex&
metadata_index)
+ : _rows(rows),
+ _metadata_index(metadata_index),
+ _inner_nulls(ColumnUInt8::create()),
+ _result_nulls(ColumnUInt8::create()),
+ _validated_metadata(metadata_index.dictionaries.size(), 0) {
+ _inner_nulls->reserve(rows);
+ _result_nulls->reserve(rows);
+ }
+
+ void append_outer_null() { append_null(1); }
+
+ void append_json_null(uint32_t dictionary_id) {
+ if (_typed_kind == UnshreddedTypedKind::UNKNOWN) {
+ _pending_json_null_dictionaries.push_back(dictionary_id);
+ } else if (_typed_kind == UnshreddedTypedKind::INTEGER &&
+ !validate_integer_metadata(dictionary_id)) {
+ mark_unsupported();
+ }
+ append_null(0);
+ }
+
+ void append_scalar(const VariantRef& found, uint32_t dictionary_id, size_t
row) {
+ if (_typed_kind == UnshreddedTypedKind::UNSUPPORTED) {
+ append_null(0);
+ return;
+ }
+
+ const VariantBasicType basic_type = found.basic_type();
+ const bool is_string = basic_type == VariantBasicType::SHORT_STRING ||
+ (basic_type == VariantBasicType::PRIMITIVE &&
+ found.primitive_id() ==
VariantPrimitiveId::STRING);
+ if (is_string) {
+ if (!prepare(UnshreddedTypedKind::STRING, row)) {
+ append_null(0);
+ return;
+ }
+ const StringRef string = found.get_string();
Review Comment:
写入的 parquet 数据是保证合法的，加入校验会导致性能问题
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
