rok commented on code in PR #13901:
URL: https://github.com/apache/arrow/pull/13901#discussion_r1753555791
##########
cpp/src/parquet/arrow/arrow_schema_test.cc:
##########
@@ -724,6 +727,88 @@ TEST_F(TestConvertParquetSchema,
ParquetRepeatedNestedSchema) {
ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(arrow_schema));
}
+Status ArrowSchemaToParquetMetadata(std::shared_ptr<::arrow::Schema>&
arrow_schema,
+ std::shared_ptr<KeyValueMetadata>&
metadata) {
+ ARROW_ASSIGN_OR_RAISE(
+ std::shared_ptr<Buffer> serialized,
+ ::arrow::ipc::SerializeSchema(*arrow_schema,
::arrow::default_memory_pool()));
+ std::string schema_as_string = serialized->ToString();
+ std::string schema_base64 = ::arrow::util::base64_encode(schema_as_string);
+ metadata = ::arrow::key_value_metadata({"ARROW:schema"}, {schema_base64});
+ return Status::OK();
+}
+
+TEST_F(TestConvertParquetSchema, ParquetSchemaArrowExtensions) {
+ std::vector<NodePtr> parquet_fields;
+ parquet_fields.push_back(PrimitiveNode::Make(
+ "json_1", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY,
ConvertedType::JSON));
+ parquet_fields.push_back(PrimitiveNode::Make(
+ "json_2", Repetition::OPTIONAL, ParquetType::BYTE_ARRAY,
ConvertedType::JSON));
+
+ {
+ // Parquet file does not contain Arrow schema.
+ // By default, both fields should be treated as utf8() fields in Arrow.
+ auto arrow_schema = ::arrow::schema(
+ {::arrow::field("json_1", UTF8, true), ::arrow::field("json_2", UTF8,
true)});
+ std::shared_ptr<KeyValueMetadata> metadata{};
+ ASSERT_OK(ConvertSchema(parquet_fields, metadata));
+ CheckFlatSchema(arrow_schema);
+ }
+
+ {
+ // Parquet file does not contain Arrow schema.
+ // If Arrow extensions are enabled, both fields should be treated as
json() extension
+ // fields.
+ ArrowReaderProperties props;
+ props.set_arrow_extensions_enabled(true);
+ auto arrow_schema = ::arrow::schema(
+ {::arrow::field("json_1", ::arrow::extension::json(), true),
+ ::arrow::field("json_2",
::arrow::extension::json(::arrow::large_utf8()),
+ true)});
+ std::shared_ptr<KeyValueMetadata> metadata{};
+ ASSERT_OK(ConvertSchema(parquet_fields, metadata, props));
+ CheckFlatSchema(arrow_schema);
+ }
+
+ {
+ // Parquet file contains Arrow schema.
+ // Parquet logical type has precedence. Both json_1 and json_2 should be
returned
+ // as a json() field even though extensions are not enabled.
+ ArrowReaderProperties props;
+ props.set_arrow_extensions_enabled(false);
+ std::shared_ptr<KeyValueMetadata> field_metadata =
+ ::arrow::key_value_metadata({"foo", "bar"}, {"biz", "baz"});
+ auto arrow_schema = ::arrow::schema(
+ {::arrow::field("json_1", ::arrow::extension::json(), true,
field_metadata),
+ ::arrow::field("json_2",
::arrow::extension::json(::arrow::large_utf8()),
+ true)});
+
+ std::shared_ptr<KeyValueMetadata> metadata;
+ ASSERT_OK(ArrowSchemaToParquetMetadata(arrow_schema, metadata));
+ ASSERT_OK(ConvertSchema(parquet_fields, metadata, props));
+ CheckFlatSchema(arrow_schema, true /* check_metadata */);
+ }
+
+ {
+ // Parquet file contains Arrow schema.
+ // A contrived example. Parquet believes both columns are JSON. Arrow
believes json_1
Review Comment:
Updated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
