uros-b commented on code in PR #56887:
URL: https://github.com/apache/spark/pull/56887#discussion_r3496940785
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2RelationSuite.scala:
##########
@@ -107,4 +112,31 @@ class DataSourceV2RelationSuite extends SparkFunSuite {
val idV2NoHist = colStats.get(FieldReference.column("id"))
assert(!idV2NoHist.histogram().isPresent)
}
+
+ test("create strips leaked internal metadata but preserves column IDs") {
+ // A column carrying both a column ID (surfaced on purpose) and a leaked
internal
+ // metadata key (METADATA_COL_ATTR_KEY, listed in INTERNAL_METADATA_KEYS).
+ val leakedMetadata = new MetadataBuilder()
+ .putString(METADATA_COL_ATTR_KEY, "leaked")
+ .build()
+ val table = new Table {
+ override def name(): String = "t"
+ override def columns(): Array[Column] = Array(
+ Column.builderFor("id", IntegerType)
+ .id("1")
+ .metadata(leakedMetadata.json)
+ .build())
+ override def capabilities(): util.Set[TableCapability] =
+ util.Collections.emptySet[TableCapability]()
+ }
+
+ val relation =
+ DataSourceV2Relation.create(table, None, None,
CaseInsensitiveStringMap.empty())
+ val field = relation.schema.head
+
+ // The leaked internal metadata key is stripped from the relation output
...
+ assert(!field.metadata.contains(METADATA_COL_ATTR_KEY))
Review Comment:
Nit: the test exercises one leaked key (METADATA_COL_ATTR_KEY) and one
preserved key (FIELD_ID_METADATA_KEY); the other INTERNAL_METADATA_KEYS
(QUALIFIED_ACCESS_ONLY, AUTO_GENERATED_ALIAS, PRESERVE_ON_*) are not separately
asserted.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
