srielau commented on code in PR #58545:
URL: https://github.com/apache/spark/pull/58545#discussion_r3969367075
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -310,27 +314,27 @@ class StaxXmlParser(
startElementName: String,
attributes: Array[Attribute]): Any = dt match {
case st: StructType => convertObject(parser, st)
- case MapType(StringType, vt, _) => convertMap(parser, vt, attributes)
+ case MapType(kt: StringType, vt, _) => convertMap(parser, kt, vt,
attributes)
case ArrayType(st, _) => convertField(parser, st, startElementName)
case VariantType =>
StaxXmlParser.convertVariant(parser, attributes, options)
- case _: StringType =>
+ case dt: StringType =>
convertTo(
StaxXmlParserUtils.currentStructureAsString(
parser, startElementName, options),
- StringType)
+ dt)
}
(parser.peek, dataType) match {
case (_: StartElement, dt: DataType) =>
convertComplicatedType(dt, startElementName, attributes)
- case (_: EndElement, _: StringType) =>
+ case (_: EndElement, dt: StringType) =>
StaxXmlParserUtils.skipNextEndElement(parser, startElementName,
options)
// Empty. It's null if "" is the null value
if (options.nullValue == "") {
null
} else {
- UTF8String.fromString("")
+ CharVarcharUtils.applyTextParseSemantics(UTF8String.fromString(""),
dt)
Review Comment:
Added in 32154c7572d. The focused test now verifies that an empty XML
element with nullValue set to a different sentinel produces exactly five spaces
for CHAR(5). The focused SPARK-59274 tests pass.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/StaxXmlParser.scala:
##########
@@ -374,31 +378,41 @@ class StaxXmlParser(
*/
private def convertMap(
parser: XMLEventReader,
+ keyType: DataType,
valueType: DataType,
attributes: Array[Attribute]): MapData = {
val kvPairs = ArrayBuffer.empty[(UTF8String, Any)]
+ def mapKey(raw: String): UTF8String = {
+ CharVarcharUtils.applyTextParseSemantics(UTF8String.fromString(raw),
keyType)
+ }
attributes.foreach { attr =>
- kvPairs += (UTF8String.fromString(options.attributePrefix +
attr.getName.getLocalPart)
- -> convertTo(attr.getValue, valueType))
+ kvPairs += (mapKey(options.attributePrefix + attr.getName.getLocalPart)
->
+ convertTo(attr.getValue, valueType))
}
var shouldStop = false
while (!shouldStop) {
parser.nextEvent match {
case e: StartElement =>
val key = StaxXmlParserUtils.getName(e.asStartElement.getName,
options)
- kvPairs +=
- (UTF8String.fromString(key) -> convertField(parser, valueType, key))
+ kvPairs += (mapKey(key) -> convertField(parser, valueType, key))
Review Comment:
Fixed in 32154c7572d. XML map values are now consumed before constrained-key
normalization, and key/value pairs are appended atomically. A key conversion
failure is retained until the enclosing map has been consumed, so permissive
row recovery resumes at the following sibling. Added the reported oversized
CHAR key plus tail-field regression. The focused SPARK-59274 tests pass.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
