This is an automated email from the ASF dual-hosted git repository.
MaxGekk pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 6ccdd98e4428 [SPARK-57634][SQL] Support TIME type inference in XML
data source
6ccdd98e4428 is described below
commit 6ccdd98e4428626bb8291b216a68cf7d06e3a7e9
Author: Shrirang Mhalgi <[email protected]>
AuthorDate: Thu Jun 25 10:31:08 2026 +0200
[SPARK-57634][SQL] Support TIME type inference in XML data source
### What changes were proposed in this pull request?
Add TIME type inference to `XmlInferSchema`, completing format parity with
CSV and JSON data sources (SPARK-57572).
### Why are the changes needed?
After SPARK-57572 added TIME inference to CSV and JSON, XML was the only
text-based format without TIME type inference. This was explicitly noted as a
follow-up in that PR's review.
### Does this PR introduce _any_ user-facing change?
Yes. When `spark.sql.timeType.enabled=true` (default), XML values matching
the time format (e.g. `13:31:24.123456`) are now inferred as `TimeType` instead
of `TimestampType`
### How was this patch tested?
Added 4 tests to XmlInferSchemaSuite:
- Basic `TIME` inference
- Disabled when `timeType.enabled=false`
- Negative cases (date/timestamp not misclassified)
- Cross-row merge (`TIME + TIME`, `TIME + string`)
### Was this patch authored or co-authored using generative AI tooling?
Co-Authored using Claude Opus 4.6
Closes #56697 from shrirangmhalgi/SPARK-57634-xml-time-type-inference.
Authored-by: Shrirang Mhalgi <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
(cherry picked from commit 5c6c6f40220d9f0d26c7ae69c5e64792707e743f)
Signed-off-by: Max Gekk <[email protected]>
---
.../spark/sql/catalyst/xml/XmlInferSchema.scala | 11 ++++-
.../sql-tests/results/xml-functions.sql.out | 4 +-
.../datasources/xml/XmlInferSchemaSuite.scala | 56 +++++++++++++++++++++-
3 files changed, 67 insertions(+), 4 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
index a0139e98a266..d6a686659589 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
@@ -38,7 +38,7 @@ import org.apache.spark.internal.Logging
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.analysis.TypeCoercion
import org.apache.spark.sql.catalyst.expressions.ExprUtils
-import org.apache.spark.sql.catalyst.util.{DateFormatter, DropMalformedMode,
FailFastMode, ParseMode, PermissiveMode, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.{DateFormatter, DropMalformedMode,
FailFastMode, ParseMode, PermissiveMode, TimeFormatter, TimestampFormatter}
import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf}
@@ -73,6 +73,10 @@ class XmlInferSchema(private val options: XmlOptions,
private val caseSensitive:
legacyFormat = FAST_DATE_FORMAT,
isParsing = true)
+ private lazy val timeFormatter = TimeFormatter(options.timeFormatInRead,
isParsing = true)
+
+ private val isTimeTypeEnabled = SQLConf.get.isTimeTypeEnabled
+
override def equals(obj: Any): Boolean = obj match {
case other: XmlInferSchema =>
options == other.options &&
@@ -324,6 +328,7 @@ class XmlInferSchema(private val options: XmlOptions,
private val caseSensitive:
case v if options.prefersDecimal && decimalTry.isDefined =>
decimalTry.get
case v if isDouble(v) => DoubleType
case v if isBoolean(v) => BooleanType
+ case v if isTimeTypeEnabled && isTime(v) =>
TimeType(TimeType.DEFAULT_PRECISION)
case v if isDate(v) => DateType
case v if timestampNTZTry.isDefined => timestampNTZTry.get
case v if isTimestamp(v) => TimestampType
@@ -549,6 +554,10 @@ class XmlInferSchema(private val options: XmlOptions,
private val caseSensitive:
(allCatch opt dateFormatter.parse(value)).isDefined
}
+ private def isTime(value: String): Boolean = {
+ (allCatch opt timeFormatter.parse(value)).isDefined
+ }
+
/**
* Convert NullType to StringType and remove StructTypes with no fields
*/
diff --git
a/sql/core/src/test/resources/sql-tests/results/xml-functions.sql.out
b/sql/core/src/test/resources/sql-tests/results/xml-functions.sql.out
index eaac972cb2fe..732d1f9e6971 100644
--- a/sql/core/src/test/resources/sql-tests/results/xml-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/xml-functions.sql.out
@@ -636,7 +636,7 @@ select
schema_of_xml('<record><time>14:30:45</time></record>', map('rowTag', 're
-- !query schema
struct<schema_of_xml(<record><time>14:30:45</time></record>):string>
-- !query output
-STRUCT<time: TIMESTAMP>
+STRUCT<time: TIME(6)>
-- !query
@@ -644,7 +644,7 @@ select
schema_of_xml('<record><time>14:30:45.123456</time></record>', map('rowTa
-- !query schema
struct<schema_of_xml(<record><time>14:30:45.123456</time></record>):string>
-- !query output
-STRUCT<time: TIMESTAMP>
+STRUCT<time: TIME(6)>
-- !query
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlInferSchemaSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlInferSchemaSuite.scala
index b013b56ccd75..bd675bd62510 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlInferSchemaSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlInferSchemaSuite.scala
@@ -28,13 +28,16 @@ import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{
ArrayType,
BooleanType,
+ DateType,
DecimalType,
DoubleType,
IntegerType,
LongType,
StringType,
StructField,
- StructType
+ StructType,
+ TimestampType,
+ TimeType
}
class XmlInferSchemaSuite
@@ -646,6 +649,57 @@ class XmlInferSchemaSuite
assert(xmlDF.schema === expectedSchema)
checkAnswer(xmlDF, expectedAns)
}
+
+ test("TIME type inference") {
+ val xmlString = Seq("""<ROW><t>13:31:24.123456</t></ROW>""")
+ val df = readData(xmlString)
+ assert(df.schema === new StructType().add("t",
TimeType(TimeType.DEFAULT_PRECISION)))
+ checkAnswer(df, Row(java.time.LocalTime.of(13, 31, 24, 123456000)))
+ }
+
+ test("TIME type inference - disabled when timeType.enabled is false") {
+ withSQLConf(SQLConf.TIME_TYPE_ENABLED.key -> "false") {
+ val xmlString = Seq("""<ROW><t>13:31:24</t></ROW>""")
+ val df = readData(xmlString)
+ // Falls through to TimestampType when TIME inference is disabled
+ assert(df.schema.fields.head.dataType === TimestampType)
+ }
+ }
+
+ test("TIME type inference - negative cases") {
+ // Date strings should not infer as TIME
+ val xmlDate = Seq("""<ROW><t>2024-01-15</t></ROW>""")
+ val dfDate = readData(xmlDate)
+ assert(dfDate.schema.fields.head.dataType === DateType)
+
+ // Timestamp strings should not infer as TIME
+ val xmlTs = Seq("""<ROW><t>2024-01-15T13:31:24</t></ROW>""")
+ val dfTs = readData(xmlTs)
+ assert(dfTs.schema.fields.head.dataType !=
TimeType(TimeType.DEFAULT_PRECISION))
+ }
+
+ test("TIME type inference - cross-row merge") {
+ // TIME + TIME -> TIME
+ val xmlTime = Seq(
+ """<ROW><t>13:31:24</t></ROW>""",
+ """<ROW><t>09:15:00.123</t></ROW>""")
+ val dfTime = readData(xmlTime)
+ assert(dfTime.schema === new StructType().add("t",
TimeType(TimeType.DEFAULT_PRECISION)))
+
+ // TIME + non-time string -> StringType
+ val xmlMixed = Seq(
+ """<ROW><t>13:31:24</t></ROW>""",
+ """<ROW><t>not-a-time</t></ROW>""")
+ val dfMixed = readData(xmlMixed)
+ assert(dfMixed.schema.fields.head.dataType === StringType)
+
+ // TIME + Date -> StringType (incompatible types widen)
+ val xmlTimeDate = Seq(
+ """<ROW><t>13:31:24</t></ROW>""",
+ """<ROW><t>2024-01-15</t></ROW>""")
+ val dfTimeDate = readData(xmlTimeDate)
+ assert(dfTimeDate.schema.fields.head.dataType === StringType)
+ }
}
trait XmlSchemaInferenceCaseSensitivityTests extends QueryTest {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
