This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new ad80f662b82e [SPARK-57572][SQL] Infer TimeType during CSV and JSON
schema inference
ad80f662b82e is described below
commit ad80f662b82eee42a462ec34a50c4180ff322d16
Author: Shrirang Mhalgi <[email protected]>
AuthorDate: Tue Jun 23 07:29:15 2026 +0900
[SPARK-57572][SQL] Infer TimeType during CSV and JSON schema inference
### What changes were proposed in this pull request?
Add `TimeType` inference to `CSVInferSchema` and `JsonInferSchema`,
following the existing `DateType/TimestampType` inference pattern. A
`tryParseTime` step is inserted in the CSV type ladder between `DoubleType` and
`DateType`, and an equivalent check is added to the JSON string inference path.
Both are gated behind `spark.sql.timeType.enabled` (default false).
### Why are the changes needed?
`CSVInferSchema` and `JsonInferSchema` infer `DateType` /
`TimestampNTZType` / `TimestampType` but never `TimeType`. With an explicit
schema, TIME read/write already works; only auto-inference is missing.
Time-only columns (e.g., `12:13:14`) currently infer as `StringType` even when
the TIME type feature is enabled.
### Does this PR introduce _any_ user-facing change?
Yes. When `spark.sql.timeType.enabled` is set to `true`, `CSV` and `JSON`
schema inference will recognize time-only string values (e.g., `12:13:14`,
`23:59:59.123456`) as `TimeType` instead of `StringType`. When disabled
(default), behavior is unchanged.
### How was this patch tested?
New tests in CSVInferSchemaSuite and JsonInferSchemaSuite:
- Time strings infer as `TimeType` when `spark.sql.timeType.enabled = true`
- Time strings do NOT infer as `TimeType` when disabled (existing behavior
preserved)
- `Date` / `timestamp` strings are not misclassified as time
- All existing + new tests pass for both CSV and JSON
### Was this patch authored or co-authored using generative AI tooling?
Co-Authored using Claude Opus 4.6
Closes #56634 from shrirangmhalgi/SPARK-57572-time-type-csv-json-inference.
Authored-by: Shrirang Mhalgi <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../spark/sql/catalyst/csv/CSVInferSchema.scala | 19 +++++++-
.../spark/sql/catalyst/json/JsonInferSchema.scala | 14 ++++--
.../sql/catalyst/csv/CSVInferSchemaSuite.scala | 51 ++++++++++++++++++++++
.../sql/catalyst/json/JsonInferSchemaSuite.scala | 43 ++++++++++++++++++
.../sql-tests/results/csv-functions.sql.out | 4 +-
.../sql/execution/datasources/csv/CSVSuite.scala | 20 ++++++++-
.../sql/execution/datasources/json/JsonSuite.scala | 15 +++++++
7 files changed, 157 insertions(+), 9 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
index 0cc11ce6bb89..48871764aa45 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
@@ -25,7 +25,7 @@ import org.apache.spark.SparkException
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.analysis.TypeCoercion
import org.apache.spark.sql.catalyst.expressions.ExprUtils
-import org.apache.spark.sql.catalyst.util.{DateFormatter, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.{DateFormatter, TimeFormatter,
TimestampFormatter}
import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
@@ -52,6 +52,12 @@ class CSVInferSchema(val options: CSVOptions) extends
Serializable {
legacyFormat = FAST_DATE_FORMAT,
isParsing = true)
+ private lazy val timeFormatter = TimeFormatter(
+ options.timeFormatInRead,
+ isParsing = true)
+
+ private val isTimeTypeEnabled = SQLConf.get.isTimeTypeEnabled
+
private val decimalParser = if (options.locale == Locale.US) {
// Special handling the default locale for backward compatibility
s: String => new java.math.BigDecimal(s)
@@ -140,6 +146,7 @@ class CSVInferSchema(val options: CSVOptions) extends
Serializable {
case LongType => tryParseLong(field)
case _: DecimalType => tryParseDecimal(field)
case DoubleType => tryParseDouble(field)
+ case _: TimeType => tryParseTime(field)
case DateType => tryParseDate(field)
case TimestampNTZType => tryParseTimestampNTZ(field)
case TimestampType => tryParseTimestamp(field)
@@ -194,12 +201,20 @@ class CSVInferSchema(val options: CSVOptions) extends
Serializable {
if ((allCatch opt field.toDouble).isDefined || isInfOrNan(field)) {
DoubleType
} else if (options.preferDate) {
- tryParseDate(field)
+ tryParseTime(field)
} else {
tryParseTimestampNTZ(field)
}
}
+ private def tryParseTime(field: String): DataType = {
+ if (isTimeTypeEnabled && (allCatch opt
timeFormatter.parse(field)).isDefined) {
+ TimeType(TimeType.DEFAULT_PRECISION)
+ } else {
+ tryParseDate(field)
+ }
+ }
+
private def tryParseDate(field: String): DataType = {
if ((allCatch opt dateFormatter.parse(field)).isDefined) {
DateType
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
index 5587bb3d30f0..9614747ffa70 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
@@ -60,6 +60,8 @@ class JsonInferSchema(private val options: JSONOptions)
extends Serializable wit
private val ignoreMissingFiles = options.ignoreMissingFiles
private val isDefaultNTZ = SQLConf.get.timestampType == TimestampNTZType
private val legacyMode = SQLConf.get.legacyTimeParserPolicy ==
LegacyBehaviorPolicy.LEGACY
+ private val isTimeTypeEnabled = SQLConf.get.isTimeTypeEnabled
+ private lazy val timeFormatter = TimeFormatter(options.timeFormatInRead,
isParsing = true)
override def equals(obj: Any): Boolean = obj match {
case other: JsonInferSchema => options == other.options
@@ -177,10 +179,14 @@ class JsonInferSchema(private val options: JSONOptions)
extends Serializable wit
if (options.prefersDecimal && decimalTry.isDefined) {
decimalTry.get
} else if (options.inferTimestamp) {
- // For text-based format, it's ambiguous to infer a timestamp string
without timezone, as
- // it can be both TIMESTAMP LTZ and NTZ. To avoid behavior changes
with the new support
- // of NTZ, here we only try to infer NTZ if the config is set to use
NTZ by default.
- if (isDefaultNTZ &&
+ if (isTimeTypeEnabled &&
+ (allCatch opt timeFormatter.parse(field)).isDefined) {
+ TimeType(TimeType.DEFAULT_PRECISION)
+ // For text-based format, it's ambiguous to infer a timestamp string
without
+ // timezone, as it can be both TIMESTAMP LTZ and NTZ. To avoid
behavior changes
+ // with the new support of NTZ, here we only try to infer NTZ if the
config is
+ // set to use NTZ by default.
+ } else if (isDefaultNTZ &&
timestampNTZFormatter.parseWithoutTimeZoneOptional(field,
false).isDefined) {
TimestampNTZType
} else if (timestampFormatter.parseOptional(field).isDefined) {
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
index fb91200557a6..871bcb1ca718 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
@@ -273,4 +273,55 @@ class CSVInferSchemaSuite extends SparkFunSuite with
SQLHelper {
val inferSchema = new CSVInferSchema(options)
assert(inferSchema.inferField(NullType, "2884-06-24T02:45:51.138") ==
StringType)
}
+
+ test("SPARK-57572: infer TimeType when timeType.enabled is true") {
+ withSQLConf(SQLConf.TIME_TYPE_ENABLED.key -> "true") {
+ val options = new CSVOptions(Map.empty[String, String],
+ columnPruning = false, defaultTimeZoneId = "UTC")
+ val inferSchema = new CSVInferSchema(options)
+
+ // Basic time inference
+ assert(inferSchema.inferField(NullType, "12:13:14") ==
TimeType(TimeType.DEFAULT_PRECISION))
+ assert(inferSchema.inferField(NullType, "23:59:59") ==
TimeType(TimeType.DEFAULT_PRECISION))
+ assert(inferSchema.inferField(NullType, "00:00:00") ==
TimeType(TimeType.DEFAULT_PRECISION))
+
+ // Time with fractional seconds
+ assert(inferSchema.inferField(NullType, "12:13:14.123") ==
+ TimeType(TimeType.DEFAULT_PRECISION))
+
+ // Not a time -- should fall through to other types
+ assert(inferSchema.inferField(NullType, "not-a-time") == StringType)
+
+ // Date should NOT be inferred as time
+ assert(inferSchema.inferField(NullType, "2023-01-01") == DateType)
+ }
+ }
+
+ test("SPARK-57572: TimeType not inferred when timeType.enabled is false") {
+ withSQLConf(SQLConf.TIME_TYPE_ENABLED.key -> "false") {
+ val options = new CSVOptions(Map.empty[String, String],
+ columnPruning = false, defaultTimeZoneId = "UTC")
+ val inferSchema = new CSVInferSchema(options)
+
+ // When disabled, a time-only string is not inferred as TimeType; the
lenient timestamp
+ // parser accepts it (using the current date), so it infers as
TimestampType.
+ assert(inferSchema.inferField(NullType, "12:13:14") == TimestampType)
+ }
+ }
+
+ test("SPARK-57572: TimeType cross-row merge via compatibleType") {
+ withSQLConf(SQLConf.TIME_TYPE_ENABLED.key -> "true") {
+ val options = new CSVOptions(Map.empty[String, String],
+ columnPruning = false, defaultTimeZoneId = "UTC")
+ val inferSchema = new CSVInferSchema(options)
+
+ val timeType = TimeType(TimeType.DEFAULT_PRECISION)
+ // Two time values merge to TimeType
+ assert(inferSchema.inferField(timeType, "23:59:59") == timeType)
+ // Time + non-time merges to StringType
+ assert(inferSchema.inferField(timeType, "not-a-time") == StringType)
+ // Time + Date merges to StringType
+ assert(inferSchema.inferField(timeType, "2023-01-01") == StringType)
+ }
+ }
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
index 81a4858dce82..f6fa4515f07a 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
@@ -120,4 +120,47 @@ class JsonInferSchemaSuite extends SparkFunSuite with
SQLHelper {
"""{"a": "2884-06-24T02:45:51.138"}""",
StringType)
}
+
+ test("SPARK-57572: infer TimeType when timeType.enabled is true") {
+ withSQLConf(SQLConf.TIME_TYPE_ENABLED.key -> "true") {
+ checkType(Map("inferTimestamp" -> "true"), """{"a": "12:13:14"}""",
+ TimeType(TimeType.DEFAULT_PRECISION))
+ checkType(Map("inferTimestamp" -> "true"), """{"a":
"23:59:59.123456"}""",
+ TimeType(TimeType.DEFAULT_PRECISION))
+ // Negative: date and timestamp strings should NOT infer as TimeType
+ checkType(Map("inferTimestamp" -> "true"), """{"a": "2023-01-01"}""",
TimestampType)
+ checkType(Map("inferTimestamp" -> "true"),
+ """{"a": "2024-06-24T02:45:51.138"}""", TimestampType)
+ }
+ }
+
+ test("SPARK-57572: TimeType not inferred when timeType.enabled is false") {
+ withSQLConf(SQLConf.TIME_TYPE_ENABLED.key -> "false") {
+ // With inferTimestamp, time-only strings fall through to timestamp
(lenient parser)
+ checkType(Map("inferTimestamp" -> "true"), """{"a": "12:13:14"}""",
TimestampType)
+ // Without inferTimestamp, time-only strings become StringType
+ checkType(Map.empty[String, String], """{"a": "12:13:14"}""", StringType)
+ }
+ }
+
+ test("SPARK-57572: TimeType not inferred without inferTimestamp") {
+ withSQLConf(SQLConf.TIME_TYPE_ENABLED.key -> "true") {
+ // inferTimestamp defaults to false; time inference requires it
+ checkType(Map.empty[String, String], """{"a": "12:13:14"}""", StringType)
+ }
+ }
+
+ test("SPARK-57572: TimeType cross-row merge via compatibleType") {
+ withSQLConf(SQLConf.TIME_TYPE_ENABLED.key -> "true") {
+ val timeType = TimeType(TimeType.DEFAULT_PRECISION)
+ // Two time values merge to TimeType
+ assert(JsonInferSchema.compatibleType(timeType, timeType) === timeType)
+ // Time + StringType merges to StringType
+ assert(JsonInferSchema.compatibleType(timeType, StringType) ===
StringType)
+ // Time + DateType merges to StringType (via findWiderDateTimeType ->
None)
+ assert(JsonInferSchema.compatibleType(timeType, DateType) === StringType)
+ // Time + NullType merges to TimeType
+ assert(JsonInferSchema.compatibleType(timeType, NullType) === timeType)
+ }
+ }
}
diff --git
a/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out
b/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out
index 33052dd7e8a1..11637861ccee 100644
--- a/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/csv-functions.sql.out
@@ -461,7 +461,7 @@ select schema_of_csv('14:30:45')
-- !query schema
struct<schema_of_csv(14:30:45):string>
-- !query output
-STRUCT<_c0: TIMESTAMP>
+STRUCT<_c0: TIME(6)>
-- !query
@@ -469,4 +469,4 @@ select schema_of_csv('14:30:45.123456')
-- !query schema
struct<schema_of_csv(14:30:45.123456):string>
-- !query output
-STRUCT<_c0: TIMESTAMP>
+STRUCT<_c0: TIME(6)>
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index e48b453309aa..0144c652b688 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -3921,6 +3921,22 @@ abstract class CSVSuite
)
}
}
+
+ test("SPARK-57572: infer TimeType from CSV via spark.read.csv") {
+ withSQLConf(
+ SQLConf.TIME_TYPE_ENABLED.key -> "true") {
+ withTempDir { dir =>
+ val path = s"${dir.getCanonicalPath}/time_infer.csv"
+ Seq("time", "12:13:14", "23:59:59.123456").toDF("value")
+ .coalesce(1).write.text(path)
+ val df = spark.read
+ .option("header", "true")
+ .option("inferSchema", "true")
+ .csv(path)
+ assert(df.schema("time").dataType ===
TimeType(TimeType.DEFAULT_PRECISION))
+ }
+ }
+ }
}
class CSVv1Suite extends CSVSuite {
@@ -4032,7 +4048,9 @@ class CSVLegacyTimeParserSuite extends CSVSuite {
Seq("Write timestamps correctly in ISO8601 format by default",
// The result is different because the date/timestamp parser behavior is
different. Not too
// much value to test it.
- "csv with variant")
+ "csv with variant",
+ // Legacy time parser does not support TIME type inference
+ "SPARK-57572: infer TimeType from CSV via spark.read.csv")
override protected def sparkConf: SparkConf =
super
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index a447e9900130..8cdae049723a 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -4236,6 +4236,21 @@ abstract class JsonSuite
}
}
}
+
+ test("SPARK-57572: infer TimeType from JSON via spark.read.json") {
+ withSQLConf(
+ SQLConf.TIME_TYPE_ENABLED.key -> "true") {
+ withTempDir { dir =>
+ val path = s"${dir.getCanonicalPath}/time_infer.json"
+ Seq("""{"time": "12:13:14"}""", """{"time": "23:59:59.123456"}""")
+ .toDF("value").coalesce(1).write.text(path)
+ val df = spark.read
+ .option("inferTimestamp", "true")
+ .json(path)
+ assert(df.schema("time").dataType ===
TimeType(TimeType.DEFAULT_PRECISION))
+ }
+ }
+ }
}
class JsonV1Suite extends JsonSuite {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
