cloud-fan commented on code in PR #44800: URL: https://github.com/apache/spark/pull/44800#discussion_r1460152048
########## sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala: ########## @@ -159,16 +161,28 @@ class JsonInferSchema(options: JSONOptions) extends Serializable with Logging { val bigDecimal = decimalParser(field) DecimalType(bigDecimal.precision, bigDecimal.scale) } - val timestampType = SQLConf.get.timestampType if (options.prefersDecimal && decimalTry.isDefined) { decimalTry.get - } else if (options.inferTimestamp && (SQLConf.get.legacyTimeParserPolicy == - LegacyBehaviorPolicy.LEGACY || timestampType == TimestampNTZType) && + } else if (options.inferTimestamp) { + // For text-based format, it's ambiguous to infer a timestamp string without timezone, as + // it can be both TIMESTAMP LTZ and NTZ. To avoid behavior changes with the new support + // of NTZ, here we only try to infer NTZ if the config is set to use NTZ by default. + if (isDefaultNTZ && timestampNTZFormatter.parseWithoutTimeZoneOptional(field, false).isDefined) { - timestampType - } else if (options.inferTimestamp && - timestampFormatter.parseOptional(field).isDefined) { - TimestampType + TimestampNTZType + } else if (timestampFormatter.parseOptional(field).isDefined) { + TimestampType + } else { + val utf8Value = UTF8String.fromString(field) + // There was a mistake that we use TIMESTAMP NTZ parser to infer LTZ type. + // The mistake makes it easier to infer TIMESTAMP LTZ type and we have to keep this + // behavior now. See SPARK-46769 for more details. + if (SparkDateTimeUtils.stringToTimestampWithoutTimeZone(utf8Value, false).isDefined) { Review Comment: @gengliangwang -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org