stevomitric commented on code in PR #58693:
URL: https://github.com/apache/spark/pull/58693#discussion_r3980072050
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypeWideningSuite.scala:
##########
@@ -225,9 +226,30 @@ class ParquetTypeWideningSuite
}
}
+ // The same widening from a legacy INT96 timestamp column. INT96 has no
logical unit; it decodes
+ // to microseconds and is promoted to (epochMicros, 0). INT96 is only
produced for the LTZ family;
+ // the write/read INT96 rebase are pinned to CORRECTED so
proleptic-Gregorian values round-trip.
for {
- outputTimestampType <-
- Seq(ParquetOutputTimestampType.INT96,
ParquetOutputTimestampType.TIMESTAMP_MILLIS)
+ toType: DataType <- Seq(
+ TimestampLTZNanosType(TimestampLTZNanosType.NANOS_PRECISION),
+ TimestampLTZNanosType(7))
+ }
+ test(s"parquet widening conversion TimestampType (int96) -> $toType") {
+ withSQLConf(
+ SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
+ SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
ParquetOutputTimestampType.INT96.toString,
+ SQLConf.PARQUET_INT96_REBASE_MODE_IN_WRITE.key ->
LegacyBehaviorPolicy.CORRECTED.toString,
+ SQLConf.PARQUET_INT96_REBASE_MODE_IN_READ.key ->
LegacyBehaviorPolicy.CORRECTED.toString) {
Review Comment:
done, the INT96 read now runs under both CORRECTED and LEGACY rebase over a
pre-1582 value.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
