hudi-agent commented on code in PR #19657:
URL: https://github.com/apache/hudi/pull/19657#discussion_r3806003095
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala:
##########
@@ -1450,6 +1450,73 @@ class TestCOWDataSource extends
HoodieSparkClientTestBase with ScalaAssertionSup
assertTrue(recordsReadDF.filter(col("_hoodie_partition_path") =!=
udf_date_format(col("current_ts"))).count() == 0)
}
+ @ParameterizedTest
+ @EnumSource(value = classOf[HoodieRecordType], names = Array("AVRO",
"SPARK"))
+ def testTimestampBasedKeyGeneratorWithVariousConfigurations(recordType:
HoodieRecordType) {
+ val (writeOpts, readOpts) =
getWriterReaderOptsLessPartitionPath(recordType)
+
+ val records = recordsToStrings(dataGen.generateInserts("000",
100)).asScala.toList
+ val inputDF = spark.read.json(spark.sparkContext.parallelize(records, 2))
+ .withColumn("current_ts_micros", col("current_ts") * 1000)
+ .withColumn("current_date_string",
+ date_format((col("current_ts") / 1000).cast("timestamp"), "yyyy-MM-dd
HH:mm:ss"))
+ .withColumn("current_ts_hours", (col("current_ts") /
3600000).cast("long"))
+ .withColumn("current_ts_seconds", (col("current_ts") /
1000).cast("long"))
+
+ case class TestCase(partitionCol: String, tsType: String, outFmt: String,
+ extraOpts: Map[String, String] = Map.empty,
+ expectedPartitionUdf:
org.apache.spark.sql.expressions.UserDefinedFunction)
+
+ def runTestCase(tc: TestCase): Unit = {
+ val writer = tc.extraOpts.foldLeft(
+ inputDF.write.format("hudi")
+ .options(writeOpts)
+ .option(KEYGENERATOR_CLASS_NAME.key(),
classOf[TimestampBasedKeyGenerator].getName)
+ .mode(SaveMode.Overwrite)
+ ) { case (w, (k, v)) => w.option(k, v) }
+ writer.partitionBy(tc.partitionCol)
+ .option(TIMESTAMP_TYPE_FIELD.key, tc.tsType)
+ .option(TIMESTAMP_OUTPUT_DATE_FORMAT.key, tc.outFmt)
+ .save(basePath)
+ val readDF =
spark.read.format("org.apache.hudi").options(readOpts).load(basePath)
+ assertTrue(readDF.filter(col("_hoodie_partition_path") =!=
tc.expectedPartitionUdf(col(tc.partitionCol))).count() == 0)
+ }
+
+ // Test 1: EPOCHMILLISECONDS with timezone GMT+08:00
+ val tzMillisOutFmt = "yyyy-MM-dd HH"
+ val udfMillisTz = udf((millis: Long) =>
+ new
DateTime(millis).withZone(org.joda.time.DateTimeZone.forID("GMT+08:00"))
+
.toString(DateTimeFormat.forPattern(tzMillisOutFmt).withZone(org.joda.time.DateTimeZone.forID("GMT+08:00"))))
+ runTestCase(TestCase("current_ts", "EPOCHMILLISECONDS", tzMillisOutFmt,
+ Map(TIMESTAMP_TIMEZONE_FORMAT.key -> "GMT+08:00"), udfMillisTz))
+
+ // Test 2: EPOCHMICROSECONDS
+ val microsOutFmt = "yyyy-MM-dd HH"
+ val udfMicros = udf((micros: Long) =>
+ new DateTime(micros /
1000).toString(DateTimeFormat.forPattern(microsOutFmt)))
+ runTestCase(TestCase("current_ts_micros", "EPOCHMICROSECONDS",
microsOutFmt,
+ expectedPartitionUdf = udfMicros))
+
+ // Test 3: DATE_STRING with timezone
+ val dateStrOutFmt = "yyyy-MM-dd HH"
+ val dateStrInFmt = "yyyy-MM-dd HH:mm:ss"
+ val udfDateStrTz = udf((s: String) =>
Review Comment:
🤖 The `"GMT+8:00"`→`"GMT+08:00"` rename doesn't address the earlier concern
here: `DateTime.parse(s, DateTimeFormat.forPattern(dateStrInFmt))` still parses
with no zone (JVM default), then `.withZone(GMT+08:00)` shifts the hour —
whereas the keygen's input formatter is
`.withZone(inputDateTimeZone=GMT+08:00)`, so it parses the string *as* GMT+8
(hour round-trips). On a UTC CI box that's an 8h skew and the `yyyy-MM-dd HH`
assertion fails. Could you set the parse zone too, e.g.
`DateTimeFormat.forPattern(dateStrInFmt).withZone(DateTimeZone.forID("GMT+08:00"))`?
<sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag
quality.</i></sub>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
