cshuo commented on code in PR #20544:
URL: https://github.com/apache/hudi/pull/20544#discussion_r4520550599
##########
hudi-common/src/main/java/org/apache/hudi/common/avro/AvroRecordContext.java:
##########
@@ -107,6 +118,36 @@ public Object getValue(IndexedRecord record, HoodieSchema
schema, String fieldNa
return getFieldValueFromIndexedRecord(record, fieldName);
}
+ @Override
+ public Comparable getOrderingValue(IndexedRecord record, HoodieSchema
schema, List<String> orderingFieldNames) {
Review Comment:
Added schema-aware delete ordering conversion, reusing
`convertValueForAvroLogicalTypes`. Regression tests cover older, equal, and
newer DATE deletes in both arrival orders.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeModeEventTimeOrdering.scala:
##########
@@ -31,6 +32,34 @@ import
org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
import
org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase.validateTableConfig
class TestMergeModeEventTimeOrdering extends HoodieSparkSqlTestBase {
+ Seq(
+ ("date", "date'2026-10-01'", "date'2026-10-02'", false),
+ ("timestamp", "timestamp'2026-10-01 00:00:00'", "timestamp'2026-10-02
00:00:00'", false),
+ ("timestamp", "timestamp'2026-10-01 00:00:00'", "timestamp'2026-10-02
00:00:00'", true),
+ ("decimal(10,2)", "cast(1.50 as decimal(10,2))", "cast(2.50 as
decimal(10,2))", false)
+ ).foreach { case (orderingType, oldValue, newValue, consistentTimestamp) =>
+ test(s"COW SQL upsert with $orderingType ordering field
(consistentTimestamp=$consistentTimestamp)") {
+ withSQLConf(SPARK_SQL_INSERT_INTO_OPERATION.key -> "upsert",
+
"hoodie.datasource.write.keygenerator.consistent.logical.timestamp.enabled" ->
consistentTimestamp.toString) {
+ withRecordType(Seq(HoodieRecordType.AVRO)) {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ | create table $tableName (id int, v int, ts $orderingType)
+ | using hudi
+ | tblproperties (type = 'cow', primaryKey = 'id',
preCombineField = 'ts')
+ | location '${tmp.getCanonicalPath}'
+ """.stripMargin)
+ spark.sql(s"insert into $tableName values (1, 1, $oldValue)")
+ spark.sql(s"insert into $tableName values (1, 2, $newValue)")
+ checkAnswer(s"select id, v from $tableName")(Seq(1, 2))
Review Comment:
Added a third upsert with `oldValue` and `v = 3`, asserting that `v` stays 2
in each case.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestUpdateTable.scala:
##########
@@ -34,6 +34,59 @@ import org.junit.jupiter.api.Assertions.assertEquals
class TestUpdateTable extends HoodieSparkSqlTestBase {
+ // (ordering field type, older literal, newer literal, older value as
string, newer value as string)
+ private val logicalTypeOrderingCases = Seq(
+ ("timestamp", "timestamp'2026-10-01 00:00:00'", "timestamp'2026-10-02
00:00:00'",
+ "2026-10-01 00:00:00", "2026-10-02 00:00:00"),
+ ("date", "date'2026-10-01'", "date'2026-10-02'", "2026-10-01",
"2026-10-02"),
+ ("decimal(10,2)", "cast(1.50 as decimal(10,2))", "cast(2.50 as
decimal(10,2))", "1.50", "2.50"))
+
+ logicalTypeOrderingCases.foreach { case (orderingType, oldLiteral,
newLiteral, oldStr, newStr) =>
+ test(s"Test Update Table With ${orderingType.toUpperCase} Ordering Field")
{
+ withTempDir { tmp =>
+ Seq("cow", "mor").foreach { tableType =>
Review Comment:
Reduced the matrix to COW with optimized writes enabled and disabled, two
table lifecycles per type.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]