This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-20756-408718220c76bff329fe992126320b63782d3a64
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 0ed986b8ad1c06da88f5465fc5e9d79de379fff6
Author: Danny Chan <[email protected]>
AuthorDate: Fri Nov 20 06:16:40 2026 +0000

    test(spark): complete keyed SDP acceptance (#20756)
---
 hudi-integ-test/spark-pipelines/KEYED_FLOWS.md | 35 ++++++++++++-----
 hudi-integ-test/spark-pipelines/keyed_flows.py | 54 +++++++++++++++++++++++---
 2 files changed, 73 insertions(+), 16 deletions(-)

diff --git a/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md 
b/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md
index cec90f50410f..bfc1a94695ed 100644
--- a/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md
+++ b/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md
@@ -17,13 +17,12 @@ limitations under the License.
 
 # Keyed SDP flow acceptance
 
-Run the real Spark 4.1.1 CLI against Connect using the #20453 harness and a
-candidate bundle containing this PR's early partition-field validation.
+Run the real Spark 4.1.1 CLI against Connect using the bundled harness and a
+candidate bundle built from the same checkout.
 Use JDK 17 and install the harness's pinned requirements in a dedicated 
environment.
 
 ```sh
-python hudi-integ-test/spark-pipelines/keyed_flows.py \
-  --harness-dir /path/to/harness/hudi-integ-test/spark-pipelines \
+PYTHONOPTIMIZE=1 python hudi-integ-test/spark-pipelines/keyed_flows.py \
   --bundle /path/to/candidate-hudi-spark4.1-bundle.jar
 ```
 
@@ -33,8 +32,11 @@ Run each table type separately with isolated targets and 
checkpoints.
 The generated SQL defines partitioned tables with `primaryKey=id`,
 `preCombineField=ts`, and `hoodie.record.merge.mode=EVENT_TIME_ORDERING`.
 A boolean `_hoodie_is_deleted` carries deletes. The scenario checks a newly
-created target and a pre-existing catalog table, then restarts each through the
-same SDP-managed checkpoint. It verifies exact contents after intra-batch
+created target and a pre-existing catalog table containing an unrelated key in
+partition `c`. The existing target's flow omits `TBLPROPERTIES`, so its table 
type,
+key, ordering field and merger must come from persisted table settings. Its 
seed
+row must survive every input stage and rejected configuration. Both targets use
+the same SDP-managed checkpoints on restart. The scenario verifies exact 
contents after intra-batch
 precombine (higher ordering value arrives first), a newer update, a late 
lower-order
 update, a late delete that must preserve the newer row, deletion, reinsertion 
and
 an unchanged restart. Partition `b` retains its unrelated key throughout.
@@ -43,11 +45,23 @@ Negative scenarios reject nonexistent key, ordering and 
partition fields,
 non-boolean delete markers and a change to persisted record-key identity.
 Failed new targets must have no completed data commit; the existing valid 
targets
 must retain their exact contents after every rejected scenario.
+Their completed data timelines must also remain unchanged after rejection.
+
+After reinsertion, the fixture models interruption between Hudi commit and 
Spark
+checkpoint completion by removing the latest Spark commit-log entry for each
+stopped AvailableNow query. It retains both Hudi timelines, source offsets and
+checkpoint identities. A retry must recreate Spark's commit-log entries, return
+the exact expected rows and add no Hudi data commits. `retry-commits.json` 
retains
+the before/after timelines and the replayed batch paths. The subsequent 
ordinary
+restart must also leave rows and completed commits unchanged. This is a 
controlled
+checkpoint replay test; the real sink failure-injection and resource-cleanup
+tests are in `TestStreamingSinkRecovery`.
 
 Generated Python/SQL and logs remain under `target/keyed-acceptance-logs`.
-Metadata indexing is disabled. This is delete input carried by a flow, not CDC
+Metadata indexing is enabled, matching the production default. Both targets 
must
+initialize their metadata tables and complete metadata writes. This is delete 
input carried by a flow, not CDC
 propagation of upstream deletes. It does not cover full refresh, table 
chaining,
-fan-in or failure between commit and checkpoint (separate recovery coverage).
+fan-in, Auto CDC or SCD1/2.
 
 For MOR, inline and asynchronous compaction are disabled. After the newer 
update,
 the test requires log files and verifies that read-optimized base-file output 
matches
@@ -59,5 +73,6 @@ before/after commit filenames for each table if the check 
fails.
 The flows rely on the default streaming upsert operation, without a session 
override.
 The catalog regression uses `PARTITIONED BY (PART)` against schema field `part`
 to verify case-insensitive normalization. The SDP scenario uses exact spelling
-because Spark separately rejects a declared/stored partition spelling 
mismatch. Land after the #20453 harness dependency.
-Checks remain active under `python -O`; SIGTERM unwinds harness cleanup.
+because Spark separately rejects a declared/stored partition spelling mismatch.
+Checks remain active under `PYTHONOPTIMIZE=1`, including verification 
subprocesses;
+SIGTERM unwinds harness cleanup.
diff --git a/hudi-integ-test/spark-pipelines/keyed_flows.py 
b/hudi-integ-test/spark-pipelines/keyed_flows.py
index 445ff0b9624e..15c9a1ad6370 100644
--- a/hudi-integ-test/spark-pipelines/keyed_flows.py
+++ b/hudi-integ-test/spark-pipelines/keyed_flows.py
@@ -26,6 +26,7 @@ import tempfile
 SOURCE = 'from pyspark import pipelines as dp\nfrom pyspark.sql import 
SparkSession\nspark = 
SparkSession.active()\[email protected]_view(name="input_events")\ndef 
input_events():\n    return spark.readStream.schema("id INT, ts LONG, part 
STRING, value STRING, _hoodie_is_deleted 
BOOLEAN").json(spark.conf.get("hudi.test.input"))\n'
 PROPERTIES = "'primaryKey'='id', 'preCombineField'='ts', 
'hoodie.record.merge.mode'='EVENT_TIME_ORDERING'"
 TABLES = ("keyed_new", "keyed_existing")
+EXISTING_ROW = [3, 7, "c", "seed"]
 
 
 def require(condition, message):
@@ -47,9 +48,12 @@ def client(mode, remote, root):
         if mode == "prepare":
             spark.sql(f"CREATE TABLE sdp_test.keyed_existing (id INT, ts 
BIGINT, part STRING, value STRING, "
                       f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY 
(part) TBLPROPERTIES ({properties})").collect()
+            spark.sql("INSERT INTO sdp_test.keyed_existing (id, ts, part, 
value, _hoodie_is_deleted) "
+                      "VALUES (3, 7L, 'c', 'seed', false)").collect()
         else:
-            expected = json.loads((root / "expected.json").read_text())
+            input_expected = json.loads((root / "expected.json").read_text())
             for table in TABLES:
+                expected = input_expected + ([EXISTING_ROW] if table == 
"keyed_existing" else [])
                 actual = [list(row) for row in spark.table(f"sdp_test.{table}")
                           .select("id", "ts", "part", 
"value").orderBy("id").collect()]
                 require(actual == expected, (table, actual, expected))
@@ -62,6 +66,8 @@ def client(mode, remote, root):
                                  .load(location.as_uri()).select("id", "ts", 
"part", "value")
                                  .orderBy("id").collect()]
                     expected_base = json.loads((root / 
"expected-base.json").read_text())
+                    if table == "keyed_existing":
+                        expected_base.append(EXISTING_ROW)
                     require(base_rows == expected_base, (table, "MOR base 
rows", base_rows, expected_base))
                     print(f"PASS MOR log merge {table}: base rows {base_rows}, 
snapshot {actual}")
     finally:
@@ -74,15 +80,18 @@ def scenario(h, table_type):
     source = h.root / "input"
     source.mkdir()
     (h.root / "source.py").write_text(SOURCE)
+    # Only the new target declares identity/merger settings. The existing 
target must load its
+    # type, key, ordering field and merge mode from the table initialized by 
prepare.
     flows = "\n".join(
-        f"CREATE STREAMING TABLE {table} USING hudi PARTITIONED BY (part) 
TBLPROPERTIES ({properties}) "
-        "AS SELECT id, ts, part, value, _hoodie_is_deleted FROM STREAM 
input_events;" for table in TABLES)
+        f"CREATE STREAMING TABLE {table} USING hudi PARTITIONED BY (part) "
+        + (f"TBLPROPERTIES ({properties}) " if table == "keyed_new" else "")
+        + "AS SELECT id, ts, part, value, _hoodie_is_deleted FROM STREAM 
input_events;" for table in TABLES)
     (h.root / "flows.sql").write_text(flows)
     spec = {"name": "keyed_flows", "storage": (h.root / 
"checkpoints").as_uri(),
             "catalog": "spark_catalog", "database": "sdp_test",
             "libraries": [{"glob": {"include": "source.py"}}, {"glob": 
{"include": "flows.sql"}}],
             "configuration": {"hudi.test.input": source.as_uri(), 
"spark.sql.shuffle.partitions": "2",
-                              "spark.hoodie.metadata.enable": "false",
+                              "spark.hoodie.metadata.enable": "true",
                               "spark.hoodie.compact.inline": "false",
                               
"spark.hoodie.datasource.write.streaming.disable.compaction": "true",
                               "spark.sql.pipelines.maxFlowRetryAttempts": "0"}}
@@ -106,6 +115,13 @@ def scenario(h, table_type):
         (h.root / "expected.json").write_text(json.dumps(expected))
         h.run_pipeline(path, f"stage-{i}")
         h.client(Path(__file__).resolve(), f"verify-{i}")
+        for table in TABLES:
+            metadata_path = h.root / "warehouse" / "sdp_test.db" / table / 
".hoodie" / "metadata"
+            require((metadata_path / ".hoodie" / 
"hoodie.properties").is_file(),
+                    f"Metadata table was not initialized for {table}")
+            require(any(p.suffix in (".commit", ".deltacommit")
+                        for p in (metadata_path / ".hoodie" / 
"timeline").iterdir()),
+                    f"No completed metadata writes for {table}")
     def completed_commits():
         return {table: sorted(p.name for p in
                 (h.root / "warehouse" / "sdp_test.db" / table / ".hoodie" / 
"timeline").glob("*")
@@ -113,6 +129,30 @@ def scenario(h, table_type):
 
     before_restart = completed_commits()
     require(all(before_restart.values()), "Expected completed commits before 
restart")
+    # Model a crash after the Hudi commit but before Spark records batch 
completion. The
+    # AvailableNow queries are stopped; remove only their latest Spark 
commit-log entries.
+    # Keep source offsets, checkpoint identities and both Hudi timelines 
intact for replay.
+    checkpoint_commits = [directory for directory in (h.root / 
"checkpoints").rglob("commits")
+                          if directory.is_dir() and (directory.parent / 
"offsets").is_dir()]
+    require(len(checkpoint_commits) == len(TABLES), checkpoint_commits)
+    removed = []
+    for directory in checkpoint_commits:
+        batches = [p for p in directory.iterdir() if p.name.isdigit()]
+        require(batches, f"No committed Spark batch under {directory}")
+        latest = max(batches, key=lambda p: int(p.name))
+        require((directory.parent / "offsets" / latest.name).is_file(), latest)
+        removed.append(latest)
+        latest.unlink()
+        (directory / f".{latest.name}.crc").unlink(missing_ok=True)
+    h.run_pipeline(path, "retry-after-commit")
+    after_retry = completed_commits()
+    require(after_retry == before_restart,
+            ("Retry recommitted a completed Hudi batch", before_restart, 
after_retry))
+    require(all(p.is_file() for p in removed), "Retry did not finish Spark's 
interrupted batch")
+    h.client(Path(__file__).resolve(), "verify-retry")
+    (h.logs / "retry-commits.json").write_text(json.dumps(
+        {"before": before_restart, "after": after_retry,
+         "replayed_spark_commits": [str(p.relative_to(h.root)) for p in 
removed]}, indent=2))
     h.run_pipeline(path, "restart")
     after_restart = completed_commits()
     require(after_restart == before_restart,
@@ -145,8 +185,10 @@ def scenario(h, table_type):
         if name != "changed_identity":
             timeline = h.root / "warehouse" / "sdp_test.db" / target / 
".hoodie" / "timeline"
             require(not [p for p in timeline.glob("*") if p.suffix in 
(".commit", ".deltacommit", ".replacecommit")], name)
+        require(completed_commits() == after_restart,
+                ("Rejected configuration changed a valid target timeline", 
name, completed_commits(), after_restart))
         h.client(Path(__file__).resolve(), f"verify-after-{name}")
-    print("PASS: keyed deduplication, updates, late values, delete, 
reinsertion, restart and invalid configurations")
+    print("PASS: keyed ordering, delete/reinsertion, persisted settings, 
metadata writes, retry, restart and rejection")
 
 
 def main():
@@ -154,7 +196,7 @@ def main():
         client(sys.argv[1], sys.argv[2], Path(sys.argv[3]))
         return
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--harness-dir", required=True, type=Path)
+    parser.add_argument("--harness-dir", type=Path, 
default=Path(__file__).resolve().parent)
     parser.add_argument("--table-type", choices=("cow", "mor"), default="cow")
     parser.add_argument("--bundle", required=True, type=Path)
     parser.add_argument("--logs", type=Path, 
default=Path("target/keyed-acceptance-logs"))

Reply via email to