This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20756-408718220c76bff329fe992126320b63782d3a64 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 0ed986b8ad1c06da88f5465fc5e9d79de379fff6 Author: Danny Chan <[email protected]> AuthorDate: Fri Nov 20 06:16:40 2026 +0000 test(spark): complete keyed SDP acceptance (#20756) --- hudi-integ-test/spark-pipelines/KEYED_FLOWS.md | 35 ++++++++++++----- hudi-integ-test/spark-pipelines/keyed_flows.py | 54 +++++++++++++++++++++++--- 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md b/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md index cec90f50410f..bfc1a94695ed 100644 --- a/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md +++ b/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md @@ -17,13 +17,12 @@ limitations under the License. # Keyed SDP flow acceptance -Run the real Spark 4.1.1 CLI against Connect using the #20453 harness and a -candidate bundle containing this PR's early partition-field validation. +Run the real Spark 4.1.1 CLI against Connect using the bundled harness and a +candidate bundle built from the same checkout. Use JDK 17 and install the harness's pinned requirements in a dedicated environment. ```sh -python hudi-integ-test/spark-pipelines/keyed_flows.py \ - --harness-dir /path/to/harness/hudi-integ-test/spark-pipelines \ +PYTHONOPTIMIZE=1 python hudi-integ-test/spark-pipelines/keyed_flows.py \ --bundle /path/to/candidate-hudi-spark4.1-bundle.jar ``` @@ -33,8 +32,11 @@ Run each table type separately with isolated targets and checkpoints. The generated SQL defines partitioned tables with `primaryKey=id`, `preCombineField=ts`, and `hoodie.record.merge.mode=EVENT_TIME_ORDERING`. A boolean `_hoodie_is_deleted` carries deletes. The scenario checks a newly -created target and a pre-existing catalog table, then restarts each through the -same SDP-managed checkpoint. It verifies exact contents after intra-batch +created target and a pre-existing catalog table containing an unrelated key in +partition `c`. The existing target's flow omits `TBLPROPERTIES`, so its table type, +key, ordering field and merger must come from persisted table settings. Its seed +row must survive every input stage and rejected configuration. Both targets use +the same SDP-managed checkpoints on restart. The scenario verifies exact contents after intra-batch precombine (higher ordering value arrives first), a newer update, a late lower-order update, a late delete that must preserve the newer row, deletion, reinsertion and an unchanged restart. Partition `b` retains its unrelated key throughout. @@ -43,11 +45,23 @@ Negative scenarios reject nonexistent key, ordering and partition fields, non-boolean delete markers and a change to persisted record-key identity. Failed new targets must have no completed data commit; the existing valid targets must retain their exact contents after every rejected scenario. +Their completed data timelines must also remain unchanged after rejection. + +After reinsertion, the fixture models interruption between Hudi commit and Spark +checkpoint completion by removing the latest Spark commit-log entry for each +stopped AvailableNow query. It retains both Hudi timelines, source offsets and +checkpoint identities. A retry must recreate Spark's commit-log entries, return +the exact expected rows and add no Hudi data commits. `retry-commits.json` retains +the before/after timelines and the replayed batch paths. The subsequent ordinary +restart must also leave rows and completed commits unchanged. This is a controlled +checkpoint replay test; the real sink failure-injection and resource-cleanup +tests are in `TestStreamingSinkRecovery`. Generated Python/SQL and logs remain under `target/keyed-acceptance-logs`. -Metadata indexing is disabled. This is delete input carried by a flow, not CDC +Metadata indexing is enabled, matching the production default. Both targets must +initialize their metadata tables and complete metadata writes. This is delete input carried by a flow, not CDC propagation of upstream deletes. It does not cover full refresh, table chaining, -fan-in or failure between commit and checkpoint (separate recovery coverage). +fan-in, Auto CDC or SCD1/2. For MOR, inline and asynchronous compaction are disabled. After the newer update, the test requires log files and verifies that read-optimized base-file output matches @@ -59,5 +73,6 @@ before/after commit filenames for each table if the check fails. The flows rely on the default streaming upsert operation, without a session override. The catalog regression uses `PARTITIONED BY (PART)` against schema field `part` to verify case-insensitive normalization. The SDP scenario uses exact spelling -because Spark separately rejects a declared/stored partition spelling mismatch. Land after the #20453 harness dependency. -Checks remain active under `python -O`; SIGTERM unwinds harness cleanup. +because Spark separately rejects a declared/stored partition spelling mismatch. +Checks remain active under `PYTHONOPTIMIZE=1`, including verification subprocesses; +SIGTERM unwinds harness cleanup. diff --git a/hudi-integ-test/spark-pipelines/keyed_flows.py b/hudi-integ-test/spark-pipelines/keyed_flows.py index 445ff0b9624e..15c9a1ad6370 100644 --- a/hudi-integ-test/spark-pipelines/keyed_flows.py +++ b/hudi-integ-test/spark-pipelines/keyed_flows.py @@ -26,6 +26,7 @@ import tempfile SOURCE = 'from pyspark import pipelines as dp\nfrom pyspark.sql import SparkSession\nspark = SparkSession.active()\[email protected]_view(name="input_events")\ndef input_events():\n return spark.readStream.schema("id INT, ts LONG, part STRING, value STRING, _hoodie_is_deleted BOOLEAN").json(spark.conf.get("hudi.test.input"))\n' PROPERTIES = "'primaryKey'='id', 'preCombineField'='ts', 'hoodie.record.merge.mode'='EVENT_TIME_ORDERING'" TABLES = ("keyed_new", "keyed_existing") +EXISTING_ROW = [3, 7, "c", "seed"] def require(condition, message): @@ -47,9 +48,12 @@ def client(mode, remote, root): if mode == "prepare": spark.sql(f"CREATE TABLE sdp_test.keyed_existing (id INT, ts BIGINT, part STRING, value STRING, " f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY (part) TBLPROPERTIES ({properties})").collect() + spark.sql("INSERT INTO sdp_test.keyed_existing (id, ts, part, value, _hoodie_is_deleted) " + "VALUES (3, 7L, 'c', 'seed', false)").collect() else: - expected = json.loads((root / "expected.json").read_text()) + input_expected = json.loads((root / "expected.json").read_text()) for table in TABLES: + expected = input_expected + ([EXISTING_ROW] if table == "keyed_existing" else []) actual = [list(row) for row in spark.table(f"sdp_test.{table}") .select("id", "ts", "part", "value").orderBy("id").collect()] require(actual == expected, (table, actual, expected)) @@ -62,6 +66,8 @@ def client(mode, remote, root): .load(location.as_uri()).select("id", "ts", "part", "value") .orderBy("id").collect()] expected_base = json.loads((root / "expected-base.json").read_text()) + if table == "keyed_existing": + expected_base.append(EXISTING_ROW) require(base_rows == expected_base, (table, "MOR base rows", base_rows, expected_base)) print(f"PASS MOR log merge {table}: base rows {base_rows}, snapshot {actual}") finally: @@ -74,15 +80,18 @@ def scenario(h, table_type): source = h.root / "input" source.mkdir() (h.root / "source.py").write_text(SOURCE) + # Only the new target declares identity/merger settings. The existing target must load its + # type, key, ordering field and merge mode from the table initialized by prepare. flows = "\n".join( - f"CREATE STREAMING TABLE {table} USING hudi PARTITIONED BY (part) TBLPROPERTIES ({properties}) " - "AS SELECT id, ts, part, value, _hoodie_is_deleted FROM STREAM input_events;" for table in TABLES) + f"CREATE STREAMING TABLE {table} USING hudi PARTITIONED BY (part) " + + (f"TBLPROPERTIES ({properties}) " if table == "keyed_new" else "") + + "AS SELECT id, ts, part, value, _hoodie_is_deleted FROM STREAM input_events;" for table in TABLES) (h.root / "flows.sql").write_text(flows) spec = {"name": "keyed_flows", "storage": (h.root / "checkpoints").as_uri(), "catalog": "spark_catalog", "database": "sdp_test", "libraries": [{"glob": {"include": "source.py"}}, {"glob": {"include": "flows.sql"}}], "configuration": {"hudi.test.input": source.as_uri(), "spark.sql.shuffle.partitions": "2", - "spark.hoodie.metadata.enable": "false", + "spark.hoodie.metadata.enable": "true", "spark.hoodie.compact.inline": "false", "spark.hoodie.datasource.write.streaming.disable.compaction": "true", "spark.sql.pipelines.maxFlowRetryAttempts": "0"}} @@ -106,6 +115,13 @@ def scenario(h, table_type): (h.root / "expected.json").write_text(json.dumps(expected)) h.run_pipeline(path, f"stage-{i}") h.client(Path(__file__).resolve(), f"verify-{i}") + for table in TABLES: + metadata_path = h.root / "warehouse" / "sdp_test.db" / table / ".hoodie" / "metadata" + require((metadata_path / ".hoodie" / "hoodie.properties").is_file(), + f"Metadata table was not initialized for {table}") + require(any(p.suffix in (".commit", ".deltacommit") + for p in (metadata_path / ".hoodie" / "timeline").iterdir()), + f"No completed metadata writes for {table}") def completed_commits(): return {table: sorted(p.name for p in (h.root / "warehouse" / "sdp_test.db" / table / ".hoodie" / "timeline").glob("*") @@ -113,6 +129,30 @@ def scenario(h, table_type): before_restart = completed_commits() require(all(before_restart.values()), "Expected completed commits before restart") + # Model a crash after the Hudi commit but before Spark records batch completion. The + # AvailableNow queries are stopped; remove only their latest Spark commit-log entries. + # Keep source offsets, checkpoint identities and both Hudi timelines intact for replay. + checkpoint_commits = [directory for directory in (h.root / "checkpoints").rglob("commits") + if directory.is_dir() and (directory.parent / "offsets").is_dir()] + require(len(checkpoint_commits) == len(TABLES), checkpoint_commits) + removed = [] + for directory in checkpoint_commits: + batches = [p for p in directory.iterdir() if p.name.isdigit()] + require(batches, f"No committed Spark batch under {directory}") + latest = max(batches, key=lambda p: int(p.name)) + require((directory.parent / "offsets" / latest.name).is_file(), latest) + removed.append(latest) + latest.unlink() + (directory / f".{latest.name}.crc").unlink(missing_ok=True) + h.run_pipeline(path, "retry-after-commit") + after_retry = completed_commits() + require(after_retry == before_restart, + ("Retry recommitted a completed Hudi batch", before_restart, after_retry)) + require(all(p.is_file() for p in removed), "Retry did not finish Spark's interrupted batch") + h.client(Path(__file__).resolve(), "verify-retry") + (h.logs / "retry-commits.json").write_text(json.dumps( + {"before": before_restart, "after": after_retry, + "replayed_spark_commits": [str(p.relative_to(h.root)) for p in removed]}, indent=2)) h.run_pipeline(path, "restart") after_restart = completed_commits() require(after_restart == before_restart, @@ -145,8 +185,10 @@ def scenario(h, table_type): if name != "changed_identity": timeline = h.root / "warehouse" / "sdp_test.db" / target / ".hoodie" / "timeline" require(not [p for p in timeline.glob("*") if p.suffix in (".commit", ".deltacommit", ".replacecommit")], name) + require(completed_commits() == after_restart, + ("Rejected configuration changed a valid target timeline", name, completed_commits(), after_restart)) h.client(Path(__file__).resolve(), f"verify-after-{name}") - print("PASS: keyed deduplication, updates, late values, delete, reinsertion, restart and invalid configurations") + print("PASS: keyed ordering, delete/reinsertion, persisted settings, metadata writes, retry, restart and rejection") def main(): @@ -154,7 +196,7 @@ def main(): client(sys.argv[1], sys.argv[2], Path(sys.argv[3])) return parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--harness-dir", required=True, type=Path) + parser.add_argument("--harness-dir", type=Path, default=Path(__file__).resolve().parent) parser.add_argument("--table-type", choices=("cow", "mor"), default="cow") parser.add_argument("--bundle", required=True, type=Path) parser.add_argument("--logs", type=Path, default=Path("target/keyed-acceptance-logs"))
