This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20676-a05e56c90bc9cf05c445769881ecc9272e2e6130 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 06dc47c2d4666d36c64fa39de4980a90bb64cde9 Author: Danny Chan <[email protected]> AuthorDate: Thu Nov 19 03:40:42 2026 +0000 test(spark): cover MOR log merging in keyed SDP flows (#20676) * test(spark): cover MOR log merging in keyed SDP flows * test(spark): tighten keyed MOR acceptance assertions --- hudi-integ-test/spark-pipelines/KEYED_FLOWS.md | 13 +++++-- hudi-integ-test/spark-pipelines/keyed_flows.py | 50 ++++++++++++++++++++------ 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md b/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md index 888f642483fb..cec90f50410f 100644 --- a/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md +++ b/hudi-integ-test/spark-pipelines/KEYED_FLOWS.md @@ -27,7 +27,10 @@ python hudi-integ-test/spark-pipelines/keyed_flows.py \ --bundle /path/to/candidate-hudi-spark4.1-bundle.jar ``` -The generated SQL defines partitioned COW tables with `primaryKey=id`, +Pass `--table-type mor` to run the same matrix with MOR; the default is COW. +Run each table type separately with isolated targets and checkpoints. + +The generated SQL defines partitioned tables with `primaryKey=id`, `preCombineField=ts`, and `hoodie.record.merge.mode=EVENT_TIME_ORDERING`. A boolean `_hoodie_is_deleted` carries deletes. The scenario checks a newly created target and a pre-existing catalog table, then restarts each through the @@ -46,7 +49,13 @@ Metadata indexing is disabled. This is delete input carried by a flow, not CDC propagation of upstream deletes. It does not cover full refresh, table chaining, fan-in or failure between commit and checkpoint (separate recovery coverage). -MOR log-merge coverage remains deferred under #20408; this matrix covers COW only. +For MOR, inline and asynchronous compaction are disabled. After the newer update, +the test requires log files and verifies that read-optimized base-file output matches +the exact stage-1 rows while the snapshot matches the updated rows. This proves the +snapshot must merge logs. Subsequent late updates, deletes and reinsertion must also +produce exact snapshots. Both table +types verify that an unchanged restart adds no completed commit, reporting the +before/after commit filenames for each table if the check fails. The flows rely on the default streaming upsert operation, without a session override. The catalog regression uses `PARTITIONED BY (PART)` against schema field `part` to verify case-insensitive normalization. The SDP scenario uses exact spelling diff --git a/hudi-integ-test/spark-pipelines/keyed_flows.py b/hudi-integ-test/spark-pipelines/keyed_flows.py index 4d1d2ed6b570..445ff0b9624e 100644 --- a/hudi-integ-test/spark-pipelines/keyed_flows.py +++ b/hudi-integ-test/spark-pipelines/keyed_flows.py @@ -24,7 +24,7 @@ import sys import tempfile SOURCE = 'from pyspark import pipelines as dp\nfrom pyspark.sql import SparkSession\nspark = SparkSession.active()\[email protected]_view(name="input_events")\ndef input_events():\n return spark.readStream.schema("id INT, ts LONG, part STRING, value STRING, _hoodie_is_deleted BOOLEAN").json(spark.conf.get("hudi.test.input"))\n' -PROPERTIES = "'type'='cow', 'primaryKey'='id', 'preCombineField'='ts', 'hoodie.record.merge.mode'='EVENT_TIME_ORDERING'" +PROPERTIES = "'primaryKey'='id', 'preCombineField'='ts', 'hoodie.record.merge.mode'='EVENT_TIME_ORDERING'" TABLES = ("keyed_new", "keyed_existing") @@ -42,26 +42,40 @@ def client(mode, remote, root): spark = SparkSession.builder.remote(remote).getOrCreate() try: require(spark.conf.get("spark.hudi.test.run.id") == root.name, "Connected to the wrong test server") + table_type = (root / "table-type").read_text() + properties = f"'type'='{table_type}', {PROPERTIES}" if mode == "prepare": spark.sql(f"CREATE TABLE sdp_test.keyed_existing (id INT, ts BIGINT, part STRING, value STRING, " - f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY (part) TBLPROPERTIES ({PROPERTIES})").collect() + f"_hoodie_is_deleted BOOLEAN) USING hudi PARTITIONED BY (part) TBLPROPERTIES ({properties})").collect() else: expected = json.loads((root / "expected.json").read_text()) for table in TABLES: actual = [list(row) for row in spark.table(f"sdp_test.{table}") .select("id", "ts", "part", "value").orderBy("id").collect()] require(actual == expected, (table, actual, expected)) - print(f"PASS {mode} {table}: {actual}") + print(f"PASS {mode} {table_type} {table}: {actual}") + if table_type == "mor" and mode == "verify-2": + location = root / "warehouse" / "sdp_test.db" / table + require(any(location.rglob("*.log.*")), f"No MOR log files in {location}") + base_rows = [list(row) for row in spark.read.format("hudi") + .option("hoodie.datasource.query.type", "read_optimized") + .load(location.as_uri()).select("id", "ts", "part", "value") + .orderBy("id").collect()] + expected_base = json.loads((root / "expected-base.json").read_text()) + require(base_rows == expected_base, (table, "MOR base rows", base_rows, expected_base)) + print(f"PASS MOR log merge {table}: base rows {base_rows}, snapshot {actual}") finally: spark.stop() -def scenario(h): +def scenario(h, table_type): + (h.root / "table-type").write_text(table_type) + properties = f"'type'='{table_type}', {PROPERTIES}" source = h.root / "input" source.mkdir() (h.root / "source.py").write_text(SOURCE) flows = "\n".join( - f"CREATE STREAMING TABLE {table} USING hudi PARTITIONED BY (part) TBLPROPERTIES ({PROPERTIES}) " + f"CREATE STREAMING TABLE {table} USING hudi PARTITIONED BY (part) TBLPROPERTIES ({properties}) " "AS SELECT id, ts, part, value, _hoodie_is_deleted FROM STREAM input_events;" for table in TABLES) (h.root / "flows.sql").write_text(flows) spec = {"name": "keyed_flows", "storage": (h.root / "checkpoints").as_uri(), @@ -69,6 +83,8 @@ def scenario(h): "libraries": [{"glob": {"include": "source.py"}}, {"glob": {"include": "flows.sql"}}], "configuration": {"hudi.test.input": source.as_uri(), "spark.sql.shuffle.partitions": "2", "spark.hoodie.metadata.enable": "false", + "spark.hoodie.compact.inline": "false", + "spark.hoodie.datasource.write.streaming.disable.compaction": "true", "spark.sql.pipelines.maxFlowRetryAttempts": "0"}} path = h.root / "spark-pipeline.yml" path.write_text(json.dumps(spec, indent=2)) @@ -83,20 +99,31 @@ def scenario(h): ([(1, 2, "a", "late-delete", True)], [[1, 3, "a", "updated"], [2, 1, "b", "keep"]]), ([(1, 4, "a", "deleted", True)], [[2, 1, "b", "keep"]]), ([(1, 5, "a", "reinserted", False)], [[1, 5, "a", "reinserted"], [2, 1, "b", "keep"]])) + (h.root / "expected-base.json").write_text(json.dumps(stages[0][1])) for i, (rows, expected) in enumerate(stages, 1): (source / f"{i}.json").write_text("".join(json.dumps(dict(zip( ("id", "ts", "part", "value", "_hoodie_is_deleted"), row))) + "\n" for row in rows)) (h.root / "expected.json").write_text(json.dumps(expected)) h.run_pipeline(path, f"stage-{i}") h.client(Path(__file__).resolve(), f"verify-{i}") + def completed_commits(): + return {table: sorted(p.name for p in + (h.root / "warehouse" / "sdp_test.db" / table / ".hoodie" / "timeline").glob("*") + if p.suffix in (".commit", ".deltacommit", ".replacecommit")) for table in TABLES} + + before_restart = completed_commits() + require(all(before_restart.values()), "Expected completed commits before restart") h.run_pipeline(path, "restart") + after_restart = completed_commits() + require(after_restart == before_restart, + ("Unchanged restart wrote an extra commit", before_restart, after_restart)) h.client(Path(__file__).resolve(), "verify-restart") invalid = ( - ("missing_key", PROPERTIES.replace("'primaryKey'='id'", "'primaryKey'='missing_key'"), "part", SOURCE, "Can't find primaryKey"), - ("missing_order", PROPERTIES.replace("'preCombineField'='ts'", "'preCombineField'='missing_order'"), "part", SOURCE, "Can't find ordering fields"), - ("missing_partition", PROPERTIES, "missing_partition", SOURCE, "Partition column 'missing_partition'"), - ("invalid_marker", PROPERTIES, "part", SOURCE.replace("_hoodie_is_deleted BOOLEAN", "_hoodie_is_deleted STRING"), "has to be BOOLEAN"), - ("changed_identity", PROPERTIES.replace("'primaryKey'='id'", "'primaryKey'='value'"), "part", SOURCE, "Can't set primaryKey to 'value' on Hudi table")) + ("missing_key", properties.replace("'primaryKey'='id'", "'primaryKey'='missing_key'"), "part", SOURCE, "Can't find primaryKey"), + ("missing_order", properties.replace("'preCombineField'='ts'", "'preCombineField'='missing_order'"), "part", SOURCE, "Can't find ordering fields"), + ("missing_partition", properties, "missing_partition", SOURCE, "Partition column 'missing_partition'"), + ("invalid_marker", properties, "part", SOURCE.replace("_hoodie_is_deleted BOOLEAN", "_hoodie_is_deleted STRING"), "has to be BOOLEAN"), + ("changed_identity", properties.replace("'primaryKey'='id'", "'primaryKey'='value'"), "part", SOURCE, "Can't set primaryKey to 'value' on Hudi table")) for name, props, partition, definition, error_text in invalid: target = "keyed_existing" if name == "changed_identity" else f"bad_{name}" (h.root / "source.py").write_text(definition) @@ -128,6 +155,7 @@ def main(): return parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--harness-dir", required=True, type=Path) + parser.add_argument("--table-type", choices=("cow", "mor"), default="cow") parser.add_argument("--bundle", required=True, type=Path) parser.add_argument("--logs", type=Path, default=Path("target/keyed-acceptance-logs")) args = parser.parse_args() @@ -138,7 +166,7 @@ def main(): print(f"Keyed logs: {logs}", flush=True) signal.signal(signal.SIGTERM, interrupted) with ConnectHarness(args.bundle, logs) as harness: - scenario(harness) + scenario(harness, args.table_type) if __name__ == "__main__":
