This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20675-f757583aa3e07d2a6e3394e9765d7398a31b8fa2 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit a05e56c90bc9cf05c445769881ecc9272e2e6130 Author: Danny Chan <[email protected]> AuthorDate: Thu Nov 19 03:32:07 2026 +0000 test(spark): cover keyless aggregate SDP materialized views (#20675) --- .../spark-pipelines/MATERIALIZED_VIEWS.md | 30 ++++++++++++------ .../spark-pipelines/materialized_views.py | 36 +++++++++++++++++++--- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/hudi-integ-test/spark-pipelines/MATERIALIZED_VIEWS.md b/hudi-integ-test/spark-pipelines/MATERIALIZED_VIEWS.md index 2fd3f2d0ee8b..08cd4c35de66 100644 --- a/hudi-integ-test/spark-pipelines/MATERIALIZED_VIEWS.md +++ b/hudi-integ-test/spark-pipelines/MATERIALIZED_VIEWS.md @@ -27,16 +27,26 @@ python hudi-integ-test/spark-pipelines/materialized_views.py \ --bundle /path/to/candidate-hudi-spark4.1-bundle.jar ``` -Two Hudi materialized views read a JSON source and a Hudi source respectively. -Four real CLI runs replace the source contents with `(1, one), (2, two)`, -then `(2, updated), (3, three)`, then `(4, four)`, then zero rows. -The JSON-backed target is partitioned by `value`. After every refresh, each -view must exactly match its defining query's expected rows. Removing old rows -ensures that accidental append behavior fails. Repeated catalog reconciliation -exercises metadata-column deletes and unchanged properties. - -Keyless aggregate materialized views remain deferred under #20405. -Land this scenario after the #20453 harness dependency. Checks remain active under +Three Hudi materialized views cover keyed JSON and Hudi sources and a keyless +JSON aggregate. The aggregate declares only `type=cow`, with no primary key, +and groups by `value`, computing `sum(id)` and `count(*)`. + +Four real CLI runs use the following exact inputs and aggregate expectations: + +| Run | Input `(id, value)` | Aggregate `(value, total, rows)` | +| --- | --- | --- | +| 1 | `(1, one), (2, one), (3, two)` | `(one, 3, 2), (two, 3, 1)` | +| 2 | `(2, one), (4, one), (3, three)` | `(one, 6, 2), (three, 3, 1)` | +| 3 | `(4, four)` | `(four, 4, 1)` | +| 4 | empty | empty | + +After every refresh, all three views must match exact expected business rows. +The repeated `one` group changes its aggregate, old groups disappear, and the +last refresh removes every group. This detects stale rows and duplicate aggregate +rows even when Hudi generates record keys. The keyed JSON target remains +partitioned by `value`; the aggregate is unpartitioned. + +Checks remain active under `python -O`, and SIGTERM unwinds the harness for server and temporary-directory cleanup. This is Phase 2 coverage on today's purge-style truncate. See the diff --git a/hudi-integ-test/spark-pipelines/materialized_views.py b/hudi-integ-test/spark-pipelines/materialized_views.py index 2eb650c0773b..6533bdaa8c02 100644 --- a/hudi-integ-test/spark-pipelines/materialized_views.py +++ b/hudi-integ-test/spark-pipelines/materialized_views.py @@ -23,7 +23,25 @@ from pathlib import Path import sys import tempfile -SOURCE = 'from pyspark import pipelines as dp\nfrom pyspark.sql import SparkSession\nspark = SparkSession.active()\n\[email protected]_view(name="json_mv", format="hudi", partition_cols=["value"], table_properties={"type": "cow", "primaryKey": "id"})\ndef json_mv():\n return spark.read.schema("id INT, value STRING").json(spark.conf.get("hudi.test.input"))\n\[email protected]_view(name="hudi_mv", format="hudi", table_properties={"type": "cow", "primaryKey": "id"})\ndef hudi_mv():\n [...] +SOURCE = """from pyspark import pipelines as dp +from pyspark.sql import SparkSession, functions as F +spark = SparkSession.active() + [email protected]_view(name="json_mv", format="hudi", partition_cols=["value"], + table_properties={"type": "cow", "primaryKey": "id"}) +def json_mv(): + return spark.read.schema("id INT, value STRING").json(spark.conf.get("hudi.test.input")) + [email protected]_view(name="hudi_mv", format="hudi", table_properties={"type": "cow", "primaryKey": "id"}) +def hudi_mv(): + return spark.read.format("hudi").load(spark.conf.get("hudi.test.source")).select("id", "value") + [email protected]_view(name="aggregate_mv", format="hudi", table_properties={"type": "cow"}) +def aggregate_mv(): + return (spark.read.schema("id INT, value STRING").json(spark.conf.get("hudi.test.input")) + .groupBy("value").agg(F.sum("id").alias("total"), F.count("*").alias("rows"))) +""" + def require(condition, message): if not condition: @@ -54,6 +72,11 @@ def client(mode, remote, root): .select("id", "value").orderBy("id").collect()] require(actual == expected, (name, actual, expected)) print(f"PASS {mode} {name}: {actual}") + expected_aggregate = json.loads((root / "expected-aggregate.json").read_text()) + actual = [list(row) for row in spark.table("sdp_test.aggregate_mv") + .select("value", "total", "rows").orderBy("value").collect()] + require(actual == expected_aggregate, ("aggregate_mv", actual, expected_aggregate)) + print(f"PASS {mode} aggregate_mv: {actual}") finally: spark.stop() @@ -75,15 +98,20 @@ def scenario(h): (h.logs / "mv.py").write_text(SOURCE) (h.logs / path.name).write_text(path.read_text()) # Remove and change rows as well as adding them: stale append output must fail. - for i, expected in enumerate(([[1, "one"], [2, "two"]], - [[2, "updated"], [3, "three"]], [[4, "four"]], []), 1): + stages = ( + ([[1, "one"], [2, "one"], [3, "two"]], [["one", 3, 2], ["two", 3, 1]]), + ([[2, "one"], [3, "three"], [4, "one"]], [["one", 6, 2], ["three", 3, 1]]), + ([[4, "four"]], [["four", 4, 1]]), + ([], [])) + for i, (expected, aggregate) in enumerate(stages, 1): + (h.root / "expected-aggregate.json").write_text(json.dumps(aggregate)) (h.root / "expected.json").write_text(json.dumps(expected)) (source / "rows.json").write_text("".join(json.dumps(dict(zip(("id", "value"), row))) + "\n" for row in expected)) h.client(Path(__file__).resolve(), f"prepare-{i}") h.run_pipeline(path, f"refresh-{i}") h.client(Path(__file__).resolve(), f"verify-{i}") - print("PASS: four exact MV refreshes over JSON and Hudi sources") + print("PASS: four exact keyed and keyless aggregate MV refreshes") def main():
