This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new b3318ae4cf94 test(spark): cover repeated SDP materialized view 
refreshes (#20494)
b3318ae4cf94 is described below

commit b3318ae4cf94ee2d23f1bf223b11ffbd7a3e985d
Author: Danny Chan <[email protected]>
AuthorDate: Thu Nov 5 12:59:42 2026 +0800

    test(spark): cover repeated SDP materialized view refreshes (#20494)
    
    * test(spark): cover repeated SDP materialized view refreshes
    
    * test(spark): address materialized view acceptance review
---
 .../spark-pipelines/MATERIALIZED_VIEWS.md          |  48 +++++++++
 .../spark-pipelines/materialized_views.py          | 109 +++++++++++++++++++++
 2 files changed, 157 insertions(+)

diff --git a/hudi-integ-test/spark-pipelines/MATERIALIZED_VIEWS.md 
b/hudi-integ-test/spark-pipelines/MATERIALIZED_VIEWS.md
new file mode 100644
index 000000000000..03d783c99d38
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/MATERIALIZED_VIEWS.md
@@ -0,0 +1,48 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Repeated materialized-view acceptance
+
+Run `materialized_views.py` with the Spark 4.1.1 Connect harness from #20453
+and a JDK 17 candidate bundle containing #20492 (catalog reconciliation).
+Install the harness's pinned `requirements.txt` in a dedicated virtual 
environment.
+
+```sh
+python hudi-integ-test/spark-pipelines/materialized_views.py \
+  --harness-dir /path/to/harness/hudi-integ-test/spark-pipelines \
+  --bundle /path/to/candidate-hudi-spark4.1-bundle.jar
+```
+
+Two Hudi materialized views read a JSON source and a Hudi source respectively.
+Four real CLI runs replace the source contents with `(1, one), (2, two)`,
+then `(2, updated), (3, three)`, then `(4, four)`, then zero rows.
+The JSON-backed target is partitioned by `value`. After every refresh, each
+view must exactly match its defining query's expected rows. Removing old rows
+ensures that accidental append behavior fails. Repeated catalog reconciliation
+exercises metadata-column deletes and unchanged properties.
+
+Keyless aggregate materialized views remain deferred under #20405.
+Land this scenario after the #20453 harness dependency. Checks remain active 
under
+`python -O`, and SIGTERM unwinds the harness for server and 
temporary-directory cleanup.
+
+This is Phase 2 coverage on today's purge-style truncate. History retention,
+atomic refresh and timeline-preserving truncate are intentionally outside this
+scenario and remain Phase 4 work. Metadata indexing is disabled. The test does
+not establish streaming-table chaining or full-refresh behavior.
+
+The harness bounds each client/CLI process and removes its temporary data.
+Logs and generated pipeline definitions remain under 
`target/mv-acceptance-logs`.
diff --git a/hudi-integ-test/spark-pipelines/materialized_views.py 
b/hudi-integ-test/spark-pipelines/materialized_views.py
new file mode 100644
index 000000000000..2eb650c0773b
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/materialized_views.py
@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Refresh Hudi materialized views four times through the real SDP CLI."""
+import argparse
+import json
+import signal
+from pathlib import Path
+import sys
+import tempfile
+
+SOURCE = 'from pyspark import pipelines as dp\nfrom pyspark.sql import 
SparkSession\nspark = 
SparkSession.active()\n\[email protected]_view(name="json_mv", format="hudi", 
partition_cols=["value"], table_properties={"type": "cow", "primaryKey": 
"id"})\ndef json_mv():\n    return spark.read.schema("id INT, value 
STRING").json(spark.conf.get("hudi.test.input"))\n\[email protected]_view(name="hudi_mv",
 format="hudi", table_properties={"type": "cow", "primaryKey": "id"})\ndef 
hudi_mv():\n     [...]
+
+def require(condition, message):
+    if not condition:
+        raise AssertionError(message)
+
+
+def interrupted(signum, _frame):
+    raise KeyboardInterrupt(f"Received signal {signum}")
+
+
+def client(mode, remote, root):
+    from pyspark.sql import SparkSession
+    spark = SparkSession.builder.remote(remote).getOrCreate()
+    try:
+        require(spark.conf.get("spark.hudi.test.run.id") == root.name, 
"Connected to the wrong test server")
+        expected = json.loads((root / "expected.json").read_text())
+        frame = spark.createDataFrame([tuple(row) for row in expected], "id 
INT, value STRING")
+        if mode.startswith("prepare"):
+            (frame.write.format("hudi").mode("overwrite")
+             .option("hoodie.table.name", "mv_source")
+             .option("hoodie.datasource.write.recordkey.field", "id")
+             .option("hoodie.datasource.write.partitionpath.field", "")
+             .option("hoodie.datasource.write.operation", "bulk_insert")
+             .option("hoodie.metadata.enable", "false").save((root / 
"hudi-source").as_uri()))
+        else:
+            for name in ("json_mv", "hudi_mv"):
+                actual = [list(row) for row in spark.table(f"sdp_test.{name}")
+                          .select("id", "value").orderBy("id").collect()]
+                require(actual == expected, (name, actual, expected))
+                print(f"PASS {mode} {name}: {actual}")
+    finally:
+        spark.stop()
+
+
+def scenario(h):
+    source = h.root / "input"
+    source.mkdir()
+    (h.root / "mv.py").write_text(SOURCE)
+    spec = {"name": "hudi_mv_refresh", "storage": (h.root / 
"checkpoints").as_uri(),
+            "catalog": "spark_catalog", "database": "sdp_test",
+            "libraries": [{"glob": {"include": "mv.py"}}],
+            "configuration": {"hudi.test.input": source.as_uri(),
+                              "hudi.test.source": (h.root / 
"hudi-source").as_uri(),
+                              "spark.sql.shuffle.partitions": "2",
+                              "spark.hoodie.metadata.enable": "false",
+                              "spark.sql.pipelines.maxFlowRetryAttempts": "0"}}
+    path = h.root / "spark-pipeline.yml"
+    path.write_text(json.dumps(spec, indent=2))
+    (h.logs / "mv.py").write_text(SOURCE)
+    (h.logs / path.name).write_text(path.read_text())
+    # Remove and change rows as well as adding them: stale append output must 
fail.
+    for i, expected in enumerate(([[1, "one"], [2, "two"]],
+                                   [[2, "updated"], [3, "three"]], [[4, 
"four"]], []), 1):
+        (h.root / "expected.json").write_text(json.dumps(expected))
+        (source / "rows.json").write_text("".join(json.dumps(dict(zip(("id", 
"value"), row))) + "\n"
+                                                   for row in expected))
+        h.client(Path(__file__).resolve(), f"prepare-{i}")
+        h.run_pipeline(path, f"refresh-{i}")
+        h.client(Path(__file__).resolve(), f"verify-{i}")
+    print("PASS: four exact MV refreshes over JSON and Hudi sources")
+
+
+def main():
+    if len(sys.argv) == 4 and sys.argv[1].startswith(("prepare-", "verify-")):
+        client(sys.argv[1], sys.argv[2], Path(sys.argv[3]))
+        return
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--harness-dir", required=True, type=Path)
+    parser.add_argument("--bundle", required=True, type=Path)
+    parser.add_argument("--logs", type=Path, 
default=Path("target/mv-acceptance-logs"))
+    args = parser.parse_args()
+    sys.path.insert(0, str(args.harness_dir.resolve()))
+    from harness import ConnectHarness
+    args.logs.mkdir(parents=True, exist_ok=True)
+    logs = Path(tempfile.mkdtemp(prefix="run-", dir=args.logs.resolve()))
+    print(f"MV logs: {logs}", flush=True)
+    signal.signal(signal.SIGTERM, interrupted)
+    with ConnectHarness(args.bundle, logs) as harness:
+        scenario(harness)
+
+
+if __name__ == "__main__":
+    main()

Reply via email to