This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-20453-7522e3af8dc46712e69d1ef1327df31decd5ace7
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 6d3b743b10151272569ea9f01558a41794885df4
Author: Danny Chan <[email protected]>
AuthorDate: Mon Nov 16 06:00:01 2026 +0000

    test(spark): add Spark Connect SDP integration harness and CI coverage 
(#20453)
    
    * test(spark): add Spark Connect SDP integration harness and CI coverage
    
    * test(spark): split SDP CLI coverage between Maven and nightly
    
    * test(spark): run SDP checks in existing CI jobs
---
 .github/workflows/bot.yml                          |  70 ++++++
 hudi-integ-test/spark-pipelines/README.md          | 172 ++++++++++++++
 hudi-integ-test/spark-pipelines/client.py          |  59 +++++
 hudi-integ-test/spark-pipelines/harness.py         | 251 +++++++++++++++++++++
 hudi-integ-test/spark-pipelines/pipeline.py        |  28 +++
 hudi-integ-test/spark-pipelines/requirements.txt   |  18 ++
 hudi-integ-test/spark-pipelines/test_harness.py    |  98 ++++++++
 hudi-spark-datasource/hudi-spark/pom.xml           |  48 ++++
 .../hudi/TestHoodieSparkConnectPipelines.scala     | 218 ++++++++++++++++++
 9 files changed, 962 insertions(+)

diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 923d0056e812..dcdf98a7cdfa 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -955,6 +955,42 @@ jobs:
         with:
           flags: spark-scala-tests
           token: ${{ secrets.CODECOV_TOKEN }}
+      # Run SDP only in the existing Spark 4.2 matrix entry, after its tests 
and
+      # coverage upload. A clean Spark 4.1 rebuild prevents mixed-profile 
classes.
+      - name: Set up Python for SDP
+        if: matrix.sparkProfile == 'spark4.2'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: 
hudi-integ-test/spark-pipelines/requirements.txt
+      - name: Install pinned SDP CLI dependencies
+        if: matrix.sparkProfile == 'spark4.2'
+        run: python3 -m pip install -r 
hudi-integ-test/spark-pipelines/requirements.txt
+      - name: Build Spark 4.1 bundle and run SDP process tests
+        if: matrix.sparkProfile == 'spark4.2'
+        timeout-minutes: 25
+        run: |
+          python3 -B -m unittest discover -s hudi-integ-test/spark-pipelines 
-p 'test_harness.py' -v
+          mvn clean package -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 -DskipTests \
+            $MVN_ARGS -am -pl packaging/hudi-spark-bundle
+          shopt -s nullglob
+          
bundles=(packaging/hudi-spark-bundle/target/hudi-spark4.1-bundle_2.13-*.jar)
+          candidates=()
+          for bundle in "${bundles[@]}"; do
+            [[ "$bundle" == *-sources.jar || "$bundle" == *-javadoc.jar ]] || 
candidates+=("$bundle")
+          done
+          [[ ${#candidates[@]} -eq 1 ]] || { echo "Expected one candidate 
bundle, found ${#candidates[@]}"; exit 1; }
+          python3 hudi-integ-test/spark-pipelines/harness.py \
+            --bundle "${candidates[0]}" --logs "$RUNNER_TEMP/sdp-logs"
+      - name: Upload SDP process diagnostics
+        if: always() && matrix.sparkProfile == 'spark4.2'
+        uses: actions/upload-artifact@v4
+        with:
+          name: spark-connect-layer3-logs
+          path: ${{ runner.temp }}/sdp-logs
+          if-no-files-found: ignore
+
 
   test-spark-java17-scala-other-tests:
     if: *route
@@ -1016,6 +1052,40 @@ jobs:
         with:
           flags: spark-scala-tests
           token: ${{ secrets.CODECOV_TOKEN }}
+      # Run SDP only in the existing Spark 4.2 matrix entry, after its tests 
and
+      # coverage upload. A clean Spark 4.1 rebuild prevents mixed-profile 
classes.
+      - name: Set up Python for SDP
+        if: matrix.sparkProfile == 'spark4.2'
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: pip
+          cache-dependency-path: 
hudi-integ-test/spark-pipelines/requirements.txt
+      - name: Install pinned SDP CLI dependencies
+        if: matrix.sparkProfile == 'spark4.2'
+        run: python3 -m pip install -r 
hudi-integ-test/spark-pipelines/requirements.txt
+      - name: Build Spark 4.1 dependencies and run SDP ScalaTests
+        if: matrix.sparkProfile == 'spark4.2'
+        timeout-minutes: 25
+        env:
+          HUDI_SDP_REQUIRE_PYTHON: 'true'
+        run: |
+          mvn clean install -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 -DskipTests \
+            $MVN_ARGS -am -pl hudi-spark-datasource/hudi-spark
+          mvn test -Punit-tests -Pjava17 -Dspark4.1 -Dscala-2.13 \
+            -Dtest=skipJavaTests -Dsurefire.failIfNoSpecifiedTests=false \
+            
-Dsuites=org.apache.spark.sql.connect.hudi.TestHoodieSparkConnectPipelines \
+            $MVN_ARGS -pl hudi-spark-datasource/hudi-spark
+      - name: Upload SDP ScalaTest diagnostics
+        if: always() && matrix.sparkProfile == 'spark4.2'
+        uses: actions/upload-artifact@v4
+        with:
+          name: spark-connect-layer2-logs
+          path: |
+            hudi-spark-datasource/hudi-spark/target/sdp-layer2
+            hudi-spark-datasource/hudi-spark/target/surefire-reports
+          if-no-files-found: ignore
+
 
   # [CI-TRIM] to revisit for CI improvement
 #  build-spark-java17:
diff --git a/hudi-integ-test/spark-pipelines/README.md 
b/hudi-integ-test/spark-pipelines/README.md
new file mode 100644
index 000000000000..00c65567cba1
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/README.md
@@ -0,0 +1,172 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+# Spark Connect / Declarative Pipelines integration harness
+
+The process harness runs the **real `spark-pipelines` CLI** against a 
separately launched
+Spark 4.1.1 Connect JVM with the candidate Hudi bundle. It supports Linux and
+macOS, JDK 17, and Python 3.10 or newer (CI uses Python 3.11).
+The server and clients share a local filesystem; no external storage or 
metastore
+is required. Use a dedicated Python virtual environment.
+
+## Two test layers
+
+Layer 2 invokes `python -m pyspark.pipelines.cli` with `SPARK_REMOTE`; layer 3
+also exercises the `spark-pipelines` launcher. Both use PySpark 4.1.1. Neither 
substitutes
+in-JVM `DataflowGraph` calls for the client/server protocol.
+
+| Layer | Server | Coverage | Execution |
+| --- | --- | --- | --- |
+| 2: `TestHoodieSparkConnectPipelines` | Connect service in the ScalaTest JVM, 
Hudi on the Maven test classpath | MV/ST exact rows, independent SQL flow 
insert/upsert settings, null-key failure and a fresh remote query afterward | 
Existing Scala-other job on PRs, pushes and merge groups |
+| 3: `harness.py` | Separate `spark-submit` JVM with the candidate bundle | 
Bundle packaging, static server configuration, remote writes/reads, process 
ownership and cleanup | Existing Scala-DML job on PRs, pushes and merge groups |
+
+Both layers run as additional steps in existing `bot.yml` jobs; no new 
workflow,
+job, or matrix entry is added. Layer 2 runs in 
`test-spark-java17-scala-other-tests`
+and layer 3 in `test-spark-java17-scala-dml-tests`, only in their existing 
Spark 4.2
+matrix entries. Each first completes its normal tests and coverage upload, then
+performs a clean Spark 4.1 rebuild so compiled classes cannot leak across 
profiles.
+Both jobs already feed `Java CI result`; an SDP failure therefore fails normal 
PR
+validation. The separate nightly workflow and its retry policy are removed.
+
+The November 11 guidance from Ethan and Siva permits every-PR execution with 
less
+than 30 minutes of additional runtime per job. Python setup/install, the Spark
+4.1 build and the tests must be included in that measurement. The build/test 
step
+has a 25-minute timeout to leave headroom; a timeout is a failure, not evidence
+that the performance target was met. CI timings must confirm the complete added
+cost independently for each job. There is no combined runtime budget across the
+two jobs and no new global nightly gate or bypass mechanism.
+
+### Run the layer-2 suite locally
+
+With JDK 17, install the pinned `requirements.txt` in a Python environment, 
then:
+
+```sh
+mvn install -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 -DskipTests \
+  -Dgpg.skip -Djacoco.skip -am -pl hudi-spark-datasource/hudi-spark
+HUDI_SDP_PYTHON=/path/to/venv/bin/python HUDI_SDP_REQUIRE_PYTHON=true \
+  mvn test -Punit-tests -Pjava17 -Dspark4.1 -Dscala-2.13 \
+  -Dtest=skipJavaTests -Dsurefire.failIfNoSpecifiedTests=false \
+  -Dsuites=org.apache.spark.sql.connect.hudi.TestHoodieSparkConnectPipelines \
+  -Dgpg.skip -Djacoco.skip -pl hudi-spark-datasource/hudi-spark
+```
+
+`HUDI_SDP_PYTHON` defaults to `python3`. Without usable Python dependencies the
+suite cancels its tests locally; `HUDI_SDP_REQUIRE_PYTHON=true` makes that a 
failure
+and is always set in CI. Only the Spark 4.1 profile compiles the suite or 
resolves
+the test-scoped Connect server dependency. Other Spark profiles are unaffected.
+The server binds an ephemeral loopback port and is stopped with the test 
session.
+Each CLI has a 180-second timeout; dependency and post-failure probes have 
shorter
+bounds. Generated pipelines and CLI logs remain under
+`hudi-spark-datasource/hudi-spark/target/sdp-layer2`, including after failures.
+
+## Run the layer-3 harness locally
+
+From the repository root, with `JAVA_HOME` pointing to JDK 17:
+
+```sh
+mvn package -T 2 -Pjava17 -Dspark4.1 -Dscala-2.13 \
+  -DskipTests -Dgpg.skip -Djacoco.skip -am -pl packaging/hudi-spark-bundle
+
+python3 -m venv /tmp/hudi-sdp-venv
+/tmp/hudi-sdp-venv/bin/python -m pip install \
+  -r hudi-integ-test/spark-pipelines/requirements.txt
+
+# Set this to the non-sources/non-javadoc jar produced by the build above.
+HUDI_BUNDLE="$PWD/packaging/hudi-spark-bundle/target/hudi-spark4.1-bundle_2.13-1.3.0-SNAPSHOT.jar"
+/tmp/hudi-sdp-venv/bin/python hudi-integ-test/spark-pipelines/harness.py \
+  --bundle "$HUDI_BUNDLE"
+```
+
+The last command starts the server, probes it with a remote query, runs the
+pipeline, reads the Hudi output, and stops the server. It uses the Spark server
+jars and executables installed by the pinned PySpark wheel; there is no 
separate
+Spark distribution to download and no released Hudi bundle is substituted.
+An inherited `SPARK_HOME` or Spark defaults file is not used.
+
+`pipeline.py` declares one Hudi streaming table over a deterministic JSON input
+with three rows. SDP controls the AvailableNow execution and checkpoint 
location.
+A second Connect client checks all business columns by both catalog name and 
Hudi
+path, verifies `.hoodie/hoodie.properties`, and the harness checks that SDP 
wrote
+a streaming offset under its checkpoint directory. Every run starts with new
+table, warehouse, input, Spark scratch, and checkpoint directories.
+
+This is infrastructure smoke coverage for 
[#20402](https://github.com/apache/hudi/issues/20402).
+It does not establish the full RFC-112 contract: repeat materialized-view 
refresh,
+streaming configuration forwarding, restart/full-refresh semantics, chaining,
+and keyed update/delete acceptance belong to the sibling tasks under
+[#20401](https://github.com/apache/hudi/issues/20401).
+
+## Configuration ownership
+
+| Configuration | Owner / location |
+| --- | --- |
+| Candidate bundle, Hudi session extension and `spark_catalog`, Kryo 
serializer/registrator | `ConnectHarness.server_configuration()` and server 
`spark-submit` arguments, before JVM startup |
+| Warehouse, in-memory catalog, local Hadoop filesystem, driver binding, 
Connect port, scratch storage | Server startup configuration |
+| Pipeline storage/checkpoint root, catalog and database | Generated 
`spark-pipeline.yml` |
+| Session shuffle partitions, SDP retry limit, input path | Pipeline YAML 
`configuration` |
+| Flow-specific shuffle partitions | `@dp.table(spark_conf=...)` in 
`pipeline.py` |
+| Hudi table type, record key, precombine field | Dataset `table_properties` 
in `pipeline.py` |
+
+Static server settings must not be passed through the pipeline YAML or CLI
+`--conf`. The readiness client verifies the startup extension/catalog and that
+the extension is not session-modifiable. The CLI always receives
+`--remote sc://127.0.0.1:<port>`; it must not start its own local server.
+All data paths are absolute URIs accessible to the server and its local 
executors.
+Tests for other storage backends must install connectors and configure storage
+access on the server at startup, before creating client sessions.
+
+## Failures and cleanup
+
+`--startup-timeout` (default 90 seconds) bounds the port wait and the readiness
+client individually. `--client-timeout` (default 180 seconds) bounds each CLI 
or
+verification process, including Spark Connect retries. SDP flow retries are
+disabled in this smoke test to report the first failure. A server exit 
interrupts
+a waiting client. All failures produce a nonzero harness exit code.
+
+Each subprocess starts in its own process group. On success, exceptions,
+timeouts, Ctrl-C, or SIGTERM, the harness sends SIGTERM and then SIGKILL after 
a
+five-second grace period if necessary, reaps the launcher, and removes the test
+directories. It does not use `pkill`, Spark's shared PID files, or the global
+`stop-connect-server.sh`. A force-killed harness cannot run cleanup; the CI job
+has an outer timeout and runs on a disposable runner.
+
+Logs survive cleanup in a unique run directory under 
`target/spark-pipelines-logs`
+(override with `--logs`). `server.log`, `probe.log`, `pipeline.log`, 
`verify.log`,
+the server configuration, and the generated pipeline specification provide the
+execution context. On failure, the final 60 lines of each available process log
+are printed. CI uploads the log directory on both success and failure.
+
+## Extend the suite
+
+Use `ConnectHarness` as a context manager. Its `root` is isolated local 
storage;
+`remote` is the explicit Connect URL. Write a scenario's definitions and spec
+under `root`, then call `run_pipeline(spec, name)` and `client(script, name)`.
+Each client script receives `name`, `remote`, and `root` as positional 
arguments.
+Use distinct log names for successive calls. Spark calls belong in those 
bounded
+client subprocesses, never in the supervising harness. Keep assertions in the
+scenario's verification client and let failures propagate.
+
+Fast lifecycle regression tests require only the Python standard library:
+
+```sh
+python3 -B -m unittest discover -s hudi-integ-test/spark-pipelines -p 
'test_*.py' -v
+```
+
+The existing Scala-DML job in `.github/workflows/bot.yml` runs the lifecycle 
tests,
+builds the candidate Spark 4.1 / Scala 2.13 bundle, and executes the 
separate-server
+harness on every PR. The existing Scala-other job runs the layer-2 Maven suite.
+The commands above also support local validation before pushing.
diff --git a/hudi-integ-test/spark-pipelines/client.py 
b/hudi-integ-test/spark-pipelines/client.py
new file mode 100644
index 000000000000..589961533de3
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/client.py
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""All Spark operations here use a remote session; the harness bounds this 
process."""
+
+from pathlib import Path
+import sys
+
+from pyspark.sql import SparkSession
+from pyspark.sql.connect.session import SparkSession as ConnectSession
+
+
+def main():
+    mode, remote, root_arg = sys.argv[1:]
+    root = Path(root_arg)
+    spark = SparkSession.builder.remote(remote).getOrCreate()
+    try:
+        assert isinstance(spark, ConnectSession), "Expected a remote Spark 
Connect session"
+        assert spark.version == "4.1.1", spark.version
+        assert spark.conf.get("spark.hudi.test.run.id") == root.name, 
"Connected to another test's server"
+        if mode == "probe":
+            assert spark.range(1).count() == 1
+            assert spark.conf.get("spark.sql.extensions") == 
"org.apache.spark.sql.hudi.HoodieSparkSessionExtension"
+            assert not spark.conf.isModifiable("spark.sql.extensions")
+            assert spark.conf.get("spark.sql.catalog.spark_catalog") == 
"org.apache.spark.sql.hudi.catalog.HoodieCatalog"
+            spark.sql("CREATE DATABASE sdp_test").collect()
+            print("Remote Spark 4.1.1 server ready with Hudi extension and 
catalog")
+        elif mode == "verify":
+            expected = [(i, 100 + i, f"value-{i}") for i in range(1, 4)]
+            table = "spark_catalog.sdp_test.events"
+            actual = [tuple(row) for row in spark.table(table).select("id", 
"ts", "value").orderBy("id").collect()]
+            assert actual == expected, (actual, expected)
+            location = root / "warehouse/sdp_test.db/events"
+            assert (location / ".hoodie/hoodie.properties").is_file(), 
"Missing Hudi table metadata"
+            by_path = 
spark.read.format("hudi").load(location.as_uri()).select("id", "ts", 
"value").orderBy("id").collect()
+            assert [tuple(row) for row in by_path] == expected, by_path
+            print(f"Verified catalog and Hudi path reads: {actual}")
+        else:
+            raise ValueError(f"Unknown client mode: {mode}")
+    finally:
+        spark.stop()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/hudi-integ-test/spark-pipelines/harness.py 
b/hudi-integ-test/spark-pipelines/harness.py
new file mode 100644
index 000000000000..a7978b590c1e
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/harness.py
@@ -0,0 +1,251 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Owned, foreground Spark Connect server and bounded SDP client processes 
(POSIX)."""
+
+import argparse
+from collections import deque
+import contextlib
+import json
+import os
+from pathlib import Path
+import signal
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+
+
+SPARK_VERSION = "4.1.1"
+HERE = Path(__file__).resolve().parent
+
+
[email protected]
+def owned_process(command, log, cwd, env):
+    """Kill only the new process group, including children left by an exited 
launcher."""
+    with log.open("w") as output:
+        output.write("Command: " + repr([str(arg) for arg in command]) + "\n")
+        output.flush()
+        process = subprocess.Popen(command, cwd=cwd, env=env, stdout=output,
+                                   stderr=subprocess.STDOUT, 
start_new_session=True)
+        try:
+            yield process
+        finally:
+            stop_process_group(process)
+
+
+def stop_process_group(process, grace=5):
+    # Retain the leader until its group has been signalled; never use pkill or 
Spark's global stop script.
+    try:
+        os.killpg(process.pid, signal.SIGTERM)
+    except ProcessLookupError:
+        process.wait()
+        return
+    deadline = time.monotonic() + grace
+    while time.monotonic() < deadline:
+        process.poll()
+        try:
+            os.killpg(process.pid, 0)
+        except ProcessLookupError:
+            break
+        time.sleep(0.1)
+    else:
+        with contextlib.suppress(ProcessLookupError):
+            os.killpg(process.pid, signal.SIGKILL)
+    process.wait(timeout=grace)
+
+
+def wait_for_process(process, timeout, log, server=None):
+    deadline = time.monotonic() + timeout
+    while process.poll() is None:
+        if server is not None and server.poll() is not None:
+            raise RuntimeError(f"Connect server exited with 
{server.returncode}; see server.log and {log}")
+        if time.monotonic() >= deadline:
+            raise TimeoutError(f"Client exceeded {timeout}s; see {log}")
+        time.sleep(0.1)
+    if process.returncode:
+        raise RuntimeError(f"Client exited with {process.returncode}; see 
{log}")
+
+
+class ConnectHarness:
+    def __init__(self, bundle, logs, startup_timeout=90, client_timeout=180):
+        import pyspark
+
+        if pyspark.__version__ != SPARK_VERSION:
+            raise ValueError(f"Install requirements.txt: expected PySpark 
{SPARK_VERSION}, got {pyspark.__version__}")
+        self.spark_home = Path(pyspark.__file__).resolve().parent
+        self.bundle = Path(bundle).resolve(strict=True)
+        if not self.bundle.is_file() or self.bundle.suffix != ".jar":
+            raise ValueError("--bundle must name the built Hudi Spark 4.1 
bundle jar")
+        self.logs = Path(logs).resolve()
+        self.logs.mkdir(parents=True, exist_ok=True)
+        self.startup_timeout = startup_timeout
+        self.client_timeout = client_timeout
+        self.server = None
+        self.stack = contextlib.ExitStack()
+
+    def __enter__(self):
+        try:
+            self.root = 
Path(self.stack.enter_context(tempfile.TemporaryDirectory(prefix="hudi-sdp-")))
+            with socket.socket() as sock:
+                sock.bind(("127.0.0.1", 0))
+                self.port = sock.getsockname()[1]
+            self.remote = f"sc://127.0.0.1:{self.port}"
+            self.env = self.process_environment()
+            conf = self.server_configuration()
+            (self.logs / "server-conf.json").write_text(json.dumps(conf, 
indent=2) + "\n")
+            command = [str(self.spark_home / "bin/spark-submit"), "--master", 
"local[2]",
+                       "--driver-memory", "1g", "--jars", str(self.bundle)]
+            for key, value in conf.items():
+                command.extend(["--conf", f"{key}={value}"])
+            command.extend(["--class", 
"org.apache.spark.sql.connect.service.SparkConnectServer"])
+            self.server = self.stack.enter_context(owned_process(
+                command, self.logs / "server.log", self.root, self.env))
+            # A TCP check alone can accept an unrelated listener. The probe 
below also verifies our run ID.
+            self.wait_for_port()
+            self.client(HERE / "client.py", "probe", 
timeout=self.startup_timeout)
+            return self
+        except BaseException:
+            self.stack.close()
+            raise
+
+    def __exit__(self, *exc):
+        self.stack.close()
+
+    def process_environment(self):
+        env = os.environ.copy()
+        for key in ("SPARK_REMOTE", "SPARK_CONNECT_MODE_ENABLED", 
"SPARK_LOCAL_REMOTE", "SPARK_CONNECT_MODE",
+                    "PYSPARK_SUBMIT_ARGS", "PYSPARK_GATEWAY_PORT", 
"PYSPARK_GATEWAY_SECRET",
+                    "SPARK_SUBMIT_OPTS", "SPARK_JAVA_OPTS", "PYTHONPATH"):
+            env.pop(key, None)
+        conf_dir = self.root / "conf"
+        conf_dir.mkdir()
+        env.update(SPARK_HOME=str(self.spark_home), 
SPARK_CONF_DIR=str(conf_dir),
+                   SPARK_LOCAL_IP="127.0.0.1", PYSPARK_PYTHON=sys.executable,
+                   PYSPARK_DRIVER_PYTHON=sys.executable, PYTHONUNBUFFERED="1")
+        return env
+
+    def server_configuration(self):
+        # Static/driver/storage configuration belongs here, never in a 
pipeline's session configuration.
+        return {
+            "spark.sql.extensions": 
"org.apache.spark.sql.hudi.HoodieSparkSessionExtension",
+            "spark.sql.catalog.spark_catalog": 
"org.apache.spark.sql.hudi.catalog.HoodieCatalog",
+            "spark.serializer": "org.apache.spark.serializer.KryoSerializer",
+            "spark.kryo.registrator": 
"org.apache.spark.HoodieSparkKryoRegistrar",
+            "spark.sql.catalogImplementation": "in-memory",
+            "spark.sql.warehouse.dir": (self.root / "warehouse").as_uri(),
+            "spark.hadoop.fs.defaultFS": "file:///",
+            "spark.local.dir": str(self.root / "spark-local"),
+            "spark.driver.host": "127.0.0.1",
+            "spark.driver.bindAddress": "127.0.0.1",
+            "spark.connect.grpc.binding.address": "127.0.0.1",
+            "spark.connect.grpc.binding.port": str(self.port),
+            "spark.port.maxRetries": "0",
+            "spark.ui.enabled": "false",
+            "spark.hudi.test.run.id": self.root.name,
+        }
+
+    def wait_for_port(self):
+        deadline = time.monotonic() + self.startup_timeout
+        while time.monotonic() < deadline:
+            if self.server.poll() is not None:
+                raise RuntimeError(f"Connect server exited with 
{self.server.returncode}; see server.log")
+            try:
+                with socket.create_connection(("127.0.0.1", self.port), 
timeout=0.5):
+                    return
+            except OSError:
+                time.sleep(0.2)
+        raise TimeoutError(f"Connect server startup exceeded 
{self.startup_timeout}s; see server.log")
+
+    def run(self, command, name, timeout=None):
+        log = self.logs / f"{name}.log"
+        with owned_process(command, log, self.root, self.env) as process:
+            wait_for_process(process, timeout or self.client_timeout, log, 
self.server)
+
+    def client(self, script, name, timeout=None):
+        self.run([sys.executable, str(script), name, self.remote, 
str(self.root)], name, timeout)
+
+    def run_pipeline(self, spec, name="pipeline"):
+        self.run([str(self.spark_home / "bin/spark-pipelines"), "run", 
"--remote", self.remote,
+                  "--spec", str(spec)], name)
+
+    def smoke(self):
+        input_dir = self.root / "input"
+        input_dir.mkdir()
+        rows = [{"id": i, "ts": 100 + i, "value": f"value-{i}"} for i in 
range(1, 4)]
+        (input_dir / "events.json").write_text("".join(json.dumps(row) + "\n" 
for row in rows))
+        spec = {
+            "name": "hudi_connect_smoke",
+            "storage": (self.root / "checkpoints").as_uri(),
+            "catalog": "spark_catalog",
+            "database": "sdp_test",
+            "libraries": [{"glob": {"include": "pipeline.py"}}],
+            "configuration": {
+                "spark.sql.shuffle.partitions": "2",
+                "spark.sql.pipelines.maxFlowRetryAttempts": "0",
+                "hudi.test.input": input_dir.as_uri(),
+            },
+        }
+        (self.root / "pipeline.py").write_text((HERE / 
"pipeline.py").read_text())
+        spec_path = self.root / "spark-pipeline.yml"
+        # JSON is a YAML subset; keep harness lifecycle tests independent of 
third-party Python packages.
+        spec_path.write_text(json.dumps(spec, indent=2) + "\n")
+        (self.logs / spec_path.name).write_text(spec_path.read_text())
+        self.run_pipeline(spec_path)
+        self.client(HERE / "client.py", "verify")
+        if not list((self.root / "checkpoints").rglob("offsets/0")):
+            raise AssertionError("SDP did not create a streaming checkpoint 
under its isolated storage directory")
+
+
+def positive_seconds(value):
+    seconds = int(value)
+    if seconds <= 0:
+        raise argparse.ArgumentTypeError("timeout must be positive")
+    return seconds
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--bundle", required=True, type=Path)
+    parser.add_argument("--logs", type=Path, 
default=Path("target/spark-pipelines-logs"))
+    parser.add_argument("--startup-timeout", type=positive_seconds, default=90)
+    parser.add_argument("--client-timeout", type=positive_seconds, default=180)
+    args = parser.parse_args()
+    args.logs.mkdir(parents=True, exist_ok=True)
+    logs = Path(tempfile.mkdtemp(prefix="run-", dir=args.logs.resolve()))
+    print(f"Spark Connect SDP logs: {logs}", flush=True)
+    # Let CI cancellation reach ExitStack, just like Ctrl-C; each child has 
its own process group.
+    def interrupted(signum, frame):
+        raise KeyboardInterrupt(f"Received signal {signum}")
+    signal.signal(signal.SIGTERM, interrupted)
+    try:
+        with ConnectHarness(args.bundle, logs, args.startup_timeout, 
args.client_timeout) as harness:
+            harness.smoke()
+    except (Exception, KeyboardInterrupt) as error:
+        print(f"FAILED: {error}", file=sys.stderr)
+        for log in sorted(logs.glob("*.log")):
+            print(f"\n--- {log.name} (last 60 lines) ---", file=sys.stderr)
+            with log.open(errors="replace") as output:
+                print("".join(deque(output, maxlen=60)), file=sys.stderr)
+        return 1
+    print("PASS: spark-pipelines wrote the expected Hudi rows through Spark 
Connect; resources removed.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/hudi-integ-test/spark-pipelines/pipeline.py 
b/hudi-integ-test/spark-pipelines/pipeline.py
new file mode 100644
index 000000000000..57926915b3bf
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/pipeline.py
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyspark import pipelines as dp
+from pyspark.sql import SparkSession
+
+spark = SparkSession.active()
+
+
[email protected](name="events", format="hudi", table_properties={
+    "type": "cow", "primaryKey": "id", "preCombineField": "ts",
+}, spark_conf={"spark.sql.shuffle.partitions": "2"})
+def events():
+    return spark.readStream.schema("id INT, ts LONG, value 
STRING").json(spark.conf.get("hudi.test.input"))
diff --git a/hudi-integ-test/spark-pipelines/requirements.txt 
b/hudi-integ-test/spark-pipelines/requirements.txt
new file mode 100644
index 000000000000..f27c90ed6b76
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/requirements.txt
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+pyspark[pipelines,connect]==4.1.1
diff --git a/hudi-integ-test/spark-pipelines/test_harness.py 
b/hudi-integ-test/spark-pipelines/test_harness.py
new file mode 100644
index 000000000000..a2969594f51b
--- /dev/null
+++ b/hudi-integ-test/spark-pipelines/test_harness.py
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Fast lifecycle regressions; no Spark installation is needed for these 
tests."""
+
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+from harness import ConnectHarness, owned_process, stop_process_group, 
wait_for_process
+
+
+class TestProcessLifecycle(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.log = self.root / "client.log"
+
+    def process(self, code):
+        return owned_process([sys.executable, "-u", "-c", code], self.log, 
self.root, os.environ.copy())
+
+    def test_client_failure_keeps_diagnostics(self):
+        with self.process("print('intentional client failure'); raise 
SystemExit(7)") as process:
+            with self.assertRaisesRegex(RuntimeError, "exited with 7"):
+                wait_for_process(process, 5, self.log)
+        self.assertIn("intentional client failure", self.log.read_text())
+
+    def 
test_timeout_terminates_owned_process_without_stopping_another_server(self):
+        other = subprocess.Popen([sys.executable, "-c", "import time; 
time.sleep(60)"], start_new_session=True)
+        self.addCleanup(stop_process_group, other)
+        with self.assertRaises(TimeoutError):
+            with self.process("import time; time.sleep(60)") as process:
+                wait_for_process(process, 0.2, self.log)
+        self.assertIsNotNone(process.poll())
+        self.assertIsNone(other.poll())
+
+    def test_server_exit_interrupts_client(self):
+        server = subprocess.Popen([sys.executable, "-c", "raise 
SystemExit(9)"])
+        server.wait(timeout=5)
+        started = time.monotonic()
+        with self.assertRaisesRegex(RuntimeError, "server exited with 9"):
+            with self.process("import time; time.sleep(60)") as process:
+                wait_for_process(process, 30, self.log, server)
+        self.assertLess(time.monotonic() - started, 10)
+        self.assertIsNotNone(process.poll())
+
+    def test_kills_process_ignoring_term(self):
+        command = [sys.executable, "-u", "-c",
+                   "import signal,time; signal.signal(signal.SIGTERM, 
signal.SIG_IGN); print('ready'); time.sleep(60)"]
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, text=True, 
start_new_session=True)
+        self.addCleanup(stop_process_group, process, 0.2)
+        self.addCleanup(process.stdout.close)
+        self.assertEqual(process.stdout.readline().strip(), "ready")
+        stop_process_group(process, grace=0.2)
+        self.assertEqual(process.returncode, -9)
+
+    def test_startup_failure_cleans_server_and_test_resources(self):
+        spark_home = self.root / "pyspark"
+        (spark_home / "bin").mkdir(parents=True)
+        submit = spark_home / "bin/spark-submit"
+        submit.write_text(f"#!{sys.executable}\nimport time\ntime.sleep(60)\n")
+        submit.chmod(0o755)
+        bundle = self.root / "bundle.jar"
+        bundle.touch()
+        fake_pyspark = SimpleNamespace(__version__="4.1.1", 
__file__=str(spark_home / "__init__.py"))
+        with patch.dict(sys.modules, pyspark=fake_pyspark):
+            harness = ConnectHarness(bundle, self.root / "logs", 
startup_timeout=0.2)
+        with self.assertRaisesRegex(TimeoutError, "startup exceeded"):
+            with harness:
+                self.fail("A server which never listens must not become ready")
+        self.assertFalse(harness.root.exists())
+        self.assertIsNotNone(harness.server.poll())
+        self.assertTrue((harness.logs / "server.log").is_file())
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/hudi-spark-datasource/hudi-spark/pom.xml 
b/hudi-spark-datasource/hudi-spark/pom.xml
index 8b3daf01fdfe..ca4a1fd4a29b 100644
--- a/hudi-spark-datasource/hudi-spark/pom.xml
+++ b/hudi-spark-datasource/hudi-spark/pom.xml
@@ -525,6 +525,54 @@
   </dependencies>
 
   <profiles>
+    <!-- SDP is a Spark 4.1 API. Keep its server dependency and tests out of 
other profiles. -->
+    <profile>
+      <id>spark4.1</id>
+      <activation>
+        <property><name>spark4.1</name></property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>spark-connect_${scala.binary.version}</artifactId>
+          <version>${spark41.version}</version>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>add-sdp-test-source</id>
+                <phase>generate-test-sources</phase>
+                <goals><goal>add-test-source</goal></goals>
+                <configuration>
+                  <sources><source>src/test/scala-spark4.1</source></sources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.scalastyle</groupId>
+            <artifactId>scalastyle-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>check-sdp-test-style</id>
+                <phase>test-compile</phase>
+                <goals><goal>check</goal></goals>
+                <configuration>
+                  
<testSourceDirectory>${project.basedir}/src/test/scala-spark4.1</testSourceDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
     <!-- Vortex requires Java 17 (vortex-jni/vortex-spark ship Java 17 
bytecode). The Vortex
          round-trip test compiles/runs only under JDK 17; Java 11 builds skip 
it. -->
     <profile>
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala-spark4.1/org/apache/spark/sql/connect/hudi/TestHoodieSparkConnectPipelines.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala-spark4.1/org/apache/spark/sql/connect/hudi/TestHoodieSparkConnectPipelines.scala
new file mode 100644
index 000000000000..5436d3efac97
--- /dev/null
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala-spark4.1/org/apache/spark/sql/connect/hudi/TestHoodieSparkConnectPipelines.scala
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connect.hudi
+
+import org.apache.spark.sql.{Row, SparkSession}
+import org.apache.spark.sql.connect.service.SparkConnectService
+import org.apache.spark.util.Utils
+import org.scalatest.{BeforeAndAfterAll, FunSuite}
+
+import java.io.{File, IOException}
+import java.nio.charset.StandardCharsets.UTF_8
+import java.nio.file.Files
+import java.util.UUID
+import java.util.concurrent.TimeUnit
+
+/** Real pipelines CLI against a Connect service in the test JVM, without 
building a bundle. */
+class TestHoodieSparkConnectPipelines extends FunSuite with BeforeAndAfterAll {
+  private val python = sys.env.getOrElse("HUDI_SDP_PYTHON", "python3")
+  private val logs = new File("target/sdp-layer2", UUID.randomUUID().toString)
+  private var spark: SparkSession = _
+  private var root: File = _
+  private var started = false
+
+  private lazy val pythonAvailable: Boolean = {
+    logs.mkdirs()
+    val probe = "import pyspark; from pyspark.pipelines import cli; " +
+      "from pyspark.sql.connect.session import SparkSession; " +
+      "assert pyspark.__version__ == '4.1.1', pyspark.__version__"
+    try {
+      runPython(Seq("-c", probe), "python-probe", 30)._1 == 0
+    } catch {
+      case _: IOException => false
+    }
+  }
+
+  private def startServer(): Unit = {
+    if (sys.env.get("HUDI_SDP_REQUIRE_PYTHON").contains("true")) {
+      assert(pythonAvailable, s"CI requires PySpark 4.1.1 pipelines/connect; 
see $logs")
+    } else {
+      assume(pythonAvailable, "Install the SDP requirements and set 
HUDI_SDP_PYTHON to enable this suite")
+    }
+    if (!started) {
+      root = Utils.createTempDir(namePrefix = "hudi-sdp-layer2")
+      spark = SparkSession.builder().master("local[2]").appName("Hudi SDP CLI 
tests")
+        .config("spark.ui.enabled", "false")
+        .config("spark.sql.warehouse.dir", new File(root, 
"warehouse").toURI.toString)
+        .config("spark.sql.catalogImplementation", "in-memory")
+        .config("spark.sql.extensions", 
"org.apache.spark.sql.hudi.HoodieSparkSessionExtension")
+        .config("spark.sql.catalog.spark_catalog", 
"org.apache.spark.sql.hudi.catalog.HoodieCatalog")
+        .config("spark.serializer", 
"org.apache.spark.serializer.KryoSerializer")
+        .config("spark.kryo.registrator", 
"org.apache.spark.HoodieSparkKryoRegistrar")
+        .config("spark.connect.grpc.binding.address", "127.0.0.1")
+        .config("spark.connect.grpc.binding.port", "0")
+        .getOrCreate()
+      SparkConnectService.start(spark.sparkContext)
+      started = true
+    }
+  }
+
+  test("CLI materialized view and streaming table write exact Hudi rows") {
+    withPipeline { (dir, database) =>
+      write(dir, "flows.sql",
+        """CREATE MATERIALIZED VIEW mv USING hudi
+          |TBLPROPERTIES ('type'='cow', 'primaryKey'='id', 
'preCombineField'='ts')
+          |AS SELECT 2 AS id, 2L AS ts, 'mv' AS value;
+          |CREATE STREAMING TABLE events USING hudi
+          |TBLPROPERTIES ('type'='cow', 'primaryKey'='id', 
'preCombineField'='ts')
+          |AS SELECT * FROM STREAM input_events;
+          |""".stripMargin)
+      assertSuccess(runPipeline(dir, database))
+      assert(rows(database, "mv") == Seq(Row(2, 2L, "mv")))
+      assert(rows(database, "events") == Seq(Row(1, 1L, "old")))
+    }
+  }
+
+  test("flow-scoped SQL settings independently select insert and upsert") {
+    withPipeline { (dir, database) =>
+      write(dir, "flows.sql", Seq("insert", "upsert").map { operation =>
+        s"""SET spark.hoodie.datasource.write.operation=$operation;
+           |CREATE STREAMING TABLE ${operation}_events USING hudi
+           |TBLPROPERTIES ('type'='cow', 'primaryKey'='id', 
'preCombineField'='ts')
+           |AS SELECT * FROM STREAM input_events;
+           |""".stripMargin
+      }.mkString("\n"))
+      assertSuccess(runPipeline(dir, database))
+      assert(rows(database, "insert_events") == Seq.fill(2)(Row(1, 1L, "old")))
+      assert(rows(database, "upsert_events") == Seq(Row(1, 1L, "old")))
+    }
+  }
+
+  test("a failing CLI flow reports the key error and leaves Connect usable") {
+    withPipeline { (dir, database) =>
+      write(dir, "flows.sql",
+        """CREATE STREAMING TABLE bad_events USING hudi
+          |TBLPROPERTIES ('type'='cow', 'primaryKey'='id', 
'preCombineField'='ts')
+          |AS SELECT CAST(NULL AS INT) AS id, ts, value FROM STREAM 
input_events;
+          |""".stripMargin)
+      val (exit, output) = runPipeline(dir, database)
+      assert(exit != 0, "The null-key flow unexpectedly succeeded")
+      assert(output.contains("recordKey") && output.contains("null"), 
output.takeRight(12000))
+      val probe = "from pyspark.sql import SparkSession; " +
+        s"s = SparkSession.builder.remote('$remote').getOrCreate(); " +
+        "assert s.range(3).count() == 3; s.stop()"
+      assertSuccess(runPython(Seq("-c", probe), s"$database-after-failure", 
60))
+    }
+  }
+
+  private def remote: String = 
s"sc://127.0.0.1:${SparkConnectService.localPort}"
+
+  private def withPipeline(f: (File, String) => Unit): Unit = {
+    startServer()
+    val database = "sdp_" + UUID.randomUUID().toString.replace("-", "")
+    val dir = new File(root, database)
+    dir.mkdirs()
+    spark.sql(s"CREATE DATABASE $database").collect()
+    val input = new File(dir, "input")
+    input.mkdirs()
+    write(input, "events.json", """{"id":1,"ts":1,"value":"old"}
+      |{"id":1,"ts":1,"value":"old"}
+      |""".stripMargin)
+    write(dir, "source.py",
+      """from pyspark import pipelines as dp
+        |from pyspark.sql import SparkSession
+        |spark = SparkSession.active()
+        |@dp.temporary_view(name="input_events")
+        |def input_events():
+        |    return spark.readStream.schema("id INT, ts LONG, value 
STRING").json(spark.conf.get("hudi.test.input"))
+        |""".stripMargin)
+    try f(dir, database) finally {
+      spark.sql(s"DROP DATABASE $database CASCADE").collect()
+    }
+  }
+
+  private def runPipeline(dir: File, database: String): (Int, String) = {
+    def quoted(value: String): String = "'" + value.replace("'", "''") + "'"
+    write(dir, "spark-pipeline.yml",
+      s"""name: $database
+         |catalog: spark_catalog
+         |database: $database
+         |storage: ${quoted(new File(dir, "checkpoints").toURI.toString)}
+         |libraries:
+         |  - glob: {include: source.py}
+         |  - glob: {include: flows.sql}
+         |configuration:
+         |  hudi.test.input: ${quoted(new File(dir, "input").toURI.toString)}
+         |  spark.sql.shuffle.partitions: '2'
+         |  spark.sql.pipelines.maxFlowRetryAttempts: '0'
+         |  spark.hoodie.metadata.enable: 'true'
+         |""".stripMargin)
+    Seq("source.py", "flows.sql", "spark-pipeline.yml").foreach { name =>
+      Files.copy(new File(dir, name).toPath, new File(logs, 
s"$database-$name").toPath)
+    }
+    runPython(Seq("-m", "pyspark.pipelines.cli", "run",
+      "--spec", new File(dir, "spark-pipeline.yml").getAbsolutePath), 
database, 180)
+  }
+
+  private def rows(database: String, table: String): Seq[Row] = {
+    spark.table(s"$database.$table").select("id", "ts", "value").orderBy("id", 
"ts").collect().toSeq
+  }
+
+  private def write(dir: File, name: String, value: String): Unit = {
+    Files.write(new File(dir, name).toPath, value.getBytes(UTF_8))
+  }
+
+  private def assertSuccess(result: (Int, String)): Unit = {
+    assert(result._1 == 0, result._2.takeRight(12000))
+  }
+
+  private def runPython(args: Seq[String], name: String, timeout: Long): (Int, 
String) = {
+    val log = new File(logs, s"$name.log")
+    val builder = new ProcessBuilder((Seq(python) ++ args): _*)
+      .redirectErrorStream(true).redirectOutput(log)
+    Seq("SPARK_REMOTE", "SPARK_CONNECT_MODE_ENABLED", "SPARK_LOCAL_REMOTE", 
"SPARK_CONNECT_MODE",
+      "PYSPARK_GATEWAY_PORT", "PYSPARK_GATEWAY_SECRET", 
"PYTHONPATH").foreach(builder.environment().remove)
+    builder.environment().put("PYTHONUNBUFFERED", "1")
+    // The spark-pipelines launcher translates --remote into SPARK_REMOTE 
before invoking cli.py.
+    if (started) builder.environment().put("SPARK_REMOTE", remote)
+    val process = builder.start()
+    try {
+      assert(process.waitFor(timeout, TimeUnit.SECONDS), s"CLI timed out after 
${timeout}s; see $log")
+      (process.exitValue(), new String(Files.readAllBytes(log.toPath), UTF_8))
+    } finally {
+      process.destroy()
+      if (!process.waitFor(5, TimeUnit.SECONDS)) {
+        process.destroyForcibly()
+        process.waitFor(5, TimeUnit.SECONDS)
+      }
+    }
+  }
+
+  override protected def afterAll(): Unit = {
+    try {
+      if (spark != null) {
+        try SparkConnectService.stop() finally spark.stop()
+      }
+    } finally {
+      if (root != null) Utils.deleteRecursively(root)
+      SparkSession.clearActiveSession()
+      SparkSession.clearDefaultSession()
+      super.afterAll()
+    }
+  }
+}

Reply via email to