This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-20462-6766f0dcf308ea2ffe978a6dd9747e7bb2df2258
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 57abf057f968e3d4f9caa9b61d7c6269d5620a63
Author: Shuo Cheng <[email protected]>
AuthorDate: Tue Nov 10 09:27:30 2026 +0000

    test(spark): add Spark Connect quickstart validation (#20462)
    
    * test(spark): add Spark Connect quickstart validation
    
    * fix(spark): bound fault experiment readbacks
    
    * fix(spark): address endpoint validation review feedback
    
    * fix(spark): reduce repeated cancellation status logs
---
 .github/workflows/hudi_spark_connect_live.yml      |  53 +++
 hudi-agent-gateway/README.md                       |   5 +
 .../tests/integration/test_spark_connect_live.py   |  46 +++
 hudi-lakehouse/charts/hudi-spark-connect/README.md | 290 +++++++++++++-
 .../hudi-spark-connect/templates/deployment.yaml   |  20 +-
 .../charts/hudi-spark-connect/values.schema.json   |  27 ++
 .../charts/hudi-spark-connect/values.yaml          |   8 +
 hudi-lakehouse/images/spark-3.5-writer/Dockerfile  |  27 ++
 hudi-lakehouse/images/spark-connect/Dockerfile     |   1 +
 hudi-lakehouse/images/spark-connect/check-ready.py |  70 ++++
 hudi-lakehouse/local-dev/example/connect_cancel.py | 141 +++++++
 .../local-dev/example/connect_disconnect.py        | 147 +++++++
 .../local-dev/example/connect_external_writer.py   |   6 +-
 .../local-dev/example/connect_fault_utils.py       | 111 ++++++
 hudi-lakehouse/local-dev/example/connect_load.py   |  58 +++
 .../local-dev/example/connect_lost_response.py     | 195 +++++++++
 .../local-dev/example/connect_nullable_array.py    |  56 +++
 .../local-dev/example/connect_readiness.py         |  64 +++
 .../local-dev/example/connect_restart_fault.py     | 262 ++++++++++++
 .../tests/test_connect_external_writer.py          |  11 +
 .../tests/test_connect_fault_regressions.py        | 437 +++++++++++++++++++++
 hudi-lakehouse/tests/test_connect_lost_response.py |  72 ++++
 hudi-lakehouse/tests/test_connect_probe.py         |  80 ++++
 hudi-lakehouse/tests/test_connect_readiness.py     |  65 +++
 hudi-lakehouse/tests/test_spark_connect_chart.py   |  13 +
 25 files changed, 2251 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/hudi_spark_connect_live.yml 
b/.github/workflows/hudi_spark_connect_live.yml
new file mode 100644
index 000000000000..924af762a5c1
--- /dev/null
+++ b/.github/workflows/hudi_spark_connect_live.yml
@@ -0,0 +1,53 @@
+name: Hudi Spark Connect live acceptance
+
+# Explicit opt-in: this creates a disposable kind cluster and runs real Spark 
SQL.
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  live-acceptance:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    env:
+      KUBECONFIG: ${{ runner.temp }}/hudi-connect-kubeconfig
+    steps:
+      - uses: actions/checkout@v5
+      - name: Check runner tools
+        run: |
+          set -euo pipefail
+          command -v docker
+          command -v kind
+          command -v kubectl
+          command -v helm
+          command -v python3
+      - name: Create isolated cluster and build image
+        run: |
+          set -euo pipefail
+          kind create cluster --name hudi-connect-ci --kubeconfig 
"$KUBECONFIG" \
+            --image 
kindest/node:v1.35.8@sha256:07b2536e30b803ed61d1677a79df6115f798ce64c80f9e22f6ed45afd09323c0
 --wait 120s
+          ./hudi-lakehouse/scripts/build-connect-image.sh
+          kind load docker-image hudi-lakehouse-spark-connect:4.1.3-hudi1.2.0 
--name hudi-connect-ci
+      - name: Deploy local lakehouse
+        run: ./hudi-lakehouse/local-dev/scripts/up.sh --spark-connect
+      - name: Install ADBC client
+        run: |
+          set -euo pipefail
+          python3 -m venv "$RUNNER_TEMP/connect-venv"
+          "$RUNNER_TEMP/connect-venv/bin/pip" install -r 
hudi-lakehouse/local-dev/example/connect-requirements.txt dbc==0.3.1
+          "$RUNNER_TEMP/connect-venv/bin/dbc" install 'spark=0.2.1'
+      - name: Verify SQL and persistence after restart
+        env:
+          PYTHON: ${{ runner.temp }}/connect-venv/bin/python
+        run: ./hudi-lakehouse/local-dev/scripts/test-spark-connect.sh 
--restart-endpoint --sql-matrix
+      - name: Diagnostics on failure
+        if: failure()
+        run: |
+          kubectl -n hudi-lakehouse get pods -o wide || true
+          kubectl -n hudi-lakehouse describe pods || true
+          kubectl -n hudi-lakehouse logs deploy/hudi-spark-connect --tail=200 
|| true
+      - name: Delete isolated cluster
+        if: always()
+        run: kind delete cluster --name hudi-connect-ci --kubeconfig 
"$KUBECONFIG"
diff --git a/hudi-agent-gateway/README.md b/hudi-agent-gateway/README.md
index eecdb2ed95c1..cf580554012c 100644
--- a/hudi-agent-gateway/README.md
+++ b/hudi-agent-gateway/README.md
@@ -191,6 +191,11 @@ GATEWAY_IT_SPARK_TABLE=bi_adbc_demo.orders \
 ```
 
 Add `GATEWAY_IT_LLM=1` to exercise the real Agent loop with the configured LLM.
+To additionally test a running gateway's HTTP Agent and MCP interfaces, set
+`GATEWAY_IT_HTTP_URL` to its base URL (for a local port-forward, for example,
+`http://127.0.0.1:18080`). The HTTP MCP check verifies a query and rejection of
+a write; the HTTP Agent check also requires `GATEWAY_IT_LLM=1`. Use a 
disposable
+read-only fixture table and a gateway accessible only to trusted clients.
 
 ## Development
 
diff --git a/hudi-agent-gateway/tests/integration/test_spark_connect_live.py 
b/hudi-agent-gateway/tests/integration/test_spark_connect_live.py
index 7d961667ff41..06abc64ce968 100644
--- a/hudi-agent-gateway/tests/integration/test_spark_connect_live.py
+++ b/hudi-agent-gateway/tests/integration/test_spark_connect_live.py
@@ -22,6 +22,7 @@ GATEWAY_IT_LLM=1 also enables the real Agent loop. No fixture 
writes here.
 
 import json
 import os
+import uuid
 
 import pytest
 from fastmcp import Client
@@ -82,3 +83,48 @@ async def test_agent_on_connect(connect_registry):
         config={"configurable": {"thread_id": "connect-it"}},
     )
     assert any(getattr(m, "name", None) == "describe_table" for m in 
out["messages"])
+
+
[email protected](not os.environ.get("GATEWAY_IT_HTTP_URL"), reason="set 
GATEWAY_IT_HTTP_URL")
+async def test_http_mcp_on_connect():
+    base = os.environ["GATEWAY_IT_HTTP_URL"].rstrip("/")
+    schema, table = TABLE.split(".")
+    sql = f"SELECT count(*) AS rows FROM `{schema}`.`{table}`"
+    async with Client(base + "/mcp") as mcp:
+        result = await mcp.call_tool("query_lakehouse", {"sql": sql})
+        data = json.loads(result.content[0].text)
+        assert "error" not in data and data["rows"][0][0] > 0
+        result = await mcp.call_tool("query_lakehouse", {"sql": f"DELETE FROM 
{TABLE}"})
+        assert "read-only" in json.loads(result.content[0].text)["error"]
+
+
[email protected](
+    not os.environ.get("GATEWAY_IT_HTTP_URL") or 
os.environ.get("GATEWAY_IT_LLM") != "1",
+    reason="set GATEWAY_IT_HTTP_URL and GATEWAY_IT_LLM=1",
+)
+async def test_http_agent_on_connect():
+    import httpx
+
+    base = os.environ["GATEWAY_IT_HTTP_URL"].rstrip("/")
+    session_id = "connect-http-it-" + uuid.uuid4().hex
+    schema, table = TABLE.split(".")
+    sql = f"SELECT COUNT(*) AS rows FROM `{schema}`.`{table}`"
+    async with httpx.AsyncClient(timeout=180) as client:
+        response = await client.post(
+            base + "/v1/chat",
+            json={"message": f"Describe the columns of {TABLE} using your 
describe_table tool.",
+                  "session_id": session_id},
+        )
+        response.raise_for_status()
+        result = response.json()
+        assert any(call["name"] == "describe_table" and not call["is_error"]
+                   for call in result["tool_trace"])
+        response = await client.post(
+            base + "/v1/chat",
+            json={"message": f"Use query_lakehouse to run {sql}.", 
"session_id": session_id},
+        )
+        response.raise_for_status()
+        trace = response.json()["tool_trace"]
+        queries = [json.loads(call["result_preview"]) for call in trace
+                   if call["name"] == "query_lakehouse" and not 
call["is_error"]]
+        assert any(query["rows"][0][0] > 0 for query in queries if 
query.get("rows"))
diff --git a/hudi-lakehouse/charts/hudi-spark-connect/README.md 
b/hudi-lakehouse/charts/hudi-spark-connect/README.md
index ebbadb1d7eb0..acbc25d4143c 100644
--- a/hudi-lakehouse/charts/hudi-spark-connect/README.md
+++ b/hudi-lakehouse/charts/hudi-spark-connect/README.md
@@ -166,8 +166,12 @@ evaluation before changing its lifecycle in a long-running 
driver.
 ## Kubernetes executor configuration (opt-in)
 
 The chart exposes the configuration needed for a shared driver with separate
-executors. Local mode remains the tested default; rendering this configuration 
is
-**not** evidence of a validated cluster deployment.
+executors. Local mode remains the default. In a single-node local kind test on
+ARM64, a driver with two 1 GiB executor pods registered both executors and ran
+Hudi INSERT/UPDATE/DELETE/MERGE, readback and driver restart/persistence checks
+against local MinIO and HMS. The test image used the pinned Spark/Hudi image 
with
+the current repository's executor entrypoint overlaid; the complete Dockerfile
+also built successfully. This is a functional test, not a scale or isolation 
claim.
 
 ```yaml
 spark:
@@ -182,7 +186,9 @@ serviceAccount:
 ```
 
 Provision this ServiceAccount and its executor-management RBAC in the release
-namespace first. The chart does not grant permissions. In Kubernetes client 
mode,
+namespace first. The local test allowed get/list/watch/create/delete/patch on 
pods,
+services and ConfigMaps, plus get/list/watch on persistent volume claims; tune 
and
+review permissions for your deployment. The chart does not grant permissions. 
In Kubernetes client mode,
 it advertises the driver's Pod IP and Pod name, selects the release namespace,
 and defaults the executor image to the same Hudi-enabled image. The image 
delegates
 executor startup to Spark's base entrypoint. Rebuild it when updating the 
entrypoint;
@@ -202,8 +208,12 @@ credentials other than the chart's S3 Secret. Driver pod 
settings do not automat
 apply to executors. Keep executors in the release namespace when using its 
Secrets.
 The chart defaults executor pull secrets from `imagePullSecrets`; override
 `spark.kubernetes.container.image.pullSecrets` for a different comma-separated 
list.
-Dynamic allocation/autoscaling is not enabled by this example and needs 
separate
-configuration and validation. Keep one driver replica; this is not a 
tenant-isolated service.
+Dynamic allocation/autoscaling is not enabled by this example. In a separate 
local
+kind test with shuffle tracking enabled, min/initial executors 1, max 2, 
1-second
+backlog thresholds and a 30-second idle timeout, a 64-partition SQL workload
+scaled from one executor to two and returned to one after becoming idle. Hudi
+CRUD/MERGE and driver restart/readback also passed in this mode. This validates
+only that local configuration and workload, not autoscaling under production 
load. Keep one driver replica; this is not a tenant-isolated service.
 Standalone (`spark://`) masters are not supported by this chart.
 
 ## Connect and validate with ADBC
@@ -221,7 +231,10 @@ kubectl -n hudi-lakehouse port-forward 
svc/hudi-spark-connect 15002:15002
 
 ADBC requires `api=connect`; its default is Thrift. In-cluster clients can use
 `hudi-spark-connect.hudi-lakehouse.svc:15002` without port-forwarding.
-The ADBC driver is installed on the client, not the server.
+The ADBC driver is installed on the client, not the server. Foundry Spark 0.2.1
+was tested with `dbc` in Linux containers and with an existing macOS ARM64 
driver
+installation. Fresh macOS installation has not been verified; use a Linux 
client
+if `dbc` cannot find a package for your platform.
 
 For the CRUD and restart test, prepare a Python 3.11 environment:
 
@@ -262,8 +275,9 @@ PYTHON="$VIRTUAL_ENV/bin/python" \
   just external-writer tables queried by both engines.
 - `--freshness`: keeps one ADBC connection open while a separate Spark JVM 
commits
   data, adds a column, and then recreates its test table. The supplied fixture 
creates
-  short-lived writer pods using the same pinned image and shared 
catalog/storage;
-  it is **not** another Connect session and does not test Spark 3.5/Flink 
compatibility.
+  short-lived writer pods using the Connect image by default and shared 
catalog/storage;
+  it is **not** another Connect session. Set `EXTERNAL_WRITER_IMAGE` to use a
+  separately built writer image, such as the Spark 3.5 fixture below.
   Allow an additional 1 GiB memory request (1.5 GiB limit), CPU, and 
permissions to
   create/delete pods and ConfigMaps. Only uniquely named test resources are 
touched.
 
@@ -275,6 +289,52 @@ does not alter shared example tables, and cleanup never 
uses `DROP DATABASE CASC
 Other external writers can use the Python CLI's `--external-writer <command 
argv>`
 hook; it receives the test database and `commit-schema` / `recreate` action.
 
+For the Spark 3.5 writer fixture, build the image and make it available to the
+cluster (for kind, load it into the cluster), then run the same check:
+
+```bash
+docker build -t hudi-spark35-writer:local 
hudi-lakehouse/images/spark-3.5-writer
+kind load docker-image hudi-spark35-writer:local --name <cluster-name>
+EXTERNAL_WRITER_IMAGE=hudi-spark35-writer:local \
+  ./hudi-lakehouse/local-dev/scripts/test-spark-connect.sh \
+  --restart-endpoint --freshness --refresh
+```
+
+The fixture uses Spark 3.5.7 and Hudi 1.2.0 in a separate JVM with the 
endpoint's
+catalog and storage configuration. With a Spark 4.1.3 / Hudi 1.2.0 Connect
+server, explicit refresh passed after external commits, schema evolution, and
+recreation; the original table remained readable after an endpoint restart. 
Without
+refresh, the existing session could not resolve the externally added column. 
This
+single combination does not establish general cross-version compatibility.
+
+A separate local-file experiment used Flink 1.20.1 / Hudi 1.2.0 to write a
+CoW table, and Spark 4.1.3 / Hudi 1.2.0 Connect to register and read that path.
+With the same Connect session open, a second Flink upsert was stale until
+`REFRESH TABLE`, after which the updated and inserted rows were visible. Both
+processes shared a disposable Docker volume; this did not exercise Hive
+Metastore sync, object storage, schema evolution, or concurrent writers. To
+reproduce the write side with the Hudi Flink bundle loaded in Flink SQL, use a
+shared path that both runtimes can access:
+
+```sql
+SET 'execution.runtime-mode' = 'batch';
+SET 'table.dml-sync' = 'true';
+SET 'parallelism.default' = '1';
+CREATE TABLE orders (id STRING PRIMARY KEY NOT ENFORCED, amount INT, ts BIGINT)
+WITH ('connector' = 'hudi', 'path' = 'file:///data/flink_orders',
+      'table.type' = 'COPY_ON_WRITE', 'write.precombine.field' = 'ts',
+      'write.tasks' = '1');
+INSERT INTO orders VALUES ('1', 100, CAST(1 AS BIGINT)), ('2', 200, CAST(1 AS 
BIGINT));
+-- In a later SQL client session, recreate the same table definition, then:
+INSERT INTO orders VALUES ('1', 150, CAST(2 AS BIGINT)), ('3', 300, CAST(2 AS 
BIGINT));
+```
+
+Register the written path with `CREATE TABLE flink_connect_orders USING hudi
+LOCATION 'file:///data/flink_orders'` in Connect. On the tested combination,
+`SELECT id, amount, ts ... ORDER BY id` returned the original two rows before
+refresh and `(1, 150, 2), (2, 200, 1), (3, 300, 2)` after
+`REFRESH TABLE flink_connect_orders` (the IDs are strings).
+
 These acceptance paths require live Spark/ADBC (and optionally Trino). Offline
 tests validate their assertions, configuration and cleanup, not engine 
behavior.
 With Spark 4.1.3, Hudi 1.2.0 and Foundry 0.2.1, the live SQL matrix, Trino 472
@@ -310,6 +370,14 @@ This retains the local-dev Ollama configuration. `/ready` 
checks Spark with
 `SELECT 1` and checks the configured LLM separately. MCP tool calls themselves
 do not require an LLM. Override the provider as described in the gateway 
README.
 
+In a single kind environment with the Helm Connect and gateway charts, a local
+Ollama `qwen3:4b-instruct` model completed the HTTP Agent describe and 
follow-up
+aggregation checks; HTTP MCP returned query results and rejected a DELETE. The
+Superset SQL Lab/chart API and Zeppelin notebook live tests also passed against
+the same Connect deployment. Gateway uses the official PySpark Connect client;
+Superset and Zeppelin use Foundry ADBC. These checks use isolated test tables 
and
+do not establish production concurrency or authentication behavior.
+
 <a id="spark-connect-client-tutorials"></a>
 
 ## BI clients
@@ -385,7 +453,40 @@ configuration and failure stage; they are not a safe-retry 
guarantee.
 In a local write test, `adbc_cancel()` returned `context canceled` to the 
client,
 but Spark jobs continued and all 200 inserted rows committed, confirmed by a
 separate connection and the Hudi timeline. Do not treat a client cancellation
-error as proof that a write was stopped or rolled back.
+error as proof that a write was stopped or rolled back. The isolated experiment
+can be repeated on a disposable, unauthenticated localhost endpoint:
+
+```bash
+python hudi-lakehouse/local-dev/example/connect_cancel.py --port 15002 --rows 
20000 --delay 1
+```
+
+The observer refreshes the table while looking for the complete expected 
commit;
+if the client reports an error and no rows are visible, it also checks the Hudi
+timeline. A completed rollback identifying the sole write instant, with no 
write
+instant remaining and no rows visible, is reported as an observation. It does 
not
+prove server-side work has stopped; the test resources are retained.
+An empty or pending timeline remains inconclusive. Wait for server-side work to
+stop before manually cleaning up retained resources.
+
+To test driver interruption at Hudi timeline stages, start a localhost
+port-forward on 15039 to a **disposable, idle** deployment, then run one stage
+at a time (15040 must be free for reconnection):
+
+```bash
+python hudi-lakehouse/local-dev/example/connect_restart_fault.py 
--restart-endpoint --stage requested
+# Re-establish the 15039 port-forward before the second run.
+python hudi-lakehouse/local-dev/example/connect_restart_fault.py 
--restart-endpoint --stage inflight
+```
+
+This forcibly deletes the driver pod and terminates all its sessions. It 
watches
+driver logs for the selected stage, checks the table after restart, verifies a
+new write and matching rollback log, then cleans up its unique table. Do not 
run
+other writes on the endpoint during the experiment: the requested-stage log 
does
+not identify the table. If the stage or rollback is not observed, the outcome 
is
+inconclusive and the test data is retained. On a local kind deployment, both
+requested and inflight interruptions led to a rollback of the captured instant 
and
+a successful subsequent write. This does not cover every point within those 
stages
+or concurrent writers.
 
 Use `LIMIT` and size client/driver memory for the expected results. Foundry 
0.2.1
 rejected Arrow responses exceeding its 4 MiB gRPC receive limit in the 
100k-row test.
@@ -394,6 +495,177 @@ to complete. Set the same property in `spark.extraConf` 
for other deployments us
 this driver. It controls batch size, not total result size or memory; oversized
 individual rows can still exceed the receive limit.
 
+### SQL, catalog and storage diagnostics
+
+By default, Kubernetes readiness checks only the TCP listener. To opt into
+SQL-aware readiness, first provision a persistent non-empty storage-backed 
probe
+table with a non-null ordinary data column and use values like:
+
+```yaml
+readiness:
+  sql:
+    enabled: true
+    schema: probe_database
+    table: probe_table
+    column: probe_column
+```
+
+The driver image must include `check-ready.py`; rebuild older images. Startup
+continues to use TCP so the driver can initialize before the SQL probe runs.
+Readiness checks SQL execution, catalog discovery and an actual table-data 
query
+in sequence. A missing or empty probe table makes the pod unready; provision it
+before enabling the option. Checks run every 30 seconds with a 30-second 
timeout.
+The table read needs a Spark task slot: a long query that occupies all slots 
can
+make this single-replica service unready and reject new clients. The probe 
process
+terminates after 25 seconds, but its Spark job may continue. Repeated timed-out
+probes can accumulate server-side queries. Leave SQL readiness disabled if this
+risk is unacceptable; the default TCP probe does not run a Spark query.
+In the local kind executor test, the probe passed against a pre-populated Hudi
+probe table; changing the configured column to a nonexistent name made the pod
+unready. Caching and catalog configuration can affect which backing services a
+successful read exercises; this check does not prove write access.
+
+For a separate read-only data-path diagnostic using ADBC, run:
+
+```bash
+python hudi-lakehouse/local-dev/example/connect_readiness.py \
+  --uri 'spark://localhost:15002?api=connect&auth_type=none&tls=false' \
+  --schema probe_database --table probe_table --column probe_column
+```
+
+The command reports SQL, catalog and table-data checks separately and exits
+nonzero at the first failure. It requires read access and never modifies the 
probe
+table. A missing/empty table is a failure; caching and catalog configuration 
can
+affect which backing services a successful read actually exercises. This 
external
+diagnostic does not itself change Kubernetes readiness or prove write access.
+
+### Spark UI, logs and metrics
+
+For local diagnosis, forward the driver UI to loopback only:
+
+```bash
+kubectl -n hudi-lakehouse port-forward --address 127.0.0.1 
deployment/hudi-spark-connect 4040:4040
+curl http://127.0.0.1:4040/api/v1/applications
+curl http://127.0.0.1:4040/metrics/json/
+```
+
+These endpoints returned application metadata and metrics in the local kind
+executor test. The UI and event logs can expose SQL, configuration and data;
+restrict access and do not expose them directly on an untrusted network.
+
+To retain Spark event logs, configure `spark.eventLog.enabled=true` and
+`spark.eventLog.dir` in `spark.extraConf` with a pre-created, writable durable
+filesystem location. A separately managed Spark History Server can read the 
same
+location through `spark.history.fs.logDirectory`; its storage credentials and
+access controls must be configured separately. In a single-node kind 
experiment,
+a test PVC replaced the driver's `/tmp` scratch volume, with
+`spark.eventLog.dir=file:/tmp/events`. A separate History Server mounted the 
same
+PVC and used `spark.history.fs.logDirectory=file:/eventlogs/events`. After a
+query and driver restart, its API listed the completed application and its
+successful jobs. This manual PVC configuration is not provisioned by the chart 
and
+does not validate multi-node volume sharing or production access controls. See
+[Spark monitoring 
documentation](https://spark.apache.org/docs/4.1.3/monitoring.html)
+for metrics sinks and history server details.
+
+Spark driver logs include Connect session IDs and Spark job details; Hudi SQL
+`CALL show_commits` exposes completed instants. Use timestamps and table names 
to
+investigate a particular write, but this setup does not provide a guaranteed
+client-request-to-job-to-commit identifier or complete trace correlation.
+
+### Client-disconnect experiment
+
+For cancellation, disconnect and lost-response experiments, `--wait` bounds
+readback and timeline observation only; process cleanup can take four 
additional
+seconds. Expiry is inconclusive and retains test data; it does not prove the
+server query stopped.
+
+On a disposable endpoint, the following creates a unique database and 
terminates
+its own ADBC client during an INSERT. It does not retry the write:
+
+```bash
+python hudi-lakehouse/local-dev/example/connect_disconnect.py \
+  --uri 'spark://localhost:15002?api=connect&auth_type=none&tls=false' \
+  --rows 2000 --delay 1 --wait 90
+```
+
+If the disconnect client receives a success response before or during 
interruption,
+the result is inconclusive. Later readback proves data is visible, not when it 
committed.
+
+If the complete expected write is observed after refreshing the table, the 
script
+cleans up its own table and database. Otherwise it reports an inconclusive 
outcome
+and leaves them for inspection. Wait until the server-side write has stopped 
before
+manually dropping those resources. On a local Spark 4.1.3 / Hudi 1.2.0 
endpoint,
+a 20,000-row write was observed after the client was terminated one second 
after
+entering the write. This timing does not guarantee the same outcome in another 
run.
+Earlier inconclusive runs used an invalid INSERT with ambiguous column names 
and
+must not be interpreted as evidence about interrupted writes.
+
+For a controlled lost-response experiment, expose a disposable unauthenticated
+endpoint only on localhost and run:
+
+```bash
+python hudi-lakehouse/local-dev/example/connect_lost_response.py --port 15002
+```
+
+A local TCP proxy forwards client requests while discarding server-to-client 
bytes
+after the connection is established. A separate connection refreshes and 
checks the
+unique table; once all expected rows are visible, the script terminates the 
client
+and cleans up. A local 1,000-row run observed the complete commit while 
responses
+were blocked. With the optional `--retry` flag, the same INSERT is repeated 
only
+after the original commit is observed; in that run it produced 2,000 rows with
+1,000 distinct IDs. This shows an ambiguous client outcome can coexist with a 
committed
+write; it does not establish cancellation, automatic retries, idempotency, or 
behavior
+at every failure phase. If the commit is not observed, the script leaves the 
table
+for inspection after the server write stops.
+
+### Result-size experiment
+
+The optional `connect_load.py` reads synthetic `range` rows without creating a
+table. Run it only on a disposable endpoint and monitor driver and client 
memory:
+
+```bash
+python hudi-lakehouse/local-dev/example/connect_load.py \
+  --uri 'spark://localhost:15002?api=connect&auth_type=none&tls=false' \
+  --rows 1000000 --payload-bytes 64 --fetch-size 10000
+```
+
+In a local Spark 4.1.3/Hudi 1.2.0 test with a 2 GiB driver heap,
+`spark.connect.grpc.arrow.maxBatchSize=1m`, Foundry 0.2.1 and Python 3.11,
+one million rows each containing an int64 ID and 64-byte ASCII payload
+passed count and checksum validation using `fetchmany(10000)`. Client peak RSS
+was about 170 MiB (the original client OS and driver installation were not
+recorded); a sampled Spark container value was 2.15 GiB, not a peak.
+The standalone container had a 10.2 GiB memory limit; this run does not 
establish
+behaviour under the chart's default 4 GiB or local-dev 3 GiB pod limit. A 
single
+5,000,000-byte string row
+failed with Foundry's 4,194,304-byte gRPC receive limit despite the 1 MiB Arrow
+batch setting. These are observations, not total-result or memory guarantees.
+
+The opt-in dialect live test also checks Spark `TIMESTAMP_NTZ` with microsecond
+precision, decimal(38,18), nullable array elements in a direct SQL result, 
nested
+structs and Hudi commit/key metadata columns. A direct SQL array result does 
not
+establish that Hudi/Parquet can write arrays with NULL elements. In a separate
+local CoW table test, an array containing NULL failed during Hudi write; the
+server log reported Parquet Avro's old-list-structure null-element restriction.
+This is a known Hudi write limitation tracked in 
[#20505](https://github.com/apache/hudi/issues/20505).
+The opt-in reproducer creates a unique database and table on a disposable 
endpoint:
+
+```bash
+python hudi-lakehouse/local-dev/example/connect_nullable_array.py --port 15002
+```
+
+The default configuration is expected to fail; wait for server-side work to 
stop
+before cleaning up the printed test resources.
+With a UTC Spark session, Mac clients launched with `TZ=UTC` and
+`TZ=America/Los_Angeles` both returned the timestamp with UTC zone information
+and the timestamp_ntz without a zone, preserving microseconds. Other server and
+client timezone combinations remain unverified.
+
+The manually triggered `Hudi Spark Connect live acceptance` GitHub Actions 
workflow
+creates a disposable kind cluster, deploys the chart, runs the SQL matrix and
+restart readback, and deletes the cluster. It is distinct from offline chart 
and
+script tests. Its first hosted execution still needs to be verified.
+
 ## Troubleshooting
 
 | Symptom | Check |
diff --git a/hudi-lakehouse/charts/hudi-spark-connect/templates/deployment.yaml 
b/hudi-lakehouse/charts/hudi-spark-connect/templates/deployment.yaml
index f9912e3506b0..301489137701 100644
--- a/hudi-lakehouse/charts/hudi-spark-connect/templates/deployment.yaml
+++ b/hudi-lakehouse/charts/hudi-spark-connect/templates/deployment.yaml
@@ -114,17 +114,33 @@ spec:
               readOnly: true
             - name: scratch
               mountPath: /tmp
-          # TCP probes check the listener, not catalog or write correctness.
-          # The ADBC smoke test verifies the complete data path.
+          # Startup always checks the listener. Optional readiness can also
+          # query a pre-provisioned non-empty table through Spark Connect.
           startupProbe:
             tcpSocket:
               port: connect
             periodSeconds: 5
             failureThreshold: 60
           readinessProbe:
+            {{- if .Values.readiness.sql.enabled }}
+            exec:
+              command:
+                - python3
+                - /opt/spark/check-ready.py
+                - --schema
+                - {{ required "readiness.sql.schema is required" 
.Values.readiness.sql.schema | quote }}
+                - --table
+                - {{ required "readiness.sql.table is required" 
.Values.readiness.sql.table | quote }}
+                - --column
+                - {{ required "readiness.sql.column is required" 
.Values.readiness.sql.column | quote }}
+            periodSeconds: 30
+            timeoutSeconds: 30
+            failureThreshold: 3
+            {{- else }}
             tcpSocket:
               port: connect
             periodSeconds: 10
+            {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       volumes:
diff --git a/hudi-lakehouse/charts/hudi-spark-connect/values.schema.json 
b/hudi-lakehouse/charts/hudi-spark-connect/values.schema.json
index 1a174f1cc9f3..5d8c81251d01 100644
--- a/hudi-lakehouse/charts/hudi-spark-connect/values.schema.json
+++ b/hudi-lakehouse/charts/hudi-spark-connect/values.schema.json
@@ -3,6 +3,33 @@
   "$comment": "Licensed to the Apache Software Foundation (ASF) under one or 
more contributor license agreements. See the NOTICE file. Licensed under the 
Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0). 
Distributed on an AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND.",
   "type": "object",
   "properties": {
+    "readiness": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "sql": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "enabled": {"type": "boolean"},
+            "schema": {"type": "string", "pattern": 
"^$|^[A-Za-z_][A-Za-z0-9_]*$"},
+            "table": {"type": "string", "pattern": 
"^$|^[A-Za-z_][A-Za-z0-9_]*$"},
+            "column": {"type": "string", "pattern": 
"^$|^[A-Za-z_][A-Za-z0-9_]*$"}
+          },
+          "allOf": [{
+            "if": {"properties": {"enabled": {"const": true}}, "required": 
["enabled"]},
+            "then": {
+              "properties": {
+                "schema": {"minLength": 1},
+                "table": {"minLength": 1},
+                "column": {"minLength": 1}
+              },
+              "required": ["schema", "table", "column"]
+            }
+          }]
+        }
+      }
+    },
     "spark": {
       "type": "object",
       "properties": {
diff --git a/hudi-lakehouse/charts/hudi-spark-connect/values.yaml 
b/hudi-lakehouse/charts/hudi-spark-connect/values.yaml
index aa2e7af66747..f2dad4878a78 100644
--- a/hudi-lakehouse/charts/hudi-spark-connect/values.yaml
+++ b/hudi-lakehouse/charts/hudi-spark-connect/values.yaml
@@ -62,6 +62,14 @@ storage:
     # Optional key in existingSecret for temporary credentials.
     sessionTokenKey: ""
 
+# Optional data-path readiness requires an existing non-empty probe table.
+readiness:
+  sql:
+    enabled: false
+    schema: ""
+    table: ""
+    column: ""
+
 # No public ingress or unauthenticated LoadBalancer is provisioned.
 service:
   port: 15002
diff --git a/hudi-lakehouse/images/spark-3.5-writer/Dockerfile 
b/hudi-lakehouse/images/spark-3.5-writer/Dockerfile
new file mode 100644
index 000000000000..e4408ec49787
--- /dev/null
+++ b/hudi-lakehouse/images/spark-3.5-writer/Dockerfile
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Independent Spark 3.5/Hudi 1.2 writer for cross-version freshness checks.
+FROM 
apache/spark:3.5.7-scala2.12-java17-python3-ubuntu@sha256:7897b9a69efa94caccc52579e310a2fe68fb75b88a3d55b97b31568e5e233176
+USER root
+RUN set -eux; cd /opt/spark/jars; for spec in \
+ 'cc79fba00822f3a6c3a860df99375c2e3d4a034d037dbf4213a4966a706a7676 
org/apache/hudi/hudi-spark3.5-bundle_2.12/1.2.0/hudi-spark3.5-bundle_2.12-1.2.0.jar'
 \
+ '53f9ae03c681a30a50aa17524bd9790ab596b28481858e54efd989a826ed3a4a 
org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar' \
+ 'e9cb23619b5b3a222e6f7ba5a79cb18ce4f606e6cabd976cd728582ec47080fa 
com/amazonaws/aws-java-sdk-bundle/1.12.772/aws-java-sdk-bundle-1.12.772.jar'; 
do \
+ set -- $spec; checksum="$1"; artifact="$2"; file="$(basename "$artifact")"; \
+ wget -q --timeout=30 --tries=3 -O "$file" 
"https://repo.maven.apache.org/maven2/$artifact";; \
+ printf '%s  %s\n' "$checksum" "$file" | sha256sum -c -; done
+USER spark
diff --git a/hudi-lakehouse/images/spark-connect/Dockerfile 
b/hudi-lakehouse/images/spark-connect/Dockerfile
index ce0d4c8234f7..7d89d3a2c9c3 100644
--- a/hudi-lakehouse/images/spark-connect/Dockerfile
+++ b/hudi-lakehouse/images/spark-connect/Dockerfile
@@ -40,6 +40,7 @@ RUN set -eux; \
 RUN python3 -m pip install --no-cache-dir pyspark-client==4.1.3 pyarrow==18.1.0
 
 COPY start-connect.sh /opt/spark/start-connect.sh
+COPY check-ready.py /opt/spark/check-ready.py
 RUN chmod 755 /opt/spark/start-connect.sh && mkdir -p /opt/spark/conf && chown 
spark:spark /opt/spark/conf
 USER spark
 EXPOSE 15002
diff --git a/hudi-lakehouse/images/spark-connect/check-ready.py 
b/hudi-lakehouse/images/spark-connect/check-ready.py
new file mode 100644
index 000000000000..8acbc2875feb
--- /dev/null
+++ b/hudi-lakehouse/images/spark-connect/check-ready.py
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Read-only, opt-in SQL/catalog/table-data readiness for Spark Connect."""
+import argparse
+import re
+import signal
+import sys
+
+# A default SIGALRM terminates even when the probe is blocked in native code.
+if __name__ == "__main__":
+    signal.signal(signal.SIGALRM, signal.SIG_DFL)
+    signal.alarm(25)
+
+from pyspark.sql import SparkSession
+
+
+def identifier(value):
+    if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', value):
+        raise argparse.ArgumentTypeError('invalid SQL identifier')
+    return value
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--schema', required=True, type=identifier)
+    parser.add_argument('--table', required=True, type=identifier)
+    parser.add_argument('--column', required=True, type=identifier)
+    args = parser.parse_args()
+    spark = None
+    stage = 'connection'
+    try:
+        spark = 
SparkSession.builder.remote('sc://127.0.0.1:15002').getOrCreate()
+        stage = 'sql'
+        if spark.sql('SELECT 1').collect()[0][0] != 1:
+            raise ValueError('unexpected SQL result')
+        print('PASS: sql', flush=True)
+        stage = 'catalog'
+        if not any(row[0] == args.schema for row in spark.sql('SHOW 
DATABASES').collect()):
+            raise ValueError('catalog schema not found')
+        print('PASS: catalog', flush=True)
+        stage = 'storage'
+        rows = spark.sql(f'SELECT `{args.column}` FROM 
`{args.schema}`.`{args.table}` '
+                         f'WHERE `{args.column}` IS NOT NULL LIMIT 
1').collect()
+        if not rows:
+            raise ValueError('probe table has no non-null probe value')
+        print('PASS: storage', flush=True)
+        return 0
+    except Exception as exc:
+        print(f'FAIL: {stage}: {exc}', file=sys.stderr, flush=True)
+        return 1
+    finally:
+        if spark is not None:
+            spark.stop()
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())
diff --git a/hudi-lakehouse/local-dev/example/connect_cancel.py 
b/hudi-lakehouse/local-dev/example/connect_cancel.py
new file mode 100644
index 000000000000..2ab492f174d3
--- /dev/null
+++ b/hudi-lakehouse/local-dev/example/connect_cancel.py
@@ -0,0 +1,141 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Opt-in ADBC cancellation experiment against a disposable localhost 
endpoint."""
+import argparse
+import os
+import threading
+import time
+import uuid
+
+from connect_fault_utils import readback, read_timeline
+
+
+def connect(port, driver):
+    from adbc_driver_manager import dbapi
+    uri = f'spark://127.0.0.1:{port}?api=connect&auth_type=none&tls=false'
+    return dbapi.connect(driver=driver, db_kwargs={'uri': uri}, 
autocommit=True)
+
+
+def rolled_back_write(timeline, rollbacks):
+    """Identify the sole rolled-back write in the isolated one-write table."""
+    if any(str(action).lower() in ('commit', 'deltacommit', 'replacecommit')
+           for _, action, _ in timeline):
+        return None
+    completed = {str(timestamp) for timestamp, action, state in timeline
+                 if str(action).lower() == 'rollback' and str(state).upper() 
== 'COMPLETED'}
+    targets = {str(row[1]) for row in rollbacks if str(row[0]) in completed 
and row[1]}
+    return next(iter(targets)) if len(targets) == 1 else None
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--port', type=int, required=True)
+    p.add_argument('--driver', default='spark')
+    p.add_argument('--rows', type=int, default=20000)
+    p.add_argument('--delay', type=float, default=1)
+    p.add_argument('--wait', type=int, default=90)
+    args = p.parse_args()
+    if not 1 <= args.port <= 65535 or args.rows < 1 or args.delay < 0 or 
args.wait < 1:
+        p.error('invalid port, row count, delay or wait')
+    database = 'connect_cancel_' + uuid.uuid4().hex
+    table = database + '.orders'
+    print(f'Test database: {database}', flush=True)
+    created = False
+    with connect(args.port, args.driver) as observer, observer.cursor() as 
read, \
+            connect(args.port, args.driver) as writer, writer.cursor() as 
write:
+        try:
+            read.execute(f'CREATE DATABASE {database}')
+            created = True
+            read.execute(f"CREATE TABLE {table} (id INT, ts BIGINT) USING hudi 
"
+                         "TBLPROPERTIES(type='cow', primaryKey='id', 
preCombineField='ts')")
+            outcome = []
+            started = threading.Event()
+
+            def submit():
+                started.set()
+                try:
+                    write.execute(f'INSERT INTO {table} SELECT CAST(id AS INT) 
AS id, CAST(id AS BIGINT) AS ts FROM range({args.rows})')
+                    outcome.append('success')
+                except Exception as exc:
+                    outcome.append(f'{type(exc).__name__}: {exc}')
+
+            worker = threading.Thread(target=submit, daemon=True)
+            worker.start()
+            started.wait()
+            time.sleep(args.delay)
+            if not worker.is_alive():
+                print('INCONCLUSIVE: write finished before cancellation', 
flush=True)
+                return 2
+            try:
+                write.adbc_cancel()
+                print('Cancellation request returned', flush=True)
+            except Exception as exc:
+                print(f'Cancellation request failed: {type(exc).__name__}: 
{exc}', flush=True)
+            worker.join(30)
+            if worker.is_alive():
+                # Do not close a cursor concurrently with an active operation.
+                print(f'INCONCLUSIVE: client did not stop; leave {database} 
for inspection', flush=True)
+                os._exit(2)
+            print(f'Client write result: {outcome}', flush=True)
+            deadline = time.monotonic() + args.wait
+            uri = 
f'spark://127.0.0.1:{args.port}?api=connect&auth_type=none&tls=false'
+            last_pending_timeline = None
+            while time.monotonic() < deadline:
+                try:
+                    count, total = readback(uri, args.driver, table, deadline)
+                except TimeoutError:
+                    print('INCONCLUSIVE: readback deadline expired; preserve 
test data', flush=True)
+                    return 2
+                except Exception as exc:
+                    print(f'Readback not yet available: {type(exc).__name__}', 
flush=True)
+                    time.sleep(min(2, max(0, deadline - time.monotonic())))
+                    continue
+                if count == args.rows and total == args.rows * (args.rows - 1) 
// 2:
+                    print(f'OBSERVED: all {count} rows visible after the 
cancellation attempt; commit timing is unknown', flush=True)
+                    read.execute(f'DROP TABLE IF EXISTS {table} PURGE')
+                    read.execute(f'DROP DATABASE IF EXISTS {database}')
+                    created = False
+                    print('PASS: owned test data cleaned', flush=True)
+                    return 0
+                if count == 0 and outcome and outcome[0] != 'success':
+                    try:
+                        timeline, rollbacks = read_timeline(uri, args.driver, 
table, deadline)
+                    except TimeoutError:
+                        print('INCONCLUSIVE: timeline deadline expired; 
preserve test data', flush=True)
+                        return 2
+                    except Exception as exc:
+                        print(f'Timeline not yet available: 
{type(exc).__name__}', flush=True)
+                    else:
+                        instant = rolled_back_write(timeline, rollbacks)
+                        if instant:
+                            print(f'OBSERVED: write instant {instant} rolled 
back; 0 rows visible. '
+                                  'Server-side work may still be running; 
preserve test data', flush=True)
+                            return 0
+                        pending_timeline = frozenset(tuple(row) for row in 
timeline)
+                        if pending_timeline != last_pending_timeline:
+                            print(f'PENDING: no visible rows; 
timeline={timeline} '
+                                  '(write outcome remains inconclusive)', 
flush=True)
+                            last_pending_timeline = pending_timeline
+                time.sleep(min(2, max(0, deadline - time.monotonic())))
+            print('INCONCLUSIVE: complete commit was not observed; outcome may 
still change', flush=True)
+            return 2
+        finally:
+            if created:
+                print(f'Leave {database} untouched until the server write has 
stopped; then remove its table and database.', flush=True)
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())
diff --git a/hudi-lakehouse/local-dev/example/connect_disconnect.py 
b/hudi-lakehouse/local-dev/example/connect_disconnect.py
new file mode 100644
index 000000000000..ff5fa7072081
--- /dev/null
+++ b/hudi-lakehouse/local-dev/example/connect_disconnect.py
@@ -0,0 +1,147 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Opt-in client-disconnect experiment on a disposable Connect endpoint.
+
+An indeterminate result leaves its unique test database for manual cleanup.
+This is not a retry, transaction, cancellation, or concurrency guarantee.
+"""
+import argparse
+import queue
+import re
+import subprocess
+import sys
+import threading
+import time
+import uuid
+
+from adbc_driver_manager import dbapi
+from connect_fault_utils import readback, stop_process
+
+
+def connect(uri, driver):
+    return dbapi.connect(driver=driver, db_kwargs={'uri': uri}, 
autocommit=True)
+
+
+def interrupt_before_response(child, delay):
+    """Only count the experiment when no successful response was observed."""
+    messages = queue.Queue()
+
+    def consume():
+        try:
+            for line in child.stdout:
+                messages.put(line.strip())
+        finally:
+            messages.put(None)
+
+    reader = threading.Thread(target=consume, daemon=True)
+    reader.start()
+    ready = False
+    try:
+        try:
+            if messages.get(timeout=60) != 'READY':
+                return False
+            ready = True
+            # Consume output continuously, including TextIOWrapper buffering.
+            messages.get(timeout=delay)
+            return False  # RESPONSE, EOF, or unexpected output before 
interruption.
+        except queue.Empty:
+            # A missing READY is not evidence that the write was submitted.
+            if child.poll() is not None or not ready:
+                return False
+        stop_process(child)
+        reader.join(timeout=5)
+        if reader.is_alive():
+            return False
+        while not messages.empty():
+            if messages.get_nowait() == 'RESPONSE':
+                return False
+        return True
+    finally:
+        stop_process(child)
+        reader.join(timeout=5)
+        if not reader.is_alive():
+            child.stdout.close()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--uri', required=True)
+    parser.add_argument('--driver', default='spark')
+    parser.add_argument('--rows', type=int, default=10000)
+    parser.add_argument('--delay', type=float, default=1.0)
+    parser.add_argument('--wait', type=int, default=90)
+    parser.add_argument('--child-table', help=argparse.SUPPRESS)
+    args = parser.parse_args()
+    if args.rows < 1 or args.delay < 0 or args.wait < 1:
+        parser.error('rows and wait must be positive and delay nonnegative')
+    if args.child_table:
+        if not re.fullmatch(r"connect_disconnect_[0-9a-f]{32}\.orders", 
args.child_table):
+            parser.error("invalid internal test table")
+        # The table name comes only from this script's generated parent 
argument.
+        with connect(args.uri, args.driver) as conn, conn.cursor() as cursor:
+            print('READY', flush=True)
+            cursor.execute(f'INSERT INTO {args.child_table} SELECT CAST(id AS 
INT) AS id, CAST(id AS BIGINT) AS ts FROM range({args.rows})')
+            print('RESPONSE', flush=True)
+        return 0
+
+    database = 'connect_disconnect_' + uuid.uuid4().hex
+    table = database + '.orders'
+    print(f'Test database: {database}', flush=True)
+    created = False
+    child = None
+    with connect(args.uri, args.driver) as conn, conn.cursor() as cursor:
+        cursor.execute(f'CREATE DATABASE {database}')
+        created = True
+        try:
+            cursor.execute(f"CREATE TABLE {table} (id INT, ts BIGINT) USING 
hudi "
+                           "TBLPROPERTIES(type='cow', primaryKey='id', 
preCombineField='ts')")
+            child = subprocess.Popen([sys.executable, __file__, '--uri', 
args.uri, '--driver', args.driver,
+                                      '--rows', str(args.rows), 
'--child-table', table],
+                                     stdout=subprocess.PIPE, 
stderr=subprocess.DEVNULL, text=True)
+            if not interrupt_before_response(child, args.delay):
+                print('INCONCLUSIVE: client was not interrupted before its 
response; preserve test data')
+                return 2
+            print(f'Client interrupted without an observed response; waiting 
up to {args.wait}s for readback', flush=True)
+            deadline = time.monotonic() + args.wait
+            while time.monotonic() < deadline:
+                try:
+                    count, total = readback(args.uri, args.driver, table, 
deadline)
+                except TimeoutError:
+                    print('INCONCLUSIVE: readback deadline expired; preserve 
test data', flush=True)
+                    return 2
+                except Exception as exc:
+                    print(f'Readback not yet available: {type(exc).__name__}', 
flush=True)
+                    time.sleep(min(2, max(0, deadline - time.monotonic())))
+                    continue
+                if count == args.rows and total == args.rows * (args.rows - 1) 
// 2:
+                    print(f'OBSERVED: all {count} rows visible after client 
disconnect; commit timing is unknown')
+                    cursor.execute(f'DROP TABLE IF EXISTS {table} PURGE')
+                    cursor.execute(f'DROP DATABASE IF EXISTS {database}')
+                    created = False
+                    print('PASS: owned test data cleaned')
+                    return 0
+                time.sleep(min(2, max(0, deadline - time.monotonic())))
+            print('INCONCLUSIVE: full write was not observed; outcome may 
still change')
+            return 2
+        finally:
+            stop_process(child)
+            if created:
+                print(f'Leave {database} untouched until the server write has 
stopped; then remove its table and database.',
+                      file=sys.stderr, flush=True)
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())
diff --git a/hudi-lakehouse/local-dev/example/connect_external_writer.py 
b/hudi-lakehouse/local-dev/example/connect_external_writer.py
index 7e3cb0110edb..278924304e45 100644
--- a/hudi-lakehouse/local-dev/example/connect_external_writer.py
+++ b/hudi-lakehouse/local-dev/example/connect_external_writer.py
@@ -49,7 +49,7 @@ def write(database, action):
         spark.stop()
 
 
-def writer_pod(template, name, database, action):
+def writer_pod(template, name, database, action, image=None):
     """Reuse only connection config; never reuse the endpoint's 
labels/probes."""
     container = next(c for c in template["containers"] if c["name"] == 
"spark-connect")
     env = [e for e in container["env"] if e["name"] == "HUDI_CONF_DIR" or 
e["name"].startswith("AWS_")]
@@ -63,7 +63,7 @@ def writer_pod(template, name, database, action):
             "securityContext": template["securityContext"],
             "imagePullSecrets": template.get("imagePullSecrets", []),
             "containers": [{
-                "name": "writer", "image": container["image"],
+                "name": "writer", "image": image or container["image"],
                 "imagePullPolicy": container["imagePullPolicy"],
                 "securityContext": container["securityContext"],
                 "env": env,
@@ -98,7 +98,7 @@ def launch(database, action):
         # create, not apply: a name collision must not authorize deleting 
others' resources.
         kubectl("create", "configmap", name, 
f"--from-file=connect_external_writer.py={Path(__file__).resolve()}")
         config_created = True
-        kubectl("create", "-f", "-", input=json.dumps(writer_pod(template, 
name, database, action)))
+        kubectl("create", "-f", "-", input=json.dumps(writer_pod(template, 
name, database, action, os.environ.get("EXTERNAL_WRITER_IMAGE"))))
         pod_created = True
         deadline = time.monotonic() + 390
         while time.monotonic() < deadline:
diff --git a/hudi-lakehouse/local-dev/example/connect_fault_utils.py 
b/hudi-lakehouse/local-dev/example/connect_fault_utils.py
new file mode 100644
index 000000000000..0c2a00f1088e
--- /dev/null
+++ b/hudi-lakehouse/local-dev/example/connect_fault_utils.py
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Bounded client-process helpers for disposable fault experiments."""
+import json
+from pathlib import Path
+import re
+import subprocess
+import sys
+import time
+
+
+def stop_process(process):
+    """Reap an owned client, escalating when graceful termination stalls."""
+    if process is None:
+        return
+    if process.poll() is None:
+        process.terminate()
+        try:
+            process.wait(timeout=2)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            process.wait(timeout=2)
+    else:
+        process.wait(timeout=2)
+
+
+def readback(uri, driver, table, deadline):
+    """Bound connection, query, fetch and close by the observation deadline.
+
+    A separate process owns the native ADBC handles. Killing it does not prove
+    that its read-only Spark query stopped; never use this helper for writes.
+    """
+    return _observe(uri, driver, table, deadline, 'rows')
+
+
+def read_timeline(uri, driver, table, deadline):
+    """Read the test table's active timeline within the same deadline."""
+    return _observe(uri, driver, table, deadline, 'timeline')
+
+
+def _observe(uri, driver, table, deadline, operation):
+    remaining = deadline - time.monotonic()
+    if remaining <= 0:
+        raise TimeoutError('readback deadline expired')
+    process = subprocess.Popen(
+        [sys.executable, str(Path(__file__).resolve())],
+        stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+        text=True,
+    )
+    try:
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            raise TimeoutError('readback deadline expired')
+        try:
+            output, error = process.communicate(
+                json.dumps({'uri': uri, 'driver': driver, 'table': table, 
'operation': operation}),
+                timeout=remaining,
+            )
+        except subprocess.TimeoutExpired as exc:
+            raise TimeoutError('readback deadline expired') from exc
+        if time.monotonic() >= deadline:
+            raise TimeoutError('readback completed after deadline')
+        if process.returncode:
+            raise RuntimeError('readback client failed: ' + error.strip())
+        return tuple(json.loads(output))
+    finally:
+        try:
+            stop_process(process)
+        finally:
+            for stream in (process.stdin, process.stdout, process.stderr):
+                if stream is not None:
+                    stream.close()
+
+
+def main():
+    from adbc_driver_manager import dbapi
+    args = json.load(sys.stdin)
+    table = args['table']
+    if not 
re.fullmatch(r'connect_(?:cancel|disconnect|lost_response)_[0-9a-f]{32}\.orders',
 table):
+        raise ValueError('invalid fault experiment table')
+    with dbapi.connect(driver=args['driver'], db_kwargs={'uri': args['uri']}, 
autocommit=True) as conn:
+        with conn.cursor() as cursor:
+            cursor.execute(f'REFRESH TABLE {table}')
+            if args['operation'] == 'rows':
+                cursor.execute(f'SELECT COUNT(*), SUM(id) FROM {table}')
+                result = cursor.fetchall()[0]
+            elif args['operation'] == 'timeline':
+                cursor.execute(f"SELECT timestamp, action, state FROM 
hudi_query_timeline('{table}', 'false')")
+                timeline = cursor.fetchall()
+                cursor.execute(f"CALL show_rollbacks(table => '{table}')")
+                result = (timeline, cursor.fetchall())
+            else:
+                raise ValueError('invalid observation operation')
+    print(json.dumps(result), flush=True)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/hudi-lakehouse/local-dev/example/connect_load.py 
b/hudi-lakehouse/local-dev/example/connect_load.py
new file mode 100644
index 000000000000..f289cad1a346
--- /dev/null
+++ b/hudi-lakehouse/local-dev/example/connect_load.py
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Opt-in result-size experiment; no tables or persistent data are created."""
+import argparse
+import platform
+import resource
+import time
+
+from adbc_driver_manager import dbapi
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--uri', required=True)
+    parser.add_argument('--driver', default='spark')
+    parser.add_argument('--rows', type=int, required=True)
+    parser.add_argument('--payload-bytes', type=int, default=64)
+    parser.add_argument('--fetch-size', type=int, default=10000)
+    args = parser.parse_args()
+    if args.rows < 1 or args.payload_bytes < 0 or args.fetch_size < 1:
+        parser.error('rows and fetch size must be positive; payload bytes must 
be nonnegative')
+    start = time.monotonic()
+    count = total = 0
+    print(f'rows={args.rows} payload_bytes={args.payload_bytes} 
fetchmany={args.fetch_size}', flush=True)
+    with dbapi.connect(driver=args.driver, db_kwargs={'uri': args.uri}, 
autocommit=True) as conn:
+        with conn.cursor() as cursor:
+            cursor.execute(f"SELECT id, repeat('x', {args.payload_bytes}) AS 
payload FROM range({args.rows})")
+            while True:
+                rows = cursor.fetchmany(args.fetch_size)
+                if not rows:
+                    break
+                count += len(rows)
+                total += sum(row[0] for row in rows)
+                if any(len(row[1]) != args.payload_bytes for row in rows):
+                    raise AssertionError('unexpected payload width')
+    expected = args.rows * (args.rows - 1) // 2
+    if count != args.rows or total != expected:
+        raise AssertionError(f'count={count} sum={total}, 
expected={args.rows}/{expected}')
+    print(f'PASS count={count} checksum={total} seconds={time.monotonic() - 
start:.1f} '
+          
f'client_peak_rss={resource.getrusage(resource.RUSAGE_SELF).ru_maxrss} '
+          f'platform={platform.system()} (RSS bytes on macOS, KiB on Linux)', 
flush=True)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/hudi-lakehouse/local-dev/example/connect_lost_response.py 
b/hudi-lakehouse/local-dev/example/connect_lost_response.py
new file mode 100644
index 000000000000..28759cf90073
--- /dev/null
+++ b/hudi-lakehouse/local-dev/example/connect_lost_response.py
@@ -0,0 +1,195 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Discard responses to one local test client while independently observing a 
commit.
+
+Only use with a disposable, unauthenticated endpoint exposed on localhost.
+An unobserved write leaves its unique test database for later manual cleanup.
+"""
+import argparse
+import select
+import socket
+import subprocess
+import sys
+import threading
+import time
+import uuid
+
+from connect_fault_utils import readback, stop_process
+
+
+def connect(port, driver):
+    from adbc_driver_manager import dbapi
+    uri = f'spark://127.0.0.1:{port}?api=connect&auth_type=none&tls=false'
+    return dbapi.connect(driver=driver, db_kwargs={'uri': uri}, 
autocommit=True)
+
+
+class ResponseDropProxy:
+    """One-connection TCP proxy; drop server bytes once the gate is closed."""
+    def __init__(self, target_port):
+        self.target_port = target_port
+        self.drop = threading.Event()
+        self.listener = socket.socket()
+        self.listener.bind(('127.0.0.1', 0))
+        self.listener.listen(1)
+        self.port = self.listener.getsockname()[1]
+        self.sockets = []
+        self.thread = threading.Thread(target=self._serve, daemon=True)
+        self.started = False
+
+    def start(self):
+        self.thread.start()
+        self.started = True
+
+    def _serve(self):
+        try:
+            client, _ = self.listener.accept()
+            self.sockets.append(client)
+            upstream = socket.create_connection(('127.0.0.1', 
self.target_port), timeout=10)
+            upstream.settimeout(None)
+            self.sockets.append(upstream)
+            while True:
+                readable, _, _ = select.select([client, upstream], [], [], 1)
+                for source in readable:
+                    data = source.recv(65536)
+                    if not data:
+                        return
+                    if source is upstream and self.drop.is_set():
+                        continue
+                    (upstream if source is client else client).sendall(data)
+        except (OSError, ValueError):
+            pass
+        finally:
+            for sock in self.sockets:
+                sock.close()
+
+    def close(self):
+        self.listener.close()
+        for sock in self.sockets:
+            try:
+                sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+            sock.close()
+        if self.started:
+            self.thread.join(timeout=5)
+
+
+def child(port, driver, table, rows):
+    with connect(port, driver) as conn, conn.cursor() as cursor:
+        cursor.execute('SELECT 1')
+        cursor.fetchall()
+        print('READY', flush=True)
+        if sys.stdin.readline().strip() != 'WRITE':
+            return 1
+        cursor.execute(f'INSERT INTO {table} SELECT CAST(id AS INT) AS id, 
CAST(id AS BIGINT) AS ts FROM range({rows})')
+        print('RESPONSE', flush=True)
+    return 0
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--port', type=int, required=True, help='localhost 
port of disposable Spark Connect endpoint')
+    parser.add_argument('--driver', default='spark')
+    parser.add_argument('--rows', type=int, default=1000)
+    parser.add_argument('--wait', type=int, default=120)
+    parser.add_argument('--retry', action='store_true', help='repeat the same 
INSERT after the lost response')
+    args = parser.parse_args()
+    if not 1 <= args.port <= 65535 or args.rows < 1 or args.wait < 1:
+        parser.error('port must be valid and rows and wait must be positive')
+    database = 'connect_lost_response_' + uuid.uuid4().hex
+    table = database + '.orders'
+    print(f'Test database: {database}', flush=True)
+    created = False
+    process = None
+    proxy = ResponseDropProxy(args.port)
+    try:
+        with connect(args.port, args.driver) as conn, conn.cursor() as cursor:
+            cursor.execute(f'CREATE DATABASE {database}')
+            created = True
+            cursor.execute(f"CREATE TABLE {table} (id INT, ts BIGINT) USING 
hudi "
+                           "TBLPROPERTIES(type='cow', primaryKey='id', 
preCombineField='ts')")
+            proxy.start()
+            # The child gets the generated table name through a pipe, not user 
input.
+            process = subprocess.Popen([sys.executable, __file__, '--port', 
str(args.port),
+                                        '--driver', args.driver, '--rows', 
str(args.rows),
+                                        '--child-port', str(proxy.port), 
'--child-table', table],
+                                       stdin=subprocess.PIPE, 
stdout=subprocess.PIPE,
+                                       stderr=subprocess.DEVNULL, text=True)
+            if not select.select([process.stdout], [], [], 60)[0] or 
process.stdout.readline().strip() != 'READY':
+                print('INCONCLUSIVE: client did not become ready')
+                return 2
+            proxy.drop.set()
+            process.stdin.write('WRITE\n')
+            process.stdin.flush()
+            print('Response gate closed; write submitted to client', 
flush=True)
+            deadline = time.monotonic() + args.wait
+            while time.monotonic() < deadline:
+                if process.poll() is not None:
+                    print('INCONCLUSIVE: client exited before commit was 
observed')
+                    return 2
+                try:
+                    count, total = readback(
+                        
f'spark://127.0.0.1:{args.port}?api=connect&auth_type=none&tls=false',
+                        args.driver, table, deadline)
+                except TimeoutError:
+                    print('INCONCLUSIVE: readback deadline expired; preserve 
test data', flush=True)
+                    return 2
+                except Exception as exc:
+                    print(f'Readback not yet available: {type(exc).__name__}', 
flush=True)
+                    time.sleep(min(2, max(0, deadline - time.monotonic())))
+                    continue
+                if count == args.rows and total == args.rows * (args.rows - 1) 
// 2:
+                    stop_process(process)
+                    print(f'OBSERVED: {count} rows committed while responses 
were discarded; client was terminated')
+                    if args.retry:
+                        cursor.execute(f'INSERT INTO {table} SELECT CAST(id AS 
INT) AS id, CAST(id AS BIGINT) AS ts FROM range({args.rows})')
+                        cursor.execute(f'REFRESH TABLE {table}')
+                        cursor.execute(f'SELECT COUNT(*), COUNT(DISTINCT id), 
SUM(id) FROM {table}')
+                        count, distinct, total = cursor.fetchall()[0]
+                        expected_sum = args.rows * (args.rows - 1) // 2
+                        if distinct != args.rows or count not in (args.rows, 2 
* args.rows) or total != expected_sum * (count // args.rows):
+                            raise RuntimeError(f'Unexpected retry result: 
count={count}, distinct={distinct}, sum={total}')
+                        print(f'OBSERVED: retry produced {count} rows with 
{distinct} distinct IDs', flush=True)
+                    cursor.execute(f'DROP TABLE IF EXISTS {table} PURGE')
+                    cursor.execute(f'DROP DATABASE IF EXISTS {database}')
+                    created = False
+                    print('PASS: owned test data cleaned')
+                    return 0
+                time.sleep(min(2, max(0, deadline - time.monotonic())))
+            print('INCONCLUSIVE: commit was not observed; outcome may still 
change')
+            return 2
+    finally:
+        stop_process(process)
+        proxy.close()
+        if created:
+            print(f'Leave {database} untouched until the server write has 
stopped; then remove its table and database.', file=sys.stderr)
+
+
+if __name__ == '__main__':
+    # Child flags are internal and generated by the parent; reject arbitrary 
SQL identifiers.
+    if '--child-table' in sys.argv:
+        import re
+        p = argparse.ArgumentParser()
+        p.add_argument('--port', type=int, required=True)
+        p.add_argument('--driver', required=True)
+        p.add_argument('--rows', type=int, required=True)
+        p.add_argument('--child-port', type=int, required=True)
+        p.add_argument('--child-table', required=True)
+        a = p.parse_args()
+        if not re.fullmatch(r'connect_lost_response_[0-9a-f]{32}\.orders', 
a.child_table) or a.rows < 1:
+            p.error('invalid internal child arguments')
+        raise SystemExit(child(a.child_port, a.driver, a.child_table, a.rows))
+    raise SystemExit(main())
diff --git a/hudi-lakehouse/local-dev/example/connect_nullable_array.py 
b/hudi-lakehouse/local-dev/example/connect_nullable_array.py
new file mode 100644
index 000000000000..2fd6574a3bee
--- /dev/null
+++ b/hudi-lakehouse/local-dev/example/connect_nullable_array.py
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Check Hudi nullable-array write/read on a disposable localhost endpoint."""
+import argparse
+import uuid
+
+from adbc_driver_manager import dbapi
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--port', type=int, required=True)
+    parser.add_argument('--driver', default='spark')
+    args = parser.parse_args()
+    if not 1 <= args.port <= 65535:
+        parser.error('invalid port')
+    uri = f'spark://127.0.0.1:{args.port}?api=connect&auth_type=none&tls=false'
+    database = 'connect_nullable_' + uuid.uuid4().hex
+    print(f'Test database: {database}', flush=True)
+    created = False
+    with dbapi.connect(driver=args.driver, db_kwargs={'uri': uri}, 
autocommit=True) as conn, conn.cursor() as cursor:
+        try:
+            cursor.execute(f'CREATE DATABASE {database}')
+            created = True
+            cursor.execute(f"CREATE TABLE {database}.orders (id INT, arr 
ARRAY<INT>, ts BIGINT) USING hudi "
+                           "TBLPROPERTIES (type='cow', primaryKey='id', 
preCombineField='ts')")
+            cursor.execute(f'INSERT INTO {database}.orders VALUES (1, array(1, 
CAST(NULL AS INT), 3), 1)')
+            cursor.execute(f'SELECT id, arr FROM {database}.orders')
+            rows = cursor.fetchall()
+            if rows != [(1, [1, None, 3])]:
+                raise AssertionError(f'Unexpected rows: {rows}')
+            print(f'PASS: nullable array readback: {rows}', flush=True)
+            cursor.execute(f'DROP TABLE IF EXISTS {database}.orders PURGE')
+            cursor.execute(f'DROP DATABASE IF EXISTS {database}')
+            created = False
+            print('PASS: owned test data cleaned', flush=True)
+        finally:
+            if created:
+                print(f'Leave {database} untouched until the server write has 
stopped; then remove its table and database.', flush=True)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/hudi-lakehouse/local-dev/example/connect_readiness.py 
b/hudi-lakehouse/local-dev/example/connect_readiness.py
new file mode 100644
index 000000000000..08a8c8cfb8b0
--- /dev/null
+++ b/hudi-lakehouse/local-dev/example/connect_readiness.py
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Read-only SQL, catalog and storage checks for an existing Connect 
endpoint."""
+import argparse
+import re
+import sys
+
+from adbc_driver_manager import dbapi
+
+
+def identifier(value):
+    if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', value):
+        raise argparse.ArgumentTypeError('identifier must contain only 
letters, numbers and underscores')
+    return value
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--uri', required=True)
+    parser.add_argument('--driver', default='spark')
+    parser.add_argument('--schema', required=True, type=identifier)
+    parser.add_argument('--table', required=True, type=identifier)
+    parser.add_argument('--column', required=True, type=identifier,
+                        help='non-null data column in an existing non-empty 
storage-backed table')
+    args = parser.parse_args()
+    checks = (
+        ('sql', 'SELECT 1', lambda rows: rows == [(1,)]),
+        ('catalog', 'SHOW DATABASES', lambda rows: any(row[0] == args.schema 
for row in rows)),
+        ('storage', f'SELECT `{args.column}` FROM 
`{args.schema}`.`{args.table}` '
+                    f'WHERE `{args.column}` IS NOT NULL LIMIT 1', lambda rows: 
bool(rows)),
+    )
+    try:
+        with dbapi.connect(driver=args.driver, db_kwargs={'uri': args.uri}, 
autocommit=True) as conn:
+            with conn.cursor() as cursor:
+                for name, sql, valid in checks:
+                    try:
+                        cursor.execute(sql)
+                        if not valid(cursor.fetchall()):
+                            raise ValueError('expected result missing')
+                        print(f'PASS: {name}', flush=True)
+                    except Exception as exc:
+                        print(f'FAIL: {name}: {exc}', file=sys.stderr, 
flush=True)
+                        return 1
+    except Exception as exc:
+        print(f'FAIL: connection: {exc}', file=sys.stderr, flush=True)
+        return 1
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())
diff --git a/hudi-lakehouse/local-dev/example/connect_restart_fault.py 
b/hudi-lakehouse/local-dev/example/connect_restart_fault.py
new file mode 100644
index 000000000000..543dff0fbefb
--- /dev/null
+++ b/hudi-lakehouse/local-dev/example/connect_restart_fault.py
@@ -0,0 +1,262 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Opt-in driver interruption experiment. Run only on a disposable endpoint."""
+
+import argparse
+from contextlib import ExitStack
+import json
+import queue
+import re
+import subprocess
+import sys
+import threading
+import time
+import uuid
+from adbc_driver_manager import dbapi
+
+
+def connect_ready(connect, port):
+    """Transfer ownership only after the connection can execute a query."""
+    with ExitStack() as stack:
+        connection = connect(port)
+        stack.callback(connection.close)
+        cursor = connection.cursor()
+        stack.callback(cursor.close)
+        cursor.execute("SELECT 1")
+        cursor.fetchall()
+        stack.pop_all()
+        return connection, cursor
+
+
+def connect(port, driver):
+    uri = f"spark://127.0.0.1:{port}?api=connect&auth_type=none&tls=false"
+    return dbapi.connect(driver=driver, db_kwargs={"uri": uri}, 
autocommit=True)
+
+
+def writer_command(uri, table, driver):
+    source = """from adbc_driver_manager import dbapi\nimport sys\nwith 
dbapi.connect(driver=sys.argv[3],db_kwargs={"uri":sys.argv[1]},autocommit=True) 
as c,c.cursor() as q:\n q.execute(f"INSERT INTO {sys.argv[2]} SELECT CAST(id AS 
INT) AS id, CAST(id AS BIGINT) AS ts FROM range(100000)")\n"""
+    return [sys.executable, "-c", source, uri, table, driver]
+
+
+def main():
+    p = argparse.ArgumentParser(
+        description="Interrupt a disposable Connect deployment at a Hudi 
commit stage."
+    )
+    p.add_argument(
+        "--restart-endpoint",
+        action="store_true",
+        required=True,
+        help="confirm that active sessions can be terminated",
+    )
+    p.add_argument("--stage", choices=("requested", "inflight"), required=True)
+    p.add_argument(
+        "--port", type=int, default=15039, help="existing localhost 
port-forward"
+    )
+    p.add_argument(
+        "--reconnect-port",
+        type=int,
+        default=15040,
+        help="unused localhost port for the restarted endpoint",
+    )
+    p.add_argument("--namespace", default="hudi-lakehouse")
+    p.add_argument("--deployment", default="hudi-spark-connect")
+    p.add_argument("--driver", default="spark")
+    a = p.parse_args()
+    if not (
+        1 <= a.port <= 65535
+        and 1 <= a.reconnect_port <= 65535
+        and a.port != a.reconnect_port
+    ):
+        p.error("ports must be distinct valid TCP ports")
+    ns = a.namespace
+    deploy = a.deployment
+    stage = a.stage
+    db = "connect_restart_" + uuid.uuid4().hex
+    table = db + ".orders"
+    print("DB", db, "STAGE", stage, flush=True)
+
+    def uri(port):
+        return f"spark://127.0.0.1:{port}?api=connect&auth_type=none&tls=false"
+
+    def kubectl(*a):
+        return subprocess.run(
+            ["kubectl", "-n", ns, *a],
+            check=True,
+            text=True,
+            capture_output=True,
+            timeout=360,
+        ).stdout
+
+    with connect(a.port, a.driver) as c, c.cursor() as q:
+        q.execute("CREATE DATABASE " + db)
+        q.execute(
+            f"CREATE TABLE {table} (id INT, ts BIGINT) USING hudi 
TBLPROPERTIES(type='cow',primaryKey='id',preCombineField='ts')"
+        )
+    deployment = json.loads(kubectl("get", "deployment", deploy, "-o", "json"))
+    selector = ",".join(
+        f"{key}={value}"
+        for key, value in deployment["spec"]["selector"]["matchLabels"].items()
+    )
+    pods = json.loads(kubectl("get", "pods", "-l", selector, "-o", 
"json"))["items"]
+    pods = [
+        item
+        for item in pods
+        if item["status"].get("phase") == "Running"
+        and not item["metadata"].get("deletionTimestamp")
+    ]
+    if len(pods) != 1:
+        print("INCONCLUSIVE: expected one running driver; preserve", db, 
flush=True)
+        sys.exit(2)
+    pod = pods[0]["metadata"]["name"]
+    print("POD", pod, flush=True)
+    logs = subprocess.Popen(
+        ["kubectl", "-n", ns, "logs", "-f", pod, "--tail=0"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        text=True,
+        bufsize=1,
+    )
+    lines = queue.Queue()
+
+    def readlogs():
+        for line in logs.stdout:
+            lines.put(line)
+
+    threading.Thread(target=readlogs, daemon=True).start()
+    time.sleep(2)
+    writer = subprocess.Popen(
+        writer_command(uri(a.port), table, a.driver),
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    instant = None
+    deadline = time.monotonic() + 90
+    try:
+        while time.monotonic() < deadline:
+            try:
+                line = lines.get(timeout=1)
+            except queue.Empty:
+                if writer.poll() is not None:
+                    print("Writer exited before targeted stage", 
writer.returncode)
+                    break
+                continue
+            if stage == "requested":
+                m = re.search(
+                    r"Creating a new instant \[==>(\d+)__commit__REQUESTED\]", 
line
+                )
+            else:
+                m = re.search(
+                    re.escape(table.replace(".", ".db/"))
+                    + r"/\.hoodie/timeline/(\d+)\.inflight",
+                    line,
+                )
+            if m:
+                instant = m.group(1)
+                print("MATCH", line.strip(), flush=True)
+                break
+        if not instant:
+            print("INCONCLUSIVE preserve", db, flush=True)
+            sys.exit(2)
+        print("DELETE DRIVER", flush=True)
+        kubectl("delete", "pod", pod, "--force", "--grace-period=0", 
"--wait=true")
+        kubectl("rollout", "status", "deployment/" + deploy, "--timeout=300s")
+        try:
+            writer.wait(timeout=30)
+        except subprocess.TimeoutExpired:
+            writer.terminate()
+            writer.wait(timeout=10)
+        print("Writer exit", writer.returncode, flush=True)
+        fwd = subprocess.Popen(
+            [
+                "kubectl",
+                "-n",
+                ns,
+                "port-forward",
+                "svc/" + deploy,
+                f"{a.reconnect_port}:15002",
+            ],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+        try:
+            time.sleep(2)
+            if fwd.poll() is not None:
+                raise RuntimeError("reconnect port-forward failed; preserve " 
+ db)
+            for i in range(60):
+                if fwd.poll() is not None:
+                    raise RuntimeError("reconnect port-forward exited; 
preserve " + db)
+                try:
+                    c, q = connect_ready(lambda port: connect(port, a.driver), 
a.reconnect_port)
+                    break
+                except Exception:
+                    time.sleep(2)
+            else:
+                raise RuntimeError("new server unavailable")
+            try:
+                q.execute(f"SELECT COUNT(*) FROM {table}")
+                rows = q.fetchall()
+                print("ROWS AFTER RESTART", rows, flush=True)
+                if rows != [(0,)]:
+                    print(
+                        "INCONCLUSIVE: original write is not empty; preserve",
+                        db,
+                        flush=True,
+                    )
+                    sys.exit(2)
+                q.execute(f"INSERT INTO {table} VALUES (-1, 999)")
+                q.execute(f"SELECT id, ts FROM {table} WHERE id=-1")
+                if q.fetchall() != [(-1, 999)]:
+                    raise RuntimeError(
+                        "recovery write returned unexpected rows; preserve " + 
db
+                    )
+                server = kubectl("logs", "deployment/" + deploy, 
"--tail=10000")
+                matches = [
+                    l
+                    for l in server.splitlines()
+                    if ("rollback" in l.lower() and (instant in l or table in 
l))
+                ]
+                print("ROLLBACK EVIDENCE", matches[:8], flush=True)
+                if not any(
+                    f"Rollback of Commits [{instant}] is complete" in line
+                    for line in matches
+                ):
+                    print(
+                        "INCONCLUSIVE: matching rollback completion not 
observed; preserve",
+                        db,
+                        flush=True,
+                    )
+                    sys.exit(2)
+                q.execute(f"DROP TABLE IF EXISTS {table} PURGE")
+                q.execute(f"DROP DATABASE IF EXISTS {db}")
+                print("PASS recovered and cleaned", flush=True)
+            finally:
+                try:
+                    q.close()
+                finally:
+                    c.close()
+        finally:
+            fwd.terminate()
+            fwd.wait(timeout=10)
+    finally:
+        logs.terminate()
+        logs.wait(timeout=10)
+        if writer.poll() is None:
+            writer.terminate()
+            writer.wait(timeout=10)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/hudi-lakehouse/tests/test_connect_external_writer.py 
b/hudi-lakehouse/tests/test_connect_external_writer.py
index 5f043f3ce044..508afcc8ad9d 100644
--- a/hudi-lakehouse/tests/test_connect_external_writer.py
+++ b/hudi-lakehouse/tests/test_connect_external_writer.py
@@ -57,6 +57,11 @@ class ExternalWriterTest(unittest.TestCase):
         self.assertEqual(container["image"], 
self.template["containers"][0]["image"])
         self.assertTrue(any(v["name"] == "config" for v in 
pod["spec"]["volumes"]))
 
+    def test_writer_image_can_be_overridden(self):
+        pod = self.writer.writer_pod(self.template, "private-writer", 
"connect_smoke_test",
+                                     "commit-schema", 
image="spark35-writer:test")
+        self.assertEqual(pod["spec"]["containers"][0]["image"], 
"spark35-writer:test")
+
     def run_launch(self, failure=None):
         calls = []
 
@@ -85,6 +90,12 @@ class ExternalWriterTest(unittest.TestCase):
                 self.writer.launch("connect_smoke_test", "commit-schema")
         return [call[3:] for call in calls]
 
+    def test_launch_passes_image_override(self):
+        with patch.dict(self.writer.os.environ, {"EXTERNAL_WRITER_IMAGE": 
"spark35-writer:test"}), \
+                patch.object(self.writer, "writer_pod", 
wraps=self.writer.writer_pod) as pod:
+            self.run_launch()
+        self.assertEqual(pod.call_args.args[-1], "spark35-writer:test")
+
     def test_cleanup_only_deletes_resources_created_by_this_run(self):
         pod = ["delete", "pod", "connect-smoke-test-writer"]
         config = ["delete", "configmap", "connect-smoke-test-writer"]
diff --git a/hudi-lakehouse/tests/test_connect_fault_regressions.py 
b/hudi-lakehouse/tests/test_connect_fault_regressions.py
new file mode 100644
index 000000000000..f8710c2a3bb5
--- /dev/null
+++ b/hudi-lakehouse/tests/test_connect_fault_regressions.py
@@ -0,0 +1,437 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import contextlib
+import importlib.util
+import io
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+import types
+import unittest
+from unittest.mock import MagicMock, patch
+
+EXAMPLES = Path(__file__).resolve().parents[1] / 'local-dev/example'
+sys.path.insert(0, str(EXAMPLES))
+
+import connect_fault_utils as utils
+
+
+def load(name):
+    spec = importlib.util.spec_from_file_location(name, EXAMPLES / (name + 
'.py'))
+    module = importlib.util.module_from_spec(spec)
+    package = types.ModuleType('adbc_driver_manager')
+    package.dbapi = MagicMock()
+    with patch.dict(sys.modules, {'adbc_driver_manager': package}):
+        spec.loader.exec_module(module)
+    return module
+
+
+def connection_with_cursor(cursor):
+    conn = MagicMock()
+    conn.__enter__.return_value = conn
+    conn.cursor.return_value.__enter__.return_value = cursor
+    return conn
+
+
+def mock_readback(cursor):
+    def observe(uri, driver, table, deadline):
+        cursor.execute(f'REFRESH TABLE {table}')
+        cursor.execute(f'SELECT COUNT(*), SUM(id) FROM {table}')
+        return cursor.fetchall()[0]
+    return observe
+
+
+class FaultFailureTest(unittest.TestCase):
+    def test_cancel_logs_pending_only_when_timeline_changes(self):
+        module = load('connect_cancel')
+        observer, writer = MagicMock(), MagicMock()
+        writer.execute.side_effect = RuntimeError('cancelled')
+        thread = MagicMock()
+        thread.is_alive.return_value = True
+        thread.join.side_effect = lambda timeout: setattr(thread.is_alive, 
'return_value', False)
+
+        def make_thread(target, daemon):
+            thread.start.side_effect = target
+            return thread
+
+        pending = [('100', 'commit', 'INFLIGHT')]
+        changed = [('101', 'rollback', 'INFLIGHT')]
+        with patch.object(module, 'connect', 
side_effect=[connection_with_cursor(observer), 
connection_with_cursor(writer)]), \
+             patch.object(module, 'readback', return_value=(0, None)), \
+             patch.object(module, 'read_timeline', side_effect=[(pending, []), 
(pending, []), (changed, [])]), \
+             patch.object(module.threading, 'Thread', 
side_effect=make_thread), \
+             patch.object(module.time, 'monotonic', side_effect=[0, 0, 0, 1, 
1, 2, 2, 3]), \
+             patch.object(module.time, 'sleep'), \
+             patch.object(sys, 'argv', ['test', '--port', '15002', '--wait', 
'3']), \
+             contextlib.redirect_stdout(io.StringIO()) as output:
+            self.assertEqual(module.main(), 2)
+        self.assertEqual(output.getvalue().count('PENDING: no visible rows'), 
2)
+        self.assertIn('INCONCLUSIVE: complete commit was not observed', 
output.getvalue())
+        self.assertNotIn('OBSERVED: no visible rows', output.getvalue())
+
+    def test_cancel_reports_completed_rollback_without_cleaning_table(self):
+        module = load('connect_cancel')
+        observer, writer = MagicMock(), MagicMock()
+        writer.execute.side_effect = RuntimeError('cancelled')
+        thread = MagicMock()
+        thread.is_alive.return_value = True
+        thread.join.side_effect = lambda timeout: setattr(thread.is_alive, 
'return_value', False)
+
+        def make_thread(target, daemon):
+            thread.start.side_effect = target
+            return thread
+
+        with patch.object(module, 'connect', 
side_effect=[connection_with_cursor(observer), 
connection_with_cursor(writer)]), \
+             patch.object(module, 'readback', return_value=(0, None)), \
+             patch.object(module, 'read_timeline', return_value=(
+                 [('101', 'rollback', 'COMPLETED')], [('101', '100', 0, 0, 
0)])), \
+             patch.object(module.threading, 'Thread', 
side_effect=make_thread), \
+             patch.object(module.time, 'sleep'), \
+             patch.object(sys, 'argv', ['test', '--port', '15002']), \
+             contextlib.redirect_stdout(io.StringIO()) as output:
+            self.assertEqual(module.main(), 0)
+        self.assertIn('write instant 100 rolled back', output.getvalue())
+        self.assertIn('Server-side work may still be running', 
output.getvalue())
+        self.assertFalse(any(call.args[0].startswith('DROP ') for call in 
observer.execute.call_args_list))
+
+    def test_pending_or_committed_timeline_is_not_rollback_success(self):
+        module = load('connect_cancel')
+        rollback = [('101', '100', 0, 0, 0)]
+        for timeline, records in (
+            ([('101', 'rollback', 'COMPLETED'), ('100', 'commit', 
'INFLIGHT')], rollback),
+            ([('101', 'rollback', 'COMPLETED'), ('100', 'commit', 
'COMPLETED')], rollback),
+            ([('101', 'rollback', 'COMPLETED')], []),
+            ([('102', 'rollback', 'COMPLETED')], rollback),
+            ([], rollback),
+        ):
+            with self.subTest(timeline=timeline, records=records):
+                self.assertIsNone(module.rolled_back_write(timeline, records))
+
+    def test_lost_response_retry_failure_stops_without_repeating_write(self):
+        module = load('connect_lost_response')
+        cursor = MagicMock()
+        cursor.fetchall.return_value = [(2, 1)]
+        def execute(sql):
+            if 'COUNT(DISTINCT id)' in sql:
+                raise RuntimeError('retry readback failed')
+        cursor.execute.side_effect = execute
+        process = MagicMock()
+        process.stdout.readline.return_value = 'READY\n'
+        process.poll.return_value = None
+        proxy = MagicMock()
+        with patch.object(module, 'readback', 
side_effect=mock_readback(cursor)), \
+             patch.object(module, 'connect', 
return_value=connection_with_cursor(cursor)), \
+             patch.object(module, 'ResponseDropProxy', return_value=proxy), \
+             patch.object(module.subprocess, 'Popen', return_value=process), \
+             patch.object(module.select, 'select', 
return_value=([process.stdout], [], [])), \
+             patch('sys.argv', ['test', '--port', '15002', '--rows', '2', 
'--retry']), \
+             self.assertRaisesRegex(RuntimeError, 'retry readback failed'):
+            module.main()
+        statements = [call.args[0] for call in cursor.execute.call_args_list]
+        self.assertEqual(sum(sql.startswith('INSERT INTO') for sql in 
statements), 1)
+        self.assertFalse(any(sql.startswith('DROP ') for sql in statements))
+
+    def test_lost_response_retries_transient_observation_only(self):
+        module = load('connect_lost_response')
+        cursor = MagicMock()
+        cursor.fetchall.return_value = [(2, 1)]
+        failures = []
+
+        def execute(sql):
+            if sql.startswith('REFRESH TABLE') and not failures:
+                failures.append(sql)
+                raise RuntimeError('temporary read error')
+
+        cursor.execute.side_effect = execute
+        process = MagicMock()
+        process.stdout.readline.return_value = 'READY\n'
+        process.poll.return_value = None
+        with patch.object(module, 'readback', 
side_effect=mock_readback(cursor)), \
+             patch.object(module, 'connect', 
return_value=connection_with_cursor(cursor)), \
+             patch.object(module, 'ResponseDropProxy', 
return_value=MagicMock()), \
+             patch.object(module.subprocess, 'Popen', return_value=process), \
+             patch.object(module.select, 'select', 
return_value=([process.stdout], [], [])), \
+             patch.object(module.time, 'sleep'), \
+             patch('sys.argv', ['test', '--port', '15002', '--rows', '2']):
+            self.assertEqual(module.main(), 0)
+        statements = [call.args[0] for call in cursor.execute.call_args_list]
+        self.assertEqual(sum(sql.startswith('REFRESH TABLE') for sql in 
statements), 2)
+        self.assertEqual(sum(sql.startswith('DROP DATABASE') for sql in 
statements), 1)
+
+    def test_lost_response_cleanup_failure_is_not_readback_failure(self):
+        module = load('connect_lost_response')
+        cursor = MagicMock()
+        cursor.fetchall.return_value = [(2, 1)]
+        def execute(sql):
+            if sql.startswith('DROP DATABASE'):
+                raise RuntimeError('cleanup failed')
+        cursor.execute.side_effect = execute
+        process = MagicMock()
+        process.stdout.readline.return_value = 'READY\n'
+        process.poll.return_value = None
+        with patch.object(module, 'readback', 
side_effect=mock_readback(cursor)), \
+             patch.object(module, 'connect', 
return_value=connection_with_cursor(cursor)), \
+             patch.object(module, 'ResponseDropProxy', 
return_value=MagicMock()), \
+             patch.object(module.subprocess, 'Popen', return_value=process), \
+             patch.object(module.select, 'select', 
return_value=([process.stdout], [], [])), \
+             patch('sys.argv', ['test', '--port', '15002', '--rows', '2']), \
+             self.assertRaisesRegex(RuntimeError, 'cleanup failed'):
+            module.main()
+        statements = [call.args[0] for call in cursor.execute.call_args_list]
+        self.assertEqual(sum(sql.startswith('REFRESH TABLE') for sql in 
statements), 1)
+        self.assertEqual(sum(sql.startswith('DROP DATABASE') for sql in 
statements), 1)
+
+    def test_disconnect_cleanup_failure_is_not_retried(self):
+        module = load('connect_disconnect')
+        cursor = MagicMock()
+        cursor.fetchall.return_value = [(2, 1)]
+        def execute(sql):
+            if sql.startswith('DROP DATABASE'):
+                raise RuntimeError('cleanup failed')
+        cursor.execute.side_effect = execute
+        process = MagicMock()
+        process.stdout.readline.return_value = 'READY\n'
+        process.poll.return_value = None
+        with patch.object(module, 'readback', 
side_effect=mock_readback(cursor)), \
+             patch.object(module, 'connect', 
return_value=connection_with_cursor(cursor)), \
+             patch.object(module.subprocess, 'Popen', return_value=process), \
+             patch.object(module, 'interrupt_before_response', 
return_value=True), \
+             patch.object(module.time, 'sleep'), \
+             patch('sys.argv', ['test', '--uri', 'unused', '--rows', '2']), \
+             self.assertRaisesRegex(RuntimeError, 'cleanup failed'):
+            module.main()
+        statements = [call.args[0] for call in cursor.execute.call_args_list]
+        self.assertEqual(sum(sql.startswith('REFRESH TABLE') for sql in 
statements), 1)
+        self.assertEqual(sum(sql.startswith('DROP DATABASE') for sql in 
statements), 1)
+
+    def test_cancel_cleanup_failure_is_not_retried(self):
+        module = load('connect_cancel')
+        read = MagicMock()
+        read.fetchall.return_value = [(2, 1)]
+        def execute(sql):
+            if sql.startswith('DROP DATABASE'):
+                raise RuntimeError('cleanup failed')
+        read.execute.side_effect = execute
+        writer = MagicMock()
+        thread = MagicMock()
+        thread.is_alive.side_effect = [True, False]
+        with patch.object(module, 'readback', 
side_effect=mock_readback(read)), \
+             patch.object(module, 'connect', 
side_effect=[connection_with_cursor(read), connection_with_cursor(writer)]), \
+             patch.object(module.threading, 'Thread', return_value=thread), \
+             patch.object(module.threading, 'Event', 
return_value=MagicMock()), \
+             patch.object(module.time, 'sleep'), \
+             patch('sys.argv', ['test', '--port', '15002', '--rows', '2']), \
+             self.assertRaisesRegex(RuntimeError, 'cleanup failed'):
+            module.main()
+        statements = [call.args[0] for call in read.execute.call_args_list]
+        self.assertEqual(sum(sql.startswith('REFRESH TABLE') for sql in 
statements), 1)
+        self.assertEqual(sum(sql.startswith('DROP DATABASE') for sql in 
statements), 1)
+
+
+class ReconnectTest(unittest.TestCase):
+    def test_custom_driver_used_for_connections_and_writer(self):
+        module = load('connect_restart_fault')
+        with patch.object(module.dbapi, 'connect') as connect:
+            module.connect(15039, '/tmp/custom driver.so')
+        self.assertEqual(connect.call_args.kwargs['driver'], '/tmp/custom 
driver.so')
+
+        with tempfile.TemporaryDirectory() as directory:
+            Path(directory, 'adbc_driver_manager.py').write_text('''
+class Connection:
+    def __enter__(self): return self
+    def __exit__(self, *args): pass
+    def cursor(self): return self
+    def execute(self, sql): print(sql)
+class dbapi:
+    @staticmethod
+    def connect(**kwargs):
+        print(kwargs['driver'])
+        return Connection()
+''')
+            result = subprocess.run(
+                module.writer_command('spark://localhost', 'test.orders', 
'/tmp/custom driver.so'),
+                env={**os.environ, 'PYTHONPATH': directory}, text=True, 
capture_output=True, check=True)
+        self.assertIn('/tmp/custom driver.so\n', result.stdout)
+        self.assertIn('INSERT INTO test.orders', result.stdout)
+
+    def test_failed_query_closes_cursor_then_connection(self):
+        module = load('connect_restart_fault')
+        conn = MagicMock()
+        cursor = conn.cursor.return_value
+        cursor.execute.side_effect = RuntimeError('not ready')
+        with self.assertRaisesRegex(RuntimeError, 'not ready'):
+            module.connect_ready(lambda port: conn, 15002)
+        cursor.close.assert_called_once()
+        conn.close.assert_called_once()
+
+    def test_failed_cursor_creation_closes_connection(self):
+        module = load('connect_restart_fault')
+        conn = MagicMock()
+        conn.cursor.side_effect = RuntimeError('cursor failed')
+        with self.assertRaisesRegex(RuntimeError, 'cursor failed'):
+            module.connect_ready(lambda port: conn, 15002)
+        conn.close.assert_called_once()
+
+    def test_success_transfers_ownership_to_caller(self):
+        module = load('connect_restart_fault')
+        conn = MagicMock()
+        actual_conn, cursor = module.connect_ready(lambda port: conn, 15002)
+        self.assertIs(actual_conn, conn)
+        self.assertIs(cursor, conn.cursor.return_value)
+        cursor.close.assert_not_called()
+        conn.close.assert_not_called()
+
+
+class ProcessTest(unittest.TestCase):
+    def test_response_and_interruption(self):
+        module = load('connect_disconnect')
+        scenarios = (
+            ('response', 'import time; print("READY\\nRESPONSE", flush=True); 
time.sleep(60)', False),
+            ('not_ready', 'import time; print("NOT_READY", flush=True); 
time.sleep(60)', False),
+            ('unanswered', 'import time; print("READY", flush=True); 
time.sleep(60)', True),
+            ('racing', 'import signal, time, sys; 
signal.signal(signal.SIGTERM, '
+             'lambda *args: (print("RESPONSE", flush=True), sys.exit(0))); '
+             'print("READY", flush=True); time.sleep(60)', False),
+        )
+        for name, code, expected in scenarios:
+            with self.subTest(scenario=name):
+                child = subprocess.Popen([sys.executable, '-c', code], 
stdout=subprocess.PIPE, text=True)
+                try:
+                    self.assertEqual(module.interrupt_before_response(child, 
0.1), expected)
+                    self.assertIsNotNone(child.poll())
+                finally:
+                    utils.stop_process(child)
+
+    def test_client_processes_are_reaped(self):
+        child = subprocess.Popen([sys.executable, '-c',
+            'import signal, time; signal.signal(signal.SIGTERM, 
signal.SIG_IGN); '
+            'print("READY", flush=True); time.sleep(60)'], 
stdout=subprocess.PIPE, text=True)
+        try:
+            self.assertEqual(child.stdout.readline().strip(), 'READY')
+            utils.stop_process(child)
+            self.assertEqual(child.returncode, -9)
+        finally:
+            utils.stop_process(child)
+            child.stdout.close()
+
+        module = load('connect_disconnect')
+        cursor = MagicMock()
+        child = MagicMock()
+        child.poll.return_value = None
+        with patch.object(module, 'connect', 
return_value=connection_with_cursor(cursor)), \
+             patch.object(module.subprocess, 'Popen', return_value=child), \
+             patch.object(module, 'interrupt_before_response', 
side_effect=KeyboardInterrupt), \
+             patch.object(sys, 'argv', ['test', '--uri', 'unused']), \
+             contextlib.redirect_stdout(io.StringIO()), 
contextlib.redirect_stderr(io.StringIO()):
+            with self.assertRaises(KeyboardInterrupt):
+                module.main()
+        child.terminate.assert_called_once()
+        child.wait.assert_called_once_with(timeout=2)
+        self.assertFalse(any(call.args[0].startswith('DROP ') for call in 
cursor.execute.call_args_list))
+
+
+class ReadbackProcessTest(unittest.TestCase):
+    def test_native_client_stalls_are_bounded_and_reaped(self):
+        # A stand-in DBAPI runs in a real child, including connection and 
close.
+        source = """import os, time
+class Connection:
+    def __init__(self): self.stall('connect')
+    def stall(self, stage):
+        if os.environ.get('STALL_STAGE') == stage: time.sleep(60)
+    def __enter__(self): return self
+    def __exit__(self, *args): self.stall('close')
+    def cursor(self): return self
+    def execute(self, sql): self.stall('execute')
+    def fetchall(self):
+        self.stall('fetch')
+        return [(2, 1)]
+class dbapi:
+    @staticmethod
+    def connect(**kwargs): return Connection()
+"""
+        with tempfile.TemporaryDirectory() as directory:
+            Path(directory, 'adbc_driver_manager.py').write_text(source)
+            for stage in ('connect', 'execute', 'fetch', 'close', ''):
+                with self.subTest(stage=stage), patch.dict(os.environ, 
{'PYTHONPATH': directory, 'STALL_STAGE': stage}):
+                    processes = []
+                    popen = subprocess.Popen
+                    def start(*args, **kwargs):
+                        process = popen(*args, **kwargs)
+                        processes.append(process)
+                        return process
+                    started = time.monotonic()
+                    with patch.object(utils.subprocess, 'Popen', 
side_effect=start):
+                        args = ('unused', 'spark', 'connect_disconnect_' + 'a' 
* 32 + '.orders', started + 0.5)
+                        if stage:
+                            with self.assertRaises(TimeoutError):
+                                utils.readback(*args)
+                        else:
+                            self.assertEqual(utils.readback(*args), (2, 1))
+                    self.assertLess(time.monotonic() - started, 5)
+                    self.assertIsNotNone(processes[0].poll())
+
+    def test_result_after_deadline_is_rejected(self):
+        process = MagicMock()
+        process.communicate.return_value = ('[2, 1]', '')
+        process.returncode = 0
+        process.poll.return_value = 0
+        with patch.object(utils.subprocess, 'Popen', return_value=process), \
+             patch.object(utils.time, 'monotonic', side_effect=[0, 0, 100]):
+            with self.assertRaises(TimeoutError):
+                utils.readback('unused', 'spark', 'unused', 1)
+
+    def test_deadline_failure_never_retries_writes_or_cleans_table(self):
+        for name, argv in (
+            ('connect_disconnect', ['--uri', 'unused']),
+            ('connect_cancel', ['--port', '15002']),
+            ('connect_lost_response', ['--port', '15002', '--retry']),
+        ):
+            with self.subTest(name=name):
+                module = load(name)
+                cursor = MagicMock()
+                connection = connection_with_cursor(cursor)
+                process = MagicMock()
+                process.poll.return_value = None
+                process.stdout.readline.return_value = 'READY\n'
+                with contextlib.ExitStack() as stack:
+                    stack.enter_context(patch.object(module, 'connect', 
return_value=connection))
+                    observe = stack.enter_context(patch.object(module, 
'readback', side_effect=TimeoutError))
+                    stack.enter_context(patch.object(sys, 'argv', ['test', 
*argv]))
+                    
stack.enter_context(contextlib.redirect_stdout(io.StringIO()))
+                    
stack.enter_context(contextlib.redirect_stderr(io.StringIO()))
+                    if name == 'connect_cancel':
+                        thread = MagicMock(); thread.is_alive.side_effect = 
[True, False]
+                        stack.enter_context(patch.object(module.threading, 
'Thread', return_value=thread))
+                        stack.enter_context(patch.object(module.threading, 
'Event', return_value=MagicMock()))
+                        stack.enter_context(patch.object(module.time, 'sleep'))
+                    else:
+                        stack.enter_context(patch.object(module.subprocess, 
'Popen', return_value=process))
+                        if name == 'connect_disconnect':
+                            stack.enter_context(patch.object(module, 
'interrupt_before_response', return_value=True))
+                        else:
+                            stack.enter_context(patch.object(module, 
'ResponseDropProxy', return_value=MagicMock()))
+                            stack.enter_context(patch.object(module.select, 
'select', return_value=([process.stdout], [], [])))
+                    self.assertEqual(module.main(), 2)
+                    observe.assert_called_once()
+                self.assertFalse(any(call.args[0].startswith(('DROP ', 'INSERT 
'))
+                                     for call in 
cursor.execute.call_args_list))
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/hudi-lakehouse/tests/test_connect_lost_response.py 
b/hudi-lakehouse/tests/test_connect_lost_response.py
new file mode 100644
index 000000000000..dd2b1dafe7cb
--- /dev/null
+++ b/hudi-lakehouse/tests/test_connect_lost_response.py
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import importlib.util
+from pathlib import Path
+import socket
+import sys
+import threading
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 
'local-dev/example'))
+
+spec = importlib.util.spec_from_file_location('lost_response', 
Path(__file__).resolve().parents[1] / 
'local-dev/example/connect_lost_response.py')
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+
+class LostResponseProxyTest(unittest.TestCase):
+    def test_forwards_requests_but_drops_gated_responses(self):
+        listener = socket.socket()
+        listener.bind(('127.0.0.1', 0))
+        listener.listen(1)
+        received = []
+        done = threading.Event()
+
+        def serve():
+            conn, _ = listener.accept()
+            with conn:
+                for _ in range(2):
+                    data = conn.recv(1024)
+                    received.append(data)
+                    conn.sendall(data.upper())
+                done.set()
+
+        server = threading.Thread(target=serve, daemon=True)
+        server.start()
+        proxy = module.ResponseDropProxy(listener.getsockname()[1])
+        proxy.start()
+        try:
+            with socket.create_connection(('127.0.0.1', proxy.port), 
timeout=2) as client:
+                client.sendall(b'first')
+                self.assertEqual(client.recv(1024), b'FIRST')
+                proxy.drop.set()
+                client.sendall(b'second')
+                self.assertTrue(done.wait(2))
+                client.settimeout(0.25)
+                self.assertEqual(client.recv(1024), b'')
+                self.assertEqual(received, [b'first', b'second'])
+        finally:
+            proxy.close()
+            listener.close()
+            server.join(timeout=2)
+
+    def test_close_before_start(self):
+        proxy = module.ResponseDropProxy(1)
+        proxy.close()
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/hudi-lakehouse/tests/test_connect_probe.py 
b/hudi-lakehouse/tests/test_connect_probe.py
new file mode 100644
index 000000000000..a3ac276b69be
--- /dev/null
+++ b/hudi-lakehouse/tests/test_connect_probe.py
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import contextlib
+import importlib.util
+import io
+import os
+import pathlib
+import signal
+import subprocess
+import sys
+import tempfile
+import types
+import unittest
+from unittest.mock import MagicMock, patch
+
+SCRIPT = pathlib.Path(__file__).resolve().parents[1] / 
'images/spark-connect/check-ready.py'
+
+
+class ProbeTest(unittest.TestCase):
+    def setUp(self):
+        sql = types.ModuleType('pyspark.sql')
+        sql.SparkSession = MagicMock()
+        with patch.dict(sys.modules, {'pyspark': types.ModuleType('pyspark'), 
'pyspark.sql': sql}):
+            spec = importlib.util.spec_from_file_location('check_ready', 
SCRIPT)
+            self.probe = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(self.probe)
+        self.spark = 
sql.SparkSession.builder.remote.return_value.getOrCreate.return_value
+
+    def run_probe(self, results):
+        self.spark.sql.side_effect = [types.SimpleNamespace(collect=lambda 
rows=rows: rows) for rows in results]
+        args = [str(SCRIPT), '--schema', 'probe', '--table', 'orders', 
'--column', 'id']
+        with patch.object(sys, 'argv', args), 
contextlib.redirect_stdout(io.StringIO()), 
contextlib.redirect_stderr(io.StringIO()):
+            return self.probe.main()
+
+    def test_three_checks_and_session_cleanup(self):
+        self.assertEqual(self.run_probe([[(1,)], [('probe',)], [(42,)]]), 0)
+        self.assertEqual(self.spark.sql.call_count, 3)
+        self.spark.stop.assert_called_once()
+
+    def test_missing_schema_or_no_rows_fail_and_close(self):
+        for rows in ([[(1,)], [('other',)]], [[(1,)], [('probe',)], []]):
+            with self.subTest(rows=rows):
+                self.spark.reset_mock()
+                self.assertEqual(self.run_probe(rows), 1)
+                self.spark.stop.assert_called_once()
+
+    def test_identifier_validation(self):
+        with self.assertRaises(argparse.ArgumentTypeError):
+            self.probe.identifier('orders;DROP TABLE')
+
+    def test_deadline_terminates_blocked_probe(self):
+        with tempfile.TemporaryDirectory() as directory:
+            package = pathlib.Path(directory) / 'pyspark'
+            package.mkdir()
+            (package / '__init__.py').write_text('')
+            (package / 'sql.py').write_text('import time; time.sleep(60)')
+            code = ("import runpy, signal; original = signal.alarm; "
+                    "signal.alarm = lambda _: original(1); "
+                    f"runpy.run_path({str(SCRIPT)!r}, run_name='__main__')")
+            result = subprocess.run([sys.executable, '-c', code],
+                                    env={**os.environ, 'PYTHONPATH': 
directory}, timeout=5)
+            self.assertEqual(result.returncode, -signal.SIGALRM)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/hudi-lakehouse/tests/test_connect_readiness.py 
b/hudi-lakehouse/tests/test_connect_readiness.py
new file mode 100644
index 000000000000..dbe8e9c25212
--- /dev/null
+++ b/hudi-lakehouse/tests/test_connect_readiness.py
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import contextlib
+import importlib.util
+import io
+import pathlib
+import sys
+import types
+import unittest
+from unittest.mock import MagicMock, patch
+
+SCRIPT = pathlib.Path(__file__).resolve().parents[1] / 
'local-dev/example/connect_readiness.py'
+
+
+class ReadinessTest(unittest.TestCase):
+    def setUp(self):
+        self.dbapi = MagicMock()
+        package = types.ModuleType('adbc_driver_manager')
+        package.dbapi = self.dbapi
+        with patch.dict(sys.modules, {'adbc_driver_manager': package}):
+            spec = importlib.util.spec_from_file_location('connect_readiness', 
SCRIPT)
+            self.module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(self.module)
+        self.cursor = 
self.dbapi.connect.return_value.__enter__.return_value.cursor.return_value.__enter__.return_value
+
+    def run_check(self):
+        args = [str(SCRIPT), '--uri', 'spark://local', '--schema', 'probe', 
'--table', 'orders', '--column', 'id']
+        with patch.object(sys, 'argv', args), 
contextlib.redirect_stdout(io.StringIO()), 
contextlib.redirect_stderr(io.StringIO()):
+            return self.module.main()
+
+    def test_sql_catalog_and_data_must_each_pass(self):
+        self.cursor.fetchall.side_effect = [[(1,)], [('probe',)], [(10,)]]
+        self.assertEqual(self.run_check(), 0)
+        self.assertEqual(self.cursor.execute.call_count, 3)
+        self.assertIn('`probe`.`orders`', 
self.cursor.execute.call_args.args[0])
+
+    def test_missing_catalog_or_empty_storage_fails(self):
+        for values in ([(1,)], [('other',)], []), ([(1,)], [('probe',)], []):
+            with self.subTest(values=values):
+                self.cursor.reset_mock()
+                self.cursor.fetchall.side_effect = list(values)
+                self.assertEqual(self.run_check(), 1)
+
+    def test_identifiers_reject_sql(self):
+        for value in ('bad.name', 'bad`x', 'x;DROP', ''):
+            with self.subTest(value=value), 
self.assertRaises(argparse.ArgumentTypeError):
+                self.module.identifier(value)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/hudi-lakehouse/tests/test_spark_connect_chart.py 
b/hudi-lakehouse/tests/test_spark_connect_chart.py
index b309aeb8206f..a4b3b6dad77e 100644
--- a/hudi-lakehouse/tests/test_spark_connect_chart.py
+++ b/hudi-lakehouse/tests/test_spark_connect_chart.py
@@ -64,6 +64,19 @@ class ConnectChartTest(unittest.TestCase):
         self.assertFalse(pod["automountServiceAccountToken"])
         self.assertTrue(pod["securityContext"]["runAsNonRoot"])
 
+    def test_optional_sql_readiness_requires_safe_probe_table(self):
+        pod = render()["Deployment"]["spec"]["template"]["spec"]
+        
self.assertEqual(pod["containers"][0]["readinessProbe"]["tcpSocket"]["port"], 
"connect")
+        config = {"readiness": {"sql": {"enabled": True, "schema": "probe_db", 
"table": "probe", "column": "id"}}}
+        pod = render(config)["Deployment"]["spec"]["template"]["spec"]
+        probe = pod["containers"][0]["readinessProbe"]
+        self.assertEqual(probe["exec"]["command"], ["python3", 
"/opt/spark/check-ready.py",
+                         "--schema", "probe_db", "--table", "probe", 
"--column", "id"])
+        self.assertEqual(probe["timeoutSeconds"], 30)
+        for invalid in ("", "bad;drop", "catalog.schema"):
+            config["readiness"]["sql"]["table"] = invalid
+            render(config, success=False)
+
     def test_scratch_volume_size_limit(self):
         for size, expected in (("", {}), ("20Gi", {"sizeLimit": "20Gi"})):
             with self.subTest(size=size):

Reply via email to