This is an automated email from the ASF dual-hosted git repository.
wenjin272 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-agents.git
The following commit(s) were added to refs/heads/main by this push:
new 720a4ece4 [hotfix][ci] Finalize PyArrow 11 on exit in Flink 1.20 tests
(#1343)
720a4ece4 is described below
commit 720a4ece4e4224f89bac799681c6f8cd1e9ab081
Author: Wenjin Xie <[email protected]>
AuthorDate: Fri Oct 30 17:59:50 2026 +0800
[hotfix][ci] Finalize PyArrow 11 on exit in Flink 1.20 tests (#1343)
Generated-by: Codex CLI 0.153.4 (GPT-6)
---
.github/workflows/ci.yml | 6 +++++
tools/ci/patch_pyarrow11.py | 58 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 64 insertions(+)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 59679ed84..73b19b413 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -240,6 +240,12 @@ jobs:
run: bash tools/build.sh
- name: Install ollama
run: bash tools/start_ollama_server.sh
+ - name: Apply PyArrow shutdown workaround for Flink 1.20
+ if: matrix.flink-version == '1.20'
+ run: |
+ # ut.sh reuses these dependencies; patch the shared wheel for pytest
and workers.
+ uv pip install --python python/.venv/bin/python 'apache-flink~=${{
matrix.flink-version }}.0'
+ python/.venv/bin/python tools/ci/patch_pyarrow11.py
- name: Run Python IT
env:
ACTION_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
diff --git a/tools/ci/patch_pyarrow11.py b/tools/ci/patch_pyarrow11.py
new file mode 100644
index 000000000..f1a6ad723
--- /dev/null
+++ b/tools/ci/patch_pyarrow11.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#################################################################################
+"""Apply a CI-only shutdown workaround to the installed PyArrow 11 wheel.
+
+Flink 1.20's Beam dependency requires PyArrow <12. The wheel initializes AWS on
+import but does not register shutdown. Register its public cleanup function so
+pytest and Beam workers finalize AWS before C++ static destruction. This is not
+a backport of Arrow's complete lazy-initialization fix (apache/arrow#38364).
+Remove this workaround when the Flink 1.20 CI dependency can be upgraded.
+"""
+
+from importlib.metadata import distribution
+
+ORIGINAL = "\nelse:\n initialize_s3()\n\n\ndef __getattr__(name):"
+PATCHED = (
+ "\nelse:\n initialize_s3()\n"
+ " # Flink Agents CI: finalize AWS before C++ static destruction.\n"
+ " import atexit\n"
+ " atexit.register(finalize_s3)\n\n\ndef __getattr__(name):"
+)
+
+
+def patch_pyarrow() -> None:
+ """Patch only the known wheel layout, without importing native Arrow
code."""
+ package = distribution("pyarrow")
+ if package.version != "11.0.0":
+ message = f"Expected PyArrow 11.0.0, found {package.version}"
+ raise RuntimeError(message)
+ path = package.locate_file("pyarrow/fs.py")
+ source = path.read_text(encoding="utf-8")
+ if source.count(PATCHED) == 1 and ORIGINAL not in source:
+ print(f"PyArrow {package.version} CI shutdown patch verified: {path}")
+ return
+ if source.count(ORIGINAL) != 1 or PATCHED in source:
+ message = f"Unexpected PyArrow fs.py; refusing to patch: {path}"
+ raise RuntimeError(message)
+ path.write_text(source.replace(ORIGINAL, PATCHED), encoding="utf-8")
+ print(f"Applied PyArrow {package.version} CI shutdown patch: {path}")
+
+
+if __name__ == "__main__":
+ patch_pyarrow()
