This is an automated email from the ASF dual-hosted git repository. wenjin272 pushed a commit to branch release-0.3 in repository https://gitbox.apache.org/repos/asf/flink-agents.git
commit 77aedb9c31dc978aa879109963f6398b81111d8b Author: Femi <[email protected]> AuthorDate: Tue Jul 14 04:46:27 2026 +0100 [tools] Update install.sh for Flink Agents 0.3.0 and Python 3.12 (#888) (cherry picked from commit 96b943edf016c96f2394ad58eb10efeb7251372a) --- tools/install.sh | 133 ++++++++++++++-- tools/test/helpers/load.bash | 7 +- tools/test/integration/dry_run_extra.bats | 2 +- tools/test/unit/revalidate_python_constraint.bats | 175 ++++++++++++++++++++++ tools/test/unit/validate_python_bin.bats | 39 ++++- 5 files changed, 341 insertions(+), 15 deletions(-) diff --git a/tools/install.sh b/tools/install.sh index 4fcd3db0..70fc516f 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -445,7 +445,7 @@ mark_explicit INSTALL_DIR mark_explicit VENV_DIR FLINK_VERSION="${FLINK_VERSION:-2.2.1}" -FLINK_AGENTS_VERSION="${FLINK_AGENTS_VERSION:-0.2.1}" +FLINK_AGENTS_VERSION="${FLINK_AGENTS_VERSION:-0.3.0}" FLINK_SCALA_VERSION="${FLINK_SCALA_VERSION:-2.12}" FLINK_BASE_URL="${FLINK_BASE_URL:-https://dlcdn.apache.org/flink}"; # Flink Agents JARs live next to Flink on the ASF mirror network. Override @@ -460,10 +460,10 @@ FLINK_RECOMMENDED_VERSION="2.2.1" # Mirrors https://flink.apache.org/downloads/#apache-flink-agents # (latest first). Note: 0.1.x only ships JARs for Flink 1.20, while 0.2.x -# ships JARs for Flink 1.20 / 2.0 / 2.1 / 2.2 — see the download page for -# the exact compatibility matrix. -FLINK_AGENTS_SUPPORTED_VERSIONS=("0.2.1" "0.2.0" "0.1.1" "0.1.0") -FLINK_AGENTS_RECOMMENDED_VERSION="0.2.1" +# and 0.3.x ship JARs for Flink 1.20 / 2.0 / 2.1 / 2.2 — see the download +# page for the exact compatibility matrix. +FLINK_AGENTS_SUPPORTED_VERSIONS=("0.3.0" "0.2.1" "0.2.0" "0.1.1" "0.1.0") +FLINK_AGENTS_RECOMMENDED_VERSION="0.3.0" INSTALL_FLINK="${INSTALL_FLINK:-Ask}" ENABLE_PYFLINK="${ENABLE_PYFLINK:-Ask}" @@ -475,6 +475,9 @@ VERBOSE="${FLINK_AGENTS_VERBOSE:-0}" DRY_RUN="${FLINK_AGENTS_DRY_RUN:-0}" HELP=0 PYFLINK_ACTUALLY_ENABLED=0 +# Set to 1 when a version edit leaves an existing venv on an incompatible +# interpreter and the user opts to recreate it; setup_python_env honors it. +RECREATE_VENV=0 print_usage() { cat <<EOF @@ -1015,6 +1018,7 @@ edit_plan_dump_state() { printf 'VENV_DIR=%s\n' "$(edit_plan_quote "$VENV_DIR")" printf 'PYTHON_BIN=%s\n' "$(edit_plan_quote "${PYTHON_BIN:-}")" printf 'FLINK_AGENTS_VERSION=%s\n' "$(edit_plan_quote "$FLINK_AGENTS_VERSION")" + printf 'RECREATE_VENV=%s\n' "$(edit_plan_quote "${RECREATE_VENV:-0}")" } > "$out" } @@ -1111,6 +1115,7 @@ edit_plan_interactive() { case "$action" in flink_agents_version) prompt_flink_agents_version_interactive || true + revalidate_python_constraint ;; install_flink) if choose_install_method_interactive "Install Flink?"; then @@ -1133,11 +1138,13 @@ edit_plan_interactive() { detect_flink_version_from_home || ui_warn "Could not auto-detect Flink version; keeping ${FLINK_VERSION}" fi FLINK_MAJOR_MINOR="$(flink_major_minor "$FLINK_VERSION")" + revalidate_python_constraint ;; flink_version) prompt_flink_version_interactive || true FLINK_HOME="${INSTALL_DIR}/flink-${FLINK_VERSION}" FLINK_MAJOR_MINOR="$(flink_major_minor "$FLINK_VERSION")" + revalidate_python_constraint ;; install_dir) INSTALL_DIR="$(prompt_path_choice_interactive \ @@ -1404,10 +1411,25 @@ pip_install_quiet() { } setup_python_env() { + # A venv the user opted to recreate (incompatible interpreter, see + # revalidate_existing_venv) is deleted here, not at plan time, so a later + # cancel never destroys it. Guard the rm to a real venv marker. + if [[ "${RECREATE_VENV:-0}" == "1" && -f "$VENV_DIR/pyvenv.cfg" ]]; then + ui_info "Recreating virtual environment: $VENV_DIR" + rm -rf "$VENV_DIR" + fi + if [[ ! -d "$VENV_DIR" ]]; then ui_info "Creating virtual environment: $VENV_DIR" create_venv else + # Reused venvs keep their own interpreter; make sure it still fits the + # selected versions before pip runs under it (belt-and-suspenders for + # any path that bypassed the plan-time revalidation, e.g. an explicit + # VENV_DIR in non-interactive mode). + if [[ -x "$VENV_DIR/bin/python" ]] && ! validate_python_bin "$VENV_DIR/bin/python"; then + die "Existing venv ${VENV_DIR} uses an interpreter incompatible with Flink Agents ${FLINK_AGENTS_VERSION} + Flink ${FLINK_VERSION} (needs >=3.10 and <3.$(python_minor_ceiling)). Remove it or set VENV_DIR to a different path." + fi ui_info "Reusing existing virtual environment: $VENV_DIR" fi @@ -1563,6 +1585,38 @@ check_java() { return 0 } +# Upper bound (exclusive) on the Python minor version, derived from BOTH +# selected versions and returning the stricter (lower) of the two: +# - Flink Agents: 0.1.x / 0.2.x publish requires-python >=3.10,<3.12, +# while 0.3.0+ publishes >=3.10,<3.13 (adds Python 3.12 support). +# - Apache Flink: Python 3.12 requires Flink 2.1+; Flink <2.1 caps at <3.12. +# An unparseable FLINK_VERSION leaves the Flink axis unrestricted (fail open), +# so only the Flink Agents ceiling applies in that case. +python_minor_ceiling() { + local agents_ceiling flink_ceiling + case "$FLINK_AGENTS_VERSION" in + 0.1.*|0.2.*) agents_ceiling=12 ;; + *) agents_ceiling=13 ;; + esac + + flink_ceiling=13 + local mm major minor + mm="$(flink_major_minor "${FLINK_VERSION:-}")" + if [[ -n "$mm" ]]; then + major="${mm%%.*}" + minor="${mm##*.}" + if (( major < 2 || (major == 2 && minor < 1) )); then + flink_ceiling=12 + fi + fi + + if (( agents_ceiling < flink_ceiling )); then + printf '%s' "$agents_ceiling" + else + printf '%s' "$flink_ceiling" + fi +} + validate_python_bin() { local bin="$1" [[ -n "$bin" ]] || return 1 @@ -1576,7 +1630,7 @@ validate_python_bin() { py_major="${py_version_output%%.*}" py_minor="${py_version_output##*.}" - if [[ "$py_major" -ne 3 ]] || [[ "$py_minor" -lt 10 ]] || [[ "$py_minor" -ge 12 ]]; then + if [[ "$py_major" -ne 3 ]] || [[ "$py_minor" -lt 10 ]] || [[ "$py_minor" -ge "$(python_minor_ceiling)" ]]; then return 1 fi return 0 @@ -1588,7 +1642,7 @@ resolve_python() { ui_success "Using Python: $PYTHON_BIN" return 0 fi - die "PYTHON_BIN is invalid or unsupported (need >=3.10 and <3.12): $PYTHON_BIN" + die "PYTHON_BIN is invalid or unsupported (need >=3.10 and <3.$(python_minor_ceiling)): $PYTHON_BIN" fi if validate_python_bin python3; then @@ -1600,7 +1654,7 @@ resolve_python() { fi if command -v python3 >/dev/null 2>&1; then - ui_warn "python3 on PATH is incompatible (Flink Agents requires Python >=3.10 and <3.12)" + ui_warn "python3 on PATH is incompatible (Flink Agents ${FLINK_AGENTS_VERSION} requires Python >=3.10 and <3.$(python_minor_ceiling))" else ui_warn "python3 not found on PATH" fi @@ -1615,13 +1669,74 @@ resolve_python() { die "No Python interpreter provided." fi if ! validate_python_bin "$input"; then - die "Provided Python is invalid or unsupported (need >=3.10 and <3.12): $input" + die "Provided Python is invalid or unsupported (need >=3.10 and <3.$(python_minor_ceiling)): $input" fi PYTHON_BIN="$input" ui_success "Using Python: $PYTHON_BIN" return 0 } +# The Python ceiling depends on BOTH the Flink Agents and the Flink version +# (see python_minor_ceiling), so editing either at the confirm screen can +# invalidate a Python setup that was already accepted — e.g. Python 3.12 +# passes for 0.3.0 + Flink 2.2 but not after editing down to 0.2.1 or to +# Flink 2.0. Called after a version edit; mirrors the enable_pyflink arm. +# No-op unless PyFlink is enabled. Two independent checks: +# 1. PYTHON_BIN — the resolved interpreter; re-resolve if it no longer fits. +# 2. VENV_DIR — see revalidate_existing_venv; a reused venv keeps its own +# interpreter, which re-resolving PYTHON_BIN does not change. +revalidate_python_constraint() { + if [[ "$ENABLE_PYFLINK" != "Yes" ]]; then + return 0 + fi + + if [[ -n "${PYTHON_BIN:-}" ]] && ! validate_python_bin "$PYTHON_BIN"; then + ui_warn "The selected Python (${PYTHON_BIN}) is incompatible with Flink Agents ${FLINK_AGENTS_VERSION} + Flink ${FLINK_VERSION} (requires Python >=3.10 and <3.$(python_minor_ceiling))" + PYTHON_BIN="" + resolve_python + fi + + revalidate_existing_venv +} + +# setup_python_env reuses an existing venv as-is (it only creates one when +# VENV_DIR is absent), so the venv's own interpreter — not the re-resolved +# PYTHON_BIN — is what pip runs under. After a version edit that lowers the +# Python ceiling, an existing venv built on a now-too-new interpreter would +# still be reused and fail at `pip install` time. Catch it at plan time: +# offer to recreate it (deferred to setup_python_env via RECREATE_VENV, which +# only ever deletes a directory carrying a pyvenv.cfg marker) or point at a +# different path. Non-interactive callers get an actionable error instead of a +# mid-install failure. No-op unless VENV_DIR is an existing venv. +revalidate_existing_venv() { + RECREATE_VENV=0 + [[ "$ENABLE_PYFLINK" == "Yes" && -n "${VENV_DIR:-}" ]] || return 0 + + local kind + kind="$(validate_venv_dir "$VENV_DIR")" || true + [[ "$kind" == "venv" ]] || return 0 + + if validate_python_bin "$VENV_DIR/bin/python"; then + return 0 + fi + + local venv_pv + venv_pv="$("$VENV_DIR/bin/python" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || printf 'unknown')" + ui_warn "Existing venv ${VENV_DIR} uses Python ${venv_pv}, incompatible with Flink Agents ${FLINK_AGENTS_VERSION} + Flink ${FLINK_VERSION} (needs >=3.10 and <3.$(python_minor_ceiling))" + + if ! is_promptable; then + die "Recreate ${VENV_DIR} or set VENV_DIR to a different path (its interpreter ${venv_pv} is unsupported for this version selection)." + fi + + if choose_install_method_interactive "Recreate ${VENV_DIR} with a compatible interpreter?"; then + RECREATE_VENV=1 + else + prompt_and_validate_venv_dir "$VENV_DIR" + # The newly chosen path may itself be an incompatible venv. + revalidate_existing_venv + fi +} + show_install_plan() { ui_section "Environment (read-only)" ui_kv "OS" "$OS" diff --git a/tools/test/helpers/load.bash b/tools/test/helpers/load.bash index 043a4fd6..e2cae738 100644 --- a/tools/test/helpers/load.bash +++ b/tools/test/helpers/load.bash @@ -28,8 +28,9 @@ reset_install_sh_state() { ENABLE_PYFLINK="Ask" PYTHON_BIN="" PYFLINK_ACTUALLY_ENABLED=0 + RECREATE_VENV=0 FLINK_VERSION="2.2.0" - FLINK_AGENTS_VERSION="0.2.1" + FLINK_AGENTS_VERSION="0.3.0" FLINK_SCALA_VERSION="2.12" FLINK_BASE_URL="https://dlcdn.apache.org/flink"; FLINK_SUPPORTED_VERSIONS=("2.2.0" "2.1.1" "2.0.1" "1.20.3") @@ -44,6 +45,6 @@ reset_install_sh_state() { FLINK_AGENTS_VERSION_EXPLICIT=0 INSTALL_DIR_EXPLICIT=0 VENV_DIR_EXPLICIT=0 - FLINK_AGENTS_SUPPORTED_VERSIONS=("0.2.1" "0.2.0" "0.1.1" "0.1.0") - FLINK_AGENTS_RECOMMENDED_VERSION="0.2.1" + FLINK_AGENTS_SUPPORTED_VERSIONS=("0.3.0" "0.2.1" "0.2.0" "0.1.1" "0.1.0") + FLINK_AGENTS_RECOMMENDED_VERSION="0.3.0" } diff --git a/tools/test/integration/dry_run_extra.bats b/tools/test/integration/dry_run_extra.bats index 1e7efac0..105ccf4c 100644 --- a/tools/test/integration/dry_run_extra.bats +++ b/tools/test/integration/dry_run_extra.bats @@ -19,7 +19,7 @@ esac @test "dry-run: plan shows Flink Agents version (review #1 — JARs are implicit)" { run bash "${BATS_TEST_DIRNAME}/../../install.sh" --dry-run --non-interactive [ "$status" -eq 0 ] - case "$output" in *"Flink Agents version"*"0.2.1"*) ;; *) false ;; esac + case "$output" in *"Flink Agents version"*"0.3.0"*) ;; *) false ;; esac } @test "dry-run: existing FLINK_HOME — plan shows detected version, not default (review #2)" { diff --git a/tools/test/unit/revalidate_python_constraint.bats b/tools/test/unit/revalidate_python_constraint.bats new file mode 100644 index 00000000..354df87a --- /dev/null +++ b/tools/test/unit/revalidate_python_constraint.bats @@ -0,0 +1,175 @@ +#!/usr/bin/env bats + +setup() { + load '../helpers/load' + load '../helpers/shim' + load_install_sh + reset_install_sh_state + shim_setup + # Default the venv to an absent path so revalidate_existing_venv is a + # no-op unless a test builds a real venv on purpose. + VENV_DIR="$BATS_TEST_TMPDIR/no-venv" +} + +# Helper: write a fake python that reports a chosen version. +fake_python() { + local name="$1" + local ver="$2" + shim_bin_script "$name" " +case \"\$*\" in + *'sys.version_info.major'*) + echo '$ver' + ;; +esac +" +} + +# Helper: build a fake existing venv whose interpreter reports $ver. +fake_venv() { + local dir="$1" + local ver="$2" + mkdir -p "$dir/bin" + : > "$dir/pyvenv.cfg" + : > "$dir/bin/activate" + cat > "$dir/bin/python" <<EOF +#!/usr/bin/env bash +case "\$*" in + *'sys.version_info.major'*) echo '$ver' ;; +esac +EOF + chmod +x "$dir/bin/python" +} + +# --- PYTHON_BIN re-resolution (Flink Agents axis) --- + +# Scenario from review: interpreter resolved under 0.3.0 (ceiling 3.13), +# then the version is edited down to 0.2.1 (ceiling 3.12) at the confirm +# screen. The stale interpreter must be dropped and re-resolved. +@test "revalidate_python_constraint: re-resolves when the edited Agents version drops the ceiling below PYTHON_BIN" { + fake_python fake_py312 "3.12" + fake_python python3 "3.11" + + ENABLE_PYFLINK="Yes" + PYTHON_BIN="fake_py312" + FLINK_AGENTS_VERSION="0.2.1" + + revalidate_python_constraint + [ "$PYTHON_BIN" = "python3" ] +} + +@test "revalidate_python_constraint: keeps an interpreter that still satisfies the new version" { + fake_python fake_py312 "3.12" + + ENABLE_PYFLINK="Yes" + PYTHON_BIN="fake_py312" + FLINK_AGENTS_VERSION="0.3.0" + + revalidate_python_constraint + [ "$PYTHON_BIN" = "fake_py312" ] +} + +@test "revalidate_python_constraint: no-op when PyFlink is disabled" { + fake_python fake_py312 "3.12" + + ENABLE_PYFLINK="No" + PYTHON_BIN="fake_py312" + FLINK_AGENTS_VERSION="0.2.1" + + revalidate_python_constraint + [ "$PYTHON_BIN" = "fake_py312" ] +} + +@test "revalidate_python_constraint: no-op when no interpreter was resolved yet" { + ENABLE_PYFLINK="Yes" + PYTHON_BIN="" + FLINK_AGENTS_VERSION="0.2.1" + + run revalidate_python_constraint + [ "$status" -eq 0 ] +} + +@test "revalidate_python_constraint: warns before re-resolving a stale interpreter" { + fake_python fake_py312 "3.12" + fake_python python3 "3.11" + + ENABLE_PYFLINK="Yes" + PYTHON_BIN="fake_py312" + FLINK_AGENTS_VERSION="0.2.1" + + run revalidate_python_constraint + [ "$status" -eq 0 ] + [[ "$output" == *"incompatible with Flink Agents 0.2.1"* ]] + [[ "$output" == *"<3.12"* ]] +} + +# --- PYTHON_BIN re-resolution (Flink axis) --- + +# Python 3.12 needs Flink 2.1+. Editing the Flink version down to 2.0.x while +# Agents stays 0.3.0 must still drop a 3.12 interpreter. +@test "revalidate_python_constraint: re-resolves when the edited Flink version drops the ceiling below PYTHON_BIN" { + fake_python fake_py312 "3.12" + fake_python python3 "3.11" + + ENABLE_PYFLINK="Yes" + PYTHON_BIN="fake_py312" + FLINK_AGENTS_VERSION="0.3.0" + FLINK_VERSION="2.0.2" + + revalidate_python_constraint + [ "$PYTHON_BIN" = "python3" ] +} + +@test "revalidate_python_constraint: keeps a 3.12 interpreter when Flink is 2.1+" { + fake_python fake_py312 "3.12" + + ENABLE_PYFLINK="Yes" + PYTHON_BIN="fake_py312" + FLINK_AGENTS_VERSION="0.3.0" + FLINK_VERSION="2.1.3" + + revalidate_python_constraint + [ "$PYTHON_BIN" = "fake_py312" ] +} + +# --- existing-venv revalidation --- + +# The reviewer's second scenario: PYTHON_BIN re-resolves fine (3.11 is valid +# for both versions), but an existing venv built on 3.12 would still be reused +# by setup_python_env. Non-interactive callers must get an actionable error. +@test "revalidate_existing_venv: non-interactive die when the existing venv interpreter is too new" { + fake_python python3 "3.11" + fake_venv "$BATS_TEST_TMPDIR/venv312" "3.12" + + NO_PROMPT=1 + ENABLE_PYFLINK="Yes" + PYTHON_BIN="python3" + FLINK_AGENTS_VERSION="0.2.1" + VENV_DIR="$BATS_TEST_TMPDIR/venv312" + + run revalidate_existing_venv + [ "$status" -ne 0 ] + [[ "$output" == *"uses Python 3.12"* ]] + [[ "$output" == *"different path"* ]] +} + +@test "revalidate_existing_venv: no-op when the existing venv interpreter is compatible" { + fake_venv "$BATS_TEST_TMPDIR/venv311" "3.11" + + ENABLE_PYFLINK="Yes" + FLINK_AGENTS_VERSION="0.2.1" + VENV_DIR="$BATS_TEST_TMPDIR/venv311" + RECREATE_VENV=0 + + run revalidate_existing_venv + [ "$status" -eq 0 ] + [ "$RECREATE_VENV" = "0" ] +} + +@test "revalidate_existing_venv: no-op when VENV_DIR is not an existing venv" { + ENABLE_PYFLINK="Yes" + FLINK_AGENTS_VERSION="0.2.1" + VENV_DIR="$BATS_TEST_TMPDIR/does-not-exist" + + run revalidate_existing_venv + [ "$status" -eq 0 ] +} diff --git a/tools/test/unit/validate_python_bin.bats b/tools/test/unit/validate_python_bin.bats index 87998bd1..2b99ecda 100644 --- a/tools/test/unit/validate_python_bin.bats +++ b/tools/test/unit/validate_python_bin.bats @@ -48,18 +48,53 @@ esac [ "$status" -ne 0 ] } -@test "validate_python_bin: Python 3.12 is rejected" { +@test "validate_python_bin: Python 3.12 is rejected for Flink Agents 0.2.x" { + FLINK_AGENTS_VERSION="0.2.1" fake_python "3.12" run validate_python_bin fake_py [ "$status" -ne 0 ] } -@test "validate_python_bin: Python 3.13 is rejected" { +@test "validate_python_bin: Python 3.12 is accepted for Flink Agents 0.3.0+" { + FLINK_AGENTS_VERSION="0.3.0" + fake_python "3.12" + run validate_python_bin fake_py + [ "$status" -eq 0 ] +} + +@test "validate_python_bin: Python 3.13 is rejected for Flink Agents 0.3.0+" { + FLINK_AGENTS_VERSION="0.3.0" fake_python "3.13" run validate_python_bin fake_py [ "$status" -ne 0 ] } +# Python 3.12 support also requires Flink 2.1+, independent of the Agents +# version: 0.3.0 alone is not enough if Flink is 2.0.x or 1.20.x. +@test "validate_python_bin: Python 3.12 is rejected for Flink Agents 0.3.0 on Flink 2.0.x" { + FLINK_AGENTS_VERSION="0.3.0" + FLINK_VERSION="2.0.2" + fake_python "3.12" + run validate_python_bin fake_py + [ "$status" -ne 0 ] +} + +@test "validate_python_bin: Python 3.12 is rejected for Flink Agents 0.3.0 on Flink 1.20.x" { + FLINK_AGENTS_VERSION="0.3.0" + FLINK_VERSION="1.20.5" + fake_python "3.12" + run validate_python_bin fake_py + [ "$status" -ne 0 ] +} + +@test "validate_python_bin: Python 3.12 is accepted for Flink Agents 0.3.0 on Flink 2.1.x" { + FLINK_AGENTS_VERSION="0.3.0" + FLINK_VERSION="2.1.3" + fake_python "3.12" + run validate_python_bin fake_py + [ "$status" -eq 0 ] +} + @test "validate_python_bin: Python 2.7 is rejected" { fake_python "2.7" run validate_python_bin fake_py
