This is an automated email from the ASF dual-hosted git repository.
jason810496 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 50f90d17dda Build Java SDK e2e example bundles natively in CI (#69794)
50f90d17dda is described below
commit 50f90d17ddab0f39d632917da722c9d4bdaa9307
Author: Jason(Zhe-You) Liu <[email protected]>
AuthorDate: Wed Jul 15 16:51:54 2026 +0900
Build Java SDK e2e example bundles natively in CI (#69794)
The java_sdk e2e mode and the lang-SDK k8s test solved the same
cold-build problem two different ways (containerized Gradle with a
repo-local cache vs host toolchain with salted home-dir caches), which
a review follow-up on #69239 asked to reconcile. Converge the e2e mode
on the #69411 native-toolchain convention: CI provisions the JDK with
actions/setup-java and caches ~/.gradle, dropping the per-run
eclipse-temurin image pull and the --network=host Gradle lock-handover
workaround; local runs keep the containerized build so a dev host
still needs no JDK.
---
.github/workflows/additional-prod-image-tests.yml | 5 ++
.github/workflows/airflow-e2e-tests.yml | 54 +++++++++++++------
.github/workflows/ci-amd.yml | 1 +
.github/workflows/ci-arm.yml | 1 +
.../tests/airflow_e2e_tests/conftest.py | 63 ++++++++++++++--------
.../tests/airflow_e2e_tests/constants.py | 4 ++
6 files changed, 89 insertions(+), 39 deletions(-)
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index e81bfb72566..99a3a4c8728 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -56,6 +56,10 @@ on: # yamllint disable-line rule:truthy
description: "Whether to run Java SDK e2e tests (true/false)"
required: true
type: string
+ java-sdk-version:
+ description: "JDK version used to build the Java SDK bundles natively
in the e2e tests"
+ required: true
+ type: string
run-go-sdk-e2e-tests:
description: "Whether to run Go SDK e2e tests (true/false)"
required: true
@@ -320,6 +324,7 @@ jobs:
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
e2e_test_mode: "java_sdk"
+ java-sdk-version: "${{ inputs.java-sdk-version }}"
if: inputs.canary-run == 'true' || inputs.run-java-sdk-e2e-tests == 'true'
test-e2e-integration-tests-go-sdk:
name: "Go SDK e2e tests with PROD image"
diff --git a/.github/workflows/airflow-e2e-tests.yml
b/.github/workflows/airflow-e2e-tests.yml
index 4fb1fcf8d84..a13a5cccf2d 100644
--- a/.github/workflows/airflow-e2e-tests.yml
+++ b/.github/workflows/airflow-e2e-tests.yml
@@ -52,6 +52,10 @@ on: # yamllint disable-line rule:truthy
description: "Test mode - basic, remote_log, remote_log_elasticsearch,
remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, or go_sdk"
# yamllint disable-line rule:line-length
type: string
default: "basic"
+ java-sdk-version:
+ description: "JDK version used to build the Java SDK bundles natively
(tracks the JAVA_SDK_VERSION breeze constant)" # yamllint disable-line
rule:line-length
+ type: string
+ default: "17"
workflow_call:
inputs:
@@ -83,6 +87,10 @@ on: # yamllint disable-line rule:truthy
description: "Test mode - basic, remote_log, remote_log_elasticsearch,
remote_log_opensearch, xcom_object_storage, event_driven, java_sdk, or go_sdk"
# yamllint disable-line rule:line-length
type: string
default: "basic"
+ java-sdk-version:
+ description: "JDK version used to build the Java SDK bundles natively
(tracks the JAVA_SDK_VERSION breeze constant)" # yamllint disable-line
rule:line-length
+ type: string
+ default: "17"
jobs:
test-e2e-integration-tests:
@@ -113,30 +121,42 @@ jobs:
use-uv: ${{ inputs.use-uv }}
make-mnt-writeable-and-cleanup: true
id: breeze
+ # Only the java_sdk mode runs the Gradle builds; every other e2e mode
skips the
+ # steps below. LANG_SDK_NATIVE_TOOLCHAIN=true (the same switch the
lang-SDK k8s
+ # job uses) makes the e2e conftest build with the host toolchain
provisioned
+ # here instead of ephemeral eclipse-temurin containers, skipping the
+ # toolchain-image pull; local runs keep the containerized build. Without
the
+ # cache each run re-downloads the example bundles' full dependency tree
(Spark
+ # alone is ~1GB) into a fresh ~/.gradle. The key carries a ``-vN-`` salt
(bump
+ # to force-invalidate a poisoned cache) and runner.arch keys the cache
per
+ # architecture: ~/.gradle/jdks holds arch-specific auto-provisioned JDKs
(the
+ # example pins a Java 11 toolchain) and this job runs on amd64 and arm64.
+ - name: "Setup Java for the Java SDK build"
+ if: inputs.e2e_test_mode == 'java_sdk'
+ uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 #
v5.4.0
+ with:
+ distribution: 'temurin'
+ java-version: ${{ inputs.java-sdk-version }}
- name: "Restore Java SDK Gradle dependency cache"
- # Only the java_sdk mode runs the Gradle builds; every other e2e mode
skips
- # this. Without it each java_sdk run re-downloads the example bundles'
full
- # dependency tree (Spark alone is ~1GB) into a fresh GRADLE_USER_HOME.
- # runner.arch keys the cache per architecture: .gradle/jdks holds
- # arch-specific auto-provisioned JDKs and this job runs on amd64 and
arm64.
if: inputs.e2e_test_mode == 'java_sdk'
id: gradle-cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
# v6.1.0
with:
path: |
- java-sdk/.gradle/caches
- java-sdk/.gradle/wrapper
- java-sdk/.gradle/jdks
- !java-sdk/.gradle/caches/*.lock
- !java-sdk/.gradle/caches/journal-1
- key: java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-${{
hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties',
'java-sdk/gradle/wrapper/gradle-wrapper.properties',
'java-sdk/gradle/libs.versions.toml') }} # yamllint disable-line
rule:line-length
+ ~/.gradle/caches
+ ~/.gradle/wrapper
+ ~/.gradle/jdks
+ !~/.gradle/caches/*.lock
+ !~/.gradle/caches/journal-1
+ key: e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-${{
hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties',
'java-sdk/gradle/wrapper/gradle-wrapper.properties',
'java-sdk/gradle/libs.versions.toml') }} # yamllint disable-line
rule:line-length
restore-keys: |
- java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-
+ e2e-java-sdk-gradle-v1-${{ runner.os }}-${{ runner.arch }}-
- name: "Test e2e integration tests"
run: breeze testing airflow-e2e-tests
env:
DOCKER_IMAGE: "${{ inputs.docker-image-tag }}"
E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}"
+ LANG_SDK_NATIVE_TOOLCHAIN: "${{ inputs.e2e_test_mode == 'java_sdk'
&& 'true' || 'false' }}"
- name: "Save Java SDK Gradle dependency cache"
# Saved even when the e2e tests fail: the Gradle warm-up is
independent of
# test outcome, and actions/cache's post step would drop it on every
red run.
@@ -144,11 +164,11 @@ jobs:
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 #
v6.1.0
with:
path: |
- java-sdk/.gradle/caches
- java-sdk/.gradle/wrapper
- java-sdk/.gradle/jdks
- !java-sdk/.gradle/caches/*.lock
- !java-sdk/.gradle/caches/journal-1
+ ~/.gradle/caches
+ ~/.gradle/wrapper
+ ~/.gradle/jdks
+ !~/.gradle/caches/*.lock
+ !~/.gradle/caches/journal-1
key: ${{ steps.gradle-cache.outputs.cache-primary-key }}
- name: Zip logs
run: |
diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index ad0b70da871..fd2892027ff 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -920,6 +920,7 @@ jobs:
run-remote-logging-s3-e2e-tests: ${{
needs.build-info.outputs.run-remote-logging-s3-e2e-tests }}
run-event-driven-e2e-tests: ${{
needs.build-info.outputs.run-event-driven-e2e-tests }}
run-java-sdk-e2e-tests: ${{
needs.build-info.outputs.run-java-sdk-e2e-tests }}
+ java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }}
run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests
}}
run-openlineage-e2e-tests: ${{
needs.build-info.outputs.run-openlineage-e2e-tests }}
run-openlineage-e2e-compat-tests: ${{
needs.build-info.outputs.run-openlineage-e2e-compat-tests }}
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index ee7ffdd916d..ed2da6cf434 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -909,6 +909,7 @@ jobs:
run-remote-logging-s3-e2e-tests: ${{
needs.build-info.outputs.run-remote-logging-s3-e2e-tests }}
run-event-driven-e2e-tests: ${{
needs.build-info.outputs.run-event-driven-e2e-tests }}
run-java-sdk-e2e-tests: ${{
needs.build-info.outputs.run-java-sdk-e2e-tests }}
+ java-sdk-version: ${{ needs.build-info.outputs.java-sdk-version }}
run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests
}}
run-openlineage-e2e-tests: ${{
needs.build-info.outputs.run-openlineage-e2e-tests }}
run-openlineage-e2e-compat-tests: ${{
needs.build-info.outputs.run-openlineage-e2e-compat-tests }}
diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
index 5b5883980c5..56b51992bb4 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
@@ -49,7 +49,9 @@ from airflow_e2e_tests.constants import (
JAVA_SDK_EXAMPLE_DAGS_PATH,
JAVA_SDK_EXAMPLE_LIBS_PATH,
JAVA_SDK_MAVEN_CACHE_PATH,
+ JAVA_SDK_ROOT_PATH,
KAFKA_DIR_PATH,
+ LANG_SDK_NATIVE_TOOLCHAIN,
LOCALSTACK_PATH,
LOGS_FOLDER,
OPENLINEAGE_COMPOSE_PATH,
@@ -281,8 +283,16 @@ _SPARK_JAVA_MODULE_OPTIONS = [
]
-def _run_java_sdk_gradle_container(workdir, *gradle_argv,
capture_output=False):
- """Run the Java SDK Gradle wrapper inside the pinned JDK container.
+def _run_java_sdk_gradle(workdir, *gradle_argv, capture_output=False,
native=False):
+ """Run the Java SDK Gradle wrapper natively or inside the pinned JDK
container.
+
+ In ``native`` mode (used in CI, where the host already has a cached JDK +
+ Gradle cache via ``actions/setup-java``) it invokes the host ``./gradlew``
+ directly, skipping the toolchain-image pull and the container workarounds
+ below while reusing the runner's ~/.gradle cache.
+
+ The containerized path stays the default for local runs so a dev host needs
+ no JDK installed:
* --user keeps build outputs owned by the current user (not root).
* --network=host shares one loopback across concurrent builds so Gradle's
@@ -300,8 +310,18 @@ def _run_java_sdk_gradle_container(workdir, *gradle_argv,
capture_output=False):
* files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes
there without nesting, and its contents are visible on the host.
"""
- return subprocess.run(
- [
+ if native:
+ cwd = workdir
+ argv = [
+ str(JAVA_SDK_ROOT_PATH / "gradlew"),
+ "--no-daemon",
+ "--console=plain",
+ *gradle_argv,
+ ]
+ else:
+ JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True)
+ cwd = None
+ argv = [
"docker",
"run",
"--rm",
@@ -317,22 +337,19 @@ def _run_java_sdk_gradle_container(workdir, *gradle_argv,
capture_output=False):
"-v",
f"{AIRFLOW_ROOT_PATH}:/repo",
"-w",
- workdir,
+ f"/repo/{workdir.relative_to(AIRFLOW_ROOT_PATH)}",
"eclipse-temurin:17-jdk",
"/repo/java-sdk/gradlew",
"--no-daemon",
*gradle_argv,
- ],
- check=True,
- capture_output=capture_output,
- text=True,
- )
+ ]
+ return subprocess.run(argv, cwd=cwd, check=True,
capture_output=capture_output, text=True)
-def _build_example_bundle(workdir):
+def _build_example_bundle(workdir, *, native=False):
"""Build one example bundle, capturing output so concurrent builds don't
interleave."""
try:
- completed = _run_java_sdk_gradle_container(workdir, "bundle",
capture_output=True)
+ completed = _run_java_sdk_gradle(workdir, "bundle",
capture_output=True, native=native)
except subprocess.CalledProcessError as e:
console.print(f"[red]Bundle build failed in {workdir}:")
console.print(e.stdout, e.stderr, sep="\n", markup=False,
soft_wrap=True)
@@ -348,14 +365,15 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir):
then builds a Java-capable Airflow worker image, copies the JARs into the
temp directory, and writes the coordinator configuration.
"""
+ native = LANG_SDK_NATIVE_TOOLCHAIN
console.print("[yellow]Publishing Java SDK artifacts to local Maven
repository...")
- JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True)
- _run_java_sdk_gradle_container("/repo/java-sdk", "publishToMavenLocal",
"-PskipSigning=true")
+ _run_java_sdk_gradle(JAVA_SDK_ROOT_PATH, "publishToMavenLocal",
"-PskipSigning=true", native=native)
# The example and scala_spark_example are independent Gradle builds that
both
# consume the SDK artifact published above, so build them concurrently.
Sharing
- # a writable Gradle user home between concurrent builds is safe only
because
- # --network=host lets each build ping the other's lock-owner port (see the
+ # a writable Gradle user home between concurrent builds is safe because
each
+ # build can ping the other's lock-owner port over one shared loopback - the
+ # host's own in native mode, --network=host in the container path (see the
# helper's docstring); publishToMavenLocal has already unpacked the shared
# wrapper distribution, so neither build races to fetch it.
#
@@ -365,15 +383,16 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir):
# start each bundle from an empty directory.
rmtree(JAVA_SDK_EXAMPLE_LIBS_PATH, ignore_errors=True)
rmtree(SCALA_SPARK_EXAMPLE_LIBS_PATH, ignore_errors=True)
- console.print(
- "[yellow]Building Java SDK and Scala Spark example bundles
concurrently (eclipse-temurin:17-jdk)..."
- )
+ toolchain = "host toolchain" if native else "eclipse-temurin:17-jdk"
+ console.print(f"[yellow]Building Java SDK and Scala Spark example bundles
concurrently ({toolchain})...")
example_bundle_workdirs = [
- "/repo/java-sdk/example",
- "/repo/java-sdk/scala_spark_example",
+ JAVA_SDK_ROOT_PATH / "example",
+ JAVA_SDK_ROOT_PATH / "scala_spark_example",
]
with ThreadPoolExecutor(max_workers=len(example_bundle_workdirs)) as pool:
- bundle_builds = [pool.submit(_build_example_bundle, workdir) for
workdir in example_bundle_workdirs]
+ bundle_builds = [
+ pool.submit(_build_example_bundle, workdir, native=native) for
workdir in example_bundle_workdirs
+ ]
for build in bundle_builds:
build.result()
diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
index b0392d96a8a..ec1732ac75a 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
@@ -54,6 +54,10 @@ KAFKA_DIR_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" /
"docker" / "kafka"
# openlineage_tests/prepare_dags.py; the overlay carries the
OpenLineage-specific env + dag_doc mount.
OPENLINEAGE_COMPOSE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker"
/ "openlineage.yml"
+# CI sets this (the same switch the lang-SDK k8s job uses) to build the
lang-SDK
+# artifacts with the host toolchain instead of ephemeral toolchain containers.
+LANG_SDK_NATIVE_TOOLCHAIN = os.environ.get("LANG_SDK_NATIVE_TOOLCHAIN",
"").lower() == "true"
+
# Java SDK E2E test paths
JAVA_SDK_ROOT_PATH = AIRFLOW_ROOT_PATH / "java-sdk"
JAVA_SDK_EXAMPLE_DAGS_PATH = JAVA_SDK_ROOT_PATH / "example" / "src" /
"resources" / "dags"
