This is an automated email from the ASF dual-hosted git repository.
jason810496 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7a4c0599ff6 Speed up the java_sdk e2e Gradle builds with concurrency
and caching (#69239)
7a4c0599ff6 is described below
commit 7a4c0599ff6048f74ec203338f63e1aff1086ca0
Author: Jason(Zhe-You) Liu <[email protected]>
AuthorDate: Fri Jul 10 16:20:46 2026 +0900
Speed up the java_sdk e2e Gradle builds with concurrency and caching
(#69239)
* Build the Java SDK and Scala Spark e2e bundles concurrently
The example and scala_spark_example bundles are independent Gradle builds
that both consume the SDK artifact published to the local Maven repo, so the
Scala Spark dependency resolution no longer has to wait for the Java example
build to finish before it starts.
The bundle containers run with --network=host because Gradle's cross-process
lock handover pings the lock owner over loopback, which isolated container
network namespaces would break, and each build's output is captured and
printed under its own header so concurrent logs do not interleave.
* Cache the Java SDK Gradle dependencies in the e2e job
The java_sdk e2e job builds the SDK and its example bundles with Gradle on
every canary and java-sdk-touching run, starting from an empty
GRADLE_USER_HOME each time. That forces a full re-download of the example
bundles' dependency tree (Spark alone is around 1GB) on every run. Reusing
the resolved dependencies across runs removes that repeated download.
The restore/save split saves the warm-up even when the e2e tests fail, the
key hashes the Kotlin-DSL build files and version catalog that pin the SDK
dependencies, and runner.arch keeps the arch-specific auto-provisioned JDKs
in .gradle/jdks from crossing between amd64 and arm64 runners.
* Address java_sdk e2e review feedback on caching and build output
Excludes churny *.lock/journal files from the Gradle cache per GitHub's
own caching guidance, makes bundle build output print stdout/stderr
symmetrically through Rich with markup disabled so raw Gradle text
can't be misparsed as markup, ties the build pool size to the bundle
list instead of a hardcoded count, and cites the Gradle lock-handover
mechanism the --network=host workaround depends on.
---
.github/workflows/airflow-e2e-tests.yml | 32 +++++
.../tests/airflow_e2e_tests/conftest.py | 140 ++++++++++-----------
2 files changed, 100 insertions(+), 72 deletions(-)
diff --git a/.github/workflows/airflow-e2e-tests.yml
b/.github/workflows/airflow-e2e-tests.yml
index 3a87ea525f7..4fb1fcf8d84 100644
--- a/.github/workflows/airflow-e2e-tests.yml
+++ b/.github/workflows/airflow-e2e-tests.yml
@@ -113,11 +113,43 @@ jobs:
use-uv: ${{ inputs.use-uv }}
make-mnt-writeable-and-cleanup: true
id: breeze
+ - name: "Restore Java SDK Gradle dependency cache"
+ # Only the java_sdk mode runs the Gradle builds; every other e2e mode
skips
+ # this. Without it each java_sdk run re-downloads the example bundles'
full
+ # dependency tree (Spark alone is ~1GB) into a fresh GRADLE_USER_HOME.
+ # runner.arch keys the cache per architecture: .gradle/jdks holds
+ # arch-specific auto-provisioned JDKs and this job runs on amd64 and
arm64.
+ if: inputs.e2e_test_mode == 'java_sdk'
+ id: gradle-cache
+ uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9
# v6.1.0
+ with:
+ path: |
+ java-sdk/.gradle/caches
+ java-sdk/.gradle/wrapper
+ java-sdk/.gradle/jdks
+ !java-sdk/.gradle/caches/*.lock
+ !java-sdk/.gradle/caches/journal-1
+ key: java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-${{
hashFiles('java-sdk/**/*.gradle*', 'java-sdk/**/gradle.properties',
'java-sdk/gradle/wrapper/gradle-wrapper.properties',
'java-sdk/gradle/libs.versions.toml') }} # yamllint disable-line
rule:line-length
+ restore-keys: |
+ java-sdk-gradle-${{ runner.os }}-${{ runner.arch }}-
- name: "Test e2e integration tests"
run: breeze testing airflow-e2e-tests
env:
DOCKER_IMAGE: "${{ inputs.docker-image-tag }}"
E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}"
+ - name: "Save Java SDK Gradle dependency cache"
+ # Saved even when the e2e tests fail: the Gradle warm-up is
independent of
+ # test outcome, and actions/cache's post step would drop it on every
red run.
+ if: always() && inputs.e2e_test_mode == 'java_sdk' &&
steps.gradle-cache.outputs.cache-hit != 'true'
+ uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 #
v6.1.0
+ with:
+ path: |
+ java-sdk/.gradle/caches
+ java-sdk/.gradle/wrapper
+ java-sdk/.gradle/jdks
+ !java-sdk/.gradle/caches/*.lock
+ !java-sdk/.gradle/caches/journal-1
+ key: ${{ steps.gradle-cache.outputs.cache-primary-key }}
- name: Zip logs
run: |
cd ./airflow-e2e-tests && zip -r logs.zip logs
diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
index 639d378bc4d..eb6ed32e9a6 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
@@ -19,6 +19,7 @@ from __future__ import annotations
import json
import os
import subprocess
+from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from pathlib import Path
from shutil import copyfile, copytree, rmtree
@@ -278,29 +279,31 @@ _SPARK_JAVA_MODULE_OPTIONS = [
]
-def _setup_java_sdk_integration(dot_env_file, tmp_dir):
- """Set up the java_sdk E2E test mode.
-
- Builds the Java example bundle via the Gradle wrapper, then builds a
- Java-capable Airflow worker image, copies the JARs into the temp directory,
- and writes the coordinator configuration.
+def _run_java_sdk_gradle_container(workdir, *gradle_argv,
capture_output=False):
+ """Run the Java SDK Gradle wrapper inside the pinned JDK container.
+
+ * --user keeps build outputs owned by the current user (not root).
+ * --network=host shares one loopback across concurrent builds so Gradle's
+ cross-process lock handover works: a UDP ping to the lock owner's port
+ (org.gradle.cache.internal.locklistener.FileLockCommunicator.pingOwner,
+ see
https://github.com/gradle/gradle/blob/v8.14.4/platforms/core-execution/persistent-cache/src/main/java/org/gradle/cache/internal/locklistener/FileLockCommunicator.java)
+ would fail across isolated container network namespaces, as reported in
+ https://github.com/gradle/gradle/issues/851.
+ * --no-daemon avoids a background JVM that would outlive the container.
+ * GRADLE_USER_HOME persists the Gradle distribution and dependency cache
+ in java-sdk/.gradle/ so subsequent runs skip straight to compilation.
+ * HOME is set explicitly because --user runs as the host UID which has no
+ entry in the container's /etc/passwd; Docker would otherwise inherit the
+ image's HOME (/root) which the non-root process cannot write to.
+ * files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes
+ there without nesting, and its contents are visible on the host.
"""
- # * --user keeps build outputs owned by the current user (not root).
- # * --no-daemon avoids a background JVM that would outlive the container.
- # * GRADLE_USER_HOME persists the Gradle distribution and dependency cache
- # in java-sdk/.gradle/ so subsequent runs skip straight to compilation.
- # * HOME is set explicitly because --user runs as the host UID which has no
- # entry in the container's /etc/passwd; Docker would otherwise inherit
the
- # image's HOME (/root) which the non-root process cannot write to.
- # * files/m2 is mounted directly as ~/.m2 so publishToMavenLocal writes
- # there without nesting, and its contents are visible on the host.
- console.print("[yellow]Publishing Java SDK artifacts to local Maven
repository...")
- JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True)
- subprocess.run(
+ return subprocess.run(
[
"docker",
"run",
"--rm",
+ "--network=host",
"--user",
f"{os.getuid()}:{os.getgid()}",
"-e",
@@ -312,72 +315,65 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir):
"-v",
f"{AIRFLOW_ROOT_PATH}:/repo",
"-w",
- "/repo/java-sdk",
+ workdir,
"eclipse-temurin:17-jdk",
- "./gradlew",
- "publishToMavenLocal",
- "-PskipSigning=true",
+ "/repo/java-sdk/gradlew",
"--no-daemon",
+ *gradle_argv,
],
check=True,
+ capture_output=capture_output,
+ text=True,
)
- # TODO: Make the following build steps parallel
+
+
+def _build_example_bundle(workdir):
+ """Build one example bundle, capturing output so concurrent builds don't
interleave."""
+ try:
+ completed = _run_java_sdk_gradle_container(workdir, "bundle",
capture_output=True)
+ except subprocess.CalledProcessError as e:
+ console.print(f"[red]Bundle build failed in {workdir}:")
+ console.print(e.stdout, e.stderr, sep="\n", markup=False,
soft_wrap=True)
+ raise
+ console.print(f"[yellow]Bundle build finished in {workdir}:")
+ console.print(completed.stdout, completed.stderr, sep="\n", markup=False,
soft_wrap=True)
+
+
+def _setup_java_sdk_integration(dot_env_file, tmp_dir):
+ """Set up the java_sdk E2E test mode.
+
+ Builds the Java SDK and Scala Spark example bundles via the Gradle wrapper,
+ then builds a Java-capable Airflow worker image, copies the JARs into the
+ temp directory, and writes the coordinator configuration.
+ """
+ console.print("[yellow]Publishing Java SDK artifacts to local Maven
repository...")
+ JAVA_SDK_MAVEN_CACHE_PATH.mkdir(parents=True, exist_ok=True)
+ _run_java_sdk_gradle_container("/repo/java-sdk", "publishToMavenLocal",
"-PskipSigning=true")
+
+ # The example and scala_spark_example are independent Gradle builds that
both
+ # consume the SDK artifact published above, so build them concurrently.
Sharing
+ # a writable Gradle user home between concurrent builds is safe only
because
+ # --network=host lets each build ping the other's lock-owner port (see the
+ # helper's docstring); publishToMavenLocal has already unpacked the shared
+ # wrapper distribution, so neither build races to fetch it.
+ #
# The Gradle `bundle` task is a Copy that never prunes its destination, so
# JARs from an earlier build linger. A stale dependency JAR with its own
# Main-Class would make JavaCoordinator's Main-Class discovery ambiguous,
so
# start each bundle from an empty directory.
rmtree(JAVA_SDK_EXAMPLE_LIBS_PATH, ignore_errors=True)
- console.print("[yellow]Building Java SDK example bundle
(eclipse-temurin:17-jdk)...")
- subprocess.run(
- [
- "docker",
- "run",
- "--rm",
- "--user",
- f"{os.getuid()}:{os.getgid()}",
- "-e",
- "GRADLE_USER_HOME=/repo/java-sdk/.gradle",
- "-e",
- "HOME=/workspace-home",
- "-v",
- f"{JAVA_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2",
- "-v",
- f"{AIRFLOW_ROOT_PATH}:/repo",
- "-w",
- "/repo/java-sdk/example",
- "eclipse-temurin:17-jdk",
- "../gradlew",
- "bundle",
- "--no-daemon",
- ],
- check=True,
- )
rmtree(SCALA_SPARK_EXAMPLE_LIBS_PATH, ignore_errors=True)
- console.print("[yellow]Building Scala Spark example bundle
(eclipse-temurin:17-jdk)...")
- subprocess.run(
- [
- "docker",
- "run",
- "--rm",
- "--user",
- f"{os.getuid()}:{os.getgid()}",
- "-e",
- "GRADLE_USER_HOME=/repo/java-sdk/.gradle",
- "-e",
- "HOME=/workspace-home",
- "-v",
- f"{JAVA_SDK_MAVEN_CACHE_PATH}:/workspace-home/.m2",
- "-v",
- f"{AIRFLOW_ROOT_PATH}:/repo",
- "-w",
- "/repo/java-sdk/scala_spark_example",
- "eclipse-temurin:17-jdk",
- "../gradlew",
- "bundle",
- "--no-daemon",
- ],
- check=True,
+ console.print(
+ "[yellow]Building Java SDK and Scala Spark example bundles
concurrently (eclipse-temurin:17-jdk)..."
)
+ example_bundle_workdirs = [
+ "/repo/java-sdk/example",
+ "/repo/java-sdk/scala_spark_example",
+ ]
+ with ThreadPoolExecutor(max_workers=len(example_bundle_workdirs)) as pool:
+ bundle_builds = [pool.submit(_build_example_bundle, workdir) for
workdir in example_bundle_workdirs]
+ for build in bundle_builds:
+ build.result()
# Copy compose override and Dockerfile into the temp directory.
copyfile(JAVA_COMPOSE_PATH, tmp_dir / "java.yml")
