This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 8f84c798ab7 Move remaining workflow to Java11 (#35559)
8f84c798ab7 is described below
commit 8f84c798ab77906c5288ce774641a61854313a77
Author: Yi Hu <[email protected]>
AuthorDate: Mon Jul 14 12:12:13 2025 -0400
Move remaining workflow to Java11 (#35559)
* Move remaining workflow to Java11
* Honor testJavaVersion property in xlang validation runner task
* Honor testJavaVersion property in toxTask
* Honor testJavaVersion property in Go Validation Runner Task
* Honor testJavaHome for XVR Samza test
* Revert tox.ini change other than Samza runner test
Breaks PVR Flink for unknown reason
---
.github/workflows/beam_PostCommit_Go_VR_Samza.yml | 4 +--
...eam_PostCommit_Python_ValidatesRunner_Samza.yml | 6 ++++-
.github/workflows/beam_PostCommit_XVR_Samza.yml | 5 ++--
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 28 +++++++++++++++++++
runners/samza/job-server/build.gradle | 14 ++++++++--
sdks/go/test/build.gradle | 31 ++++++++++++++++++++++
sdks/go/test/run_validatesrunner_tests.sh | 20 +++++++++-----
sdks/python/scripts/run_job_server.sh | 7 ++++-
sdks/python/tox.ini | 1 +
9 files changed, 101 insertions(+), 15 deletions(-)
diff --git a/.github/workflows/beam_PostCommit_Go_VR_Samza.yml
b/.github/workflows/beam_PostCommit_Go_VR_Samza.yml
index a056d13e3e8..a358e6bea08 100644
--- a/.github/workflows/beam_PostCommit_Go_VR_Samza.yml
+++ b/.github/workflows/beam_PostCommit_Go_VR_Samza.yml
@@ -75,11 +75,11 @@ jobs:
# TODO(https://github.com/apache/beam/issues/32208) move to Java11
after bump to Samza 1.8
with:
java-version: |
- 11
8
+ 11
- name: run Go Samza ValidatesRunner script
env:
CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
uses: ./.github/actions/gradle-command-self-hosted-action
with:
- gradle-command: :sdks:go:test:samzaValidatesRunner
-Pjava11Home=$JAVA_HOME_11_X64
+ gradle-command: :sdks:go:test:samzaValidatesRunner
-Pjava8Home=$JAVA_HOME_8_X64 -PtestJavaVersion=8
diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml
b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml
index 300becc9ddb..bf1a1536053 100644
--- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml
+++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml
@@ -76,7 +76,9 @@ jobs:
uses: ./.github/actions/setup-environment-action
# TODO(https://github.com/apache/beam/issues/32208) move to Java11
after bump to Samza 1.8
with:
- java-version: 8
+ java-version: |
+ 8
+ 11
python-version: ${{ matrix.python_version }}
- name: Set PY_VER_CLEAN
id: set_py_ver_clean
@@ -90,6 +92,8 @@ jobs:
gradle-command:
:sdks:python:test-suites:portable:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:samzaValidatesRunner
arguments: |
-PpythonVersion=${{ matrix.python_version }} \
+ -PtestJavaVersion=8 \
+ -Pjava8Home=$JAVA_HOME_8_X64 \
- name: Archive Python Test Results
uses: actions/upload-artifact@v4
if: failure()
diff --git a/.github/workflows/beam_PostCommit_XVR_Samza.yml
b/.github/workflows/beam_PostCommit_XVR_Samza.yml
index ebd95bfd210..a06b7782ad4 100644
--- a/.github/workflows/beam_PostCommit_XVR_Samza.yml
+++ b/.github/workflows/beam_PostCommit_XVR_Samza.yml
@@ -76,8 +76,8 @@ jobs:
# TODO(https://github.com/apache/beam/issues/32208) move to Java11
after bump to Samza 1.8
with:
java-version: |
- 11
8
+ 11
python-version: |
${{ matrix.python_version }}
- name: run PostCommit XVR Samza script
@@ -88,7 +88,8 @@ jobs:
gradle-command:
:runners:samza:job-server:validatesCrossLanguageRunner
arguments: |
-PpythonVersion=${{ matrix.python_version }} \
- -Pjava11Home=$JAVA_HOME_11_X64 \
+ -PtestJavaVersion=8 \
+ -Pjava8Home=$JAVA_HOME_8_X64 \
-PskipNonPythonTask=${{ (matrix.python_version == '3.9' && true)
|| false }} \
- name: Archive JUnit Test Results
uses: actions/upload-artifact@v4
diff --git
a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 52fcd2a63a8..b7da748b16a 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2785,6 +2785,12 @@ class BeamModulePlugin implements Plugin<Project> {
]
}
+ String testJavaVersion = project.findProperty('testJavaVersion')
+ String testJavaHome = null
+ if (testJavaVersion) {
+ testJavaHome = project.findProperty("java${testJavaVersion}Home")
+ }
+
['Java': javaPort, 'Python': pythonPort].each { sdk, port ->
// Task for running testcases in Java SDK
def javaTask = project.tasks.register(config.name+"JavaUsing"+sdk,
Test) {
@@ -2810,6 +2816,9 @@ class BeamModulePlugin implements Plugin<Project> {
} else {
throw new GradleException("unsupported expansion service for Java
validate runner tests.")
}
+ if (testJavaHome) {
+ executable = "${testJavaHome}/bin/java"
+ }
// increase maxHeapSize as this is directly correlated to direct
memory,
// see https://issues.apache.org/jira/browse/BEAM-6698
maxHeapSize = '4g'
@@ -2845,6 +2854,9 @@ class BeamModulePlugin implements Plugin<Project> {
args '-c', ". $envDir/bin/activate && cd $pythonDir &&
./scripts/run_integration_test.sh $cmdArgs"
dependsOn setupTask
dependsOn config.startJobServer
+ if (testJavaHome) {
+ environment "JAVA_HOME", testJavaHome
+ }
}
if (sdk != "Python") {
mainTask.configure{dependsOn pythonTask}
@@ -2869,6 +2881,9 @@ class BeamModulePlugin implements Plugin<Project> {
dependsOn setupTask
dependsOn config.startJobServer
dependsOn ':sdks:java:extensions:sql:expansion-service:shadowJar'
+ if (testJavaHome) {
+ environment "JAVA_HOME", testJavaHome
+ }
}
mainTask.configure{dependsOn pythonSqlTask}
cleanupTask.configure{mustRunAfter pythonSqlTask}
@@ -3131,6 +3146,13 @@ class BeamModulePlugin implements Plugin<Project> {
project.tasks.register(name) {
dependsOn setupVirtualenv
dependsOn ':sdks:python:sdist'
+
+ def testJavaVersion = project.findProperty('testJavaVersion')
+ String testJavaHome = null
+ if (testJavaVersion) {
+ testJavaHome = project.findProperty("java${testJavaVersion}Home")
+ }
+
if (project.hasProperty('useWheelDistribution')) {
def pythonVersionNumber = project.ext.pythonVersion.replace('.',
'')
dependsOn ":sdks:python:bdistPy${pythonVersionNumber}linux"
@@ -3142,6 +3164,9 @@ class BeamModulePlugin implements Plugin<Project> {
}
String packageFilename = collection.singleFile.toString()
project.exec {
+ if (testJavaHome) {
+ environment "JAVA_HOME", testJavaHome
+ }
executable 'sh'
args '-c', ". ${project.ext.envdir}/bin/activate && cd
${copiedPyRoot} && scripts/run_tox.sh $tox_env ${packageFilename} '$posargs' "
}
@@ -3152,6 +3177,9 @@ class BeamModulePlugin implements Plugin<Project> {
project.copy { from project.pythonSdkDeps; into copiedSrcRoot }
def copiedPyRoot = "${copiedSrcRoot}/sdks/python"
project.exec {
+ if (testJavaHome) {
+ environment "JAVA_HOME", testJavaHome
+ }
executable 'sh'
args '-c', ". ${project.ext.envdir}/bin/activate && cd
${copiedPyRoot} && scripts/run_tox.sh $tox_env '$posargs'"
}
diff --git a/runners/samza/job-server/build.gradle
b/runners/samza/job-server/build.gradle
index 6fc8db98a4f..05f6de39254 100644
--- a/runners/samza/job-server/build.gradle
+++ b/runners/samza/job-server/build.gradle
@@ -198,6 +198,12 @@ tasks.register("validatesPortableRunner") {
dependsOn validatesPortableRunnerEmbedded
}
+def testJavaVersion = project.findProperty('testJavaVersion')
+String testJavaHome = null
+if (testJavaVersion) {
+ testJavaHome = project.findProperty("java${testJavaVersion}Home")
+}
+
def jobPort = BeamModulePlugin.getRandomPort()
def artifactPort = BeamModulePlugin.getRandomPort()
@@ -205,14 +211,18 @@ def setupTask =
project.tasks.register("samzaJobServerSetup", Exec) {
dependsOn shadowJar
def pythonDir = project.project(":sdks:python").projectDir
def samzaJobServerJar = shadowJar.archivePath
-
+ if (testJavaHome) {
+ environment "JAVA_HOME", testJavaHome
+ }
executable 'sh'
args '-c', "$pythonDir/scripts/run_job_server.sh stop --group_id
${project.name} && $pythonDir/scripts/run_job_server.sh start --group_id
${project.name} --job_port ${jobPort} --artifact_port ${artifactPort}
--job_server_jar ${samzaJobServerJar}"
}
def cleanupTask = project.tasks.register("samzaJobServerCleanup", Exec) {
def pythonDir = project.project(":sdks:python").projectDir
-
+ if (testJavaHome) {
+ environment "JAVA_HOME", testJavaHome
+ }
executable 'sh'
args '-c', "$pythonDir/scripts/run_job_server.sh stop --group_id
${project.name}"
}
diff --git a/sdks/go/test/build.gradle b/sdks/go/test/build.gradle
index 7a4c0a63aa4..74b6a10cad4 100644
--- a/sdks/go/test/build.gradle
+++ b/sdks/go/test/build.gradle
@@ -22,6 +22,12 @@ applyPythonNature()
description = "Apache Beam :: SDKs :: Go :: Test"
+def testJavaVersion = project.findProperty('testJavaVersion')
+String fork_java_home = null
+if (testJavaVersion) {
+ fork_java_home = project.findProperty("java${testJavaVersion}Home")
+}
+
// ValidatesRunner tests for Dataflow. Runs tests in the integration directory
// with Dataflow to validate that the runner behaves as expected.
task dataflowValidatesRunner() {
@@ -34,6 +40,9 @@ task dataflowValidatesRunner() {
"--runner dataflow",
]
exec {
+ if (fork_java_home) {
+ environment "JAVA_HOME", fork_java_home
+ }
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
}
@@ -56,6 +65,9 @@ task dataflowValidatesRunnerARM64() {
"--pipeline_opts \"${pipelineOptions.join(' ')}\"",
]
exec {
+ if (fork_java_home) {
+ environment "JAVA_HOME", fork_java_home
+ }
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
}
@@ -82,6 +94,9 @@ task flinkValidatesRunner {
"--pipeline_opts \"${pipelineOptions.join(' ')}\"",
]
exec {
+ if (fork_java_home) {
+ environment "JAVA_HOME", fork_java_home
+ }
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
}
@@ -96,6 +111,7 @@ task samzaValidatesRunner {
dependsOn ":sdks:java:container:${project.ext.currentJavaVersion}:docker"
dependsOn ":runners:samza:job-server:shadowJar"
dependsOn ":sdks:java:testing:expansion-service:buildTestExpansionServiceJar"
+
doLast {
def pipelineOptions = [ // Pipeline options piped directly to Go SDK
flags.
"--expansion_jar=test:${project(":sdks:java:testing:expansion-service").buildTestExpansionServiceJar.archivePath}",
@@ -106,6 +122,9 @@ task samzaValidatesRunner {
"--pipeline_opts \"${pipelineOptions.join(' ')}\"",
]
exec {
+ if (fork_java_home) {
+ environment "JAVA_HOME", fork_java_home
+ }
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
}
@@ -131,6 +150,9 @@ task sparkValidatesRunner {
"--pipeline_opts \"${pipelineOptions.join(' ')}\"",
]
exec {
+ if (fork_java_home) {
+ environment "JAVA_HOME", fork_java_home
+ }
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
}
@@ -167,6 +189,9 @@ tasks.register("ulrValidatesRunner") {
args "-c", ". ${envdir}/bin/activate && pip install -e ."
}
exec {
+ if (fork_java_home) {
+ environment "JAVA_HOME", fork_java_home
+ }
executable "sh"
args "-c", ". ${envdir}/bin/activate && ./run_validatesrunner_tests.sh
${options.join(' ')}"
}
@@ -191,6 +216,9 @@ task prismValidatesRunner {
"--pipeline_opts \"${pipelineOptions.join(' ')}\"",
]
exec {
+ if (fork_java_home) {
+ environment "JAVA_HOME", fork_java_home
+ }
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
}
@@ -233,6 +261,9 @@ ext.goIoValidatesRunnerTask = { proj, name, scriptOpts,
pipelineOpts ->
options.addAll(scriptOpts)
logger.info("Running the command: sh -c ./run_validatesrunner_tests.sh
${options.join(' ')}")
exec {
+ if (fork_java_home) {
+ environment "JAVA_HOME", fork_java_home
+ }
executable "sh"
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
}
diff --git a/sdks/go/test/run_validatesrunner_tests.sh
b/sdks/go/test/run_validatesrunner_tests.sh
index 60dd0cd97f1..4278c817d0a 100755
--- a/sdks/go/test/run_validatesrunner_tests.sh
+++ b/sdks/go/test/run_validatesrunner_tests.sh
@@ -259,6 +259,12 @@ s.close()
TMPDIR=$(mktemp -d)
+if [[ -n "$JAVA_HOME" ]]; then
+ JAVA_CMD="$JAVA_HOME/bin/java"
+else
+ JAVA_CMD="java"
+fi
+
# Set up environment based on runner.
if [[ "$RUNNER" == "flink" || "$RUNNER" == "spark" || "$RUNNER" == "samza" ||
"$RUNNER" == "portable" || "$RUNNER" == "prism" ]]; then
if [[ -z "$ENDPOINT" ]]; then
@@ -266,7 +272,7 @@ if [[ "$RUNNER" == "flink" || "$RUNNER" == "spark" ||
"$RUNNER" == "samza" || "$
ENDPOINT="localhost:$JOB_PORT"
echo "No endpoint specified; starting a new $RUNNER job server on
$ENDPOINT"
if [[ "$RUNNER" == "flink" ]]; then
- java \
+ "$JAVA_CMD" \
-jar $FLINK_JOB_SERVER_JAR \
--flink-master [local] \
--flink-conf-dir
$CURRENT_DIRECTORY/../../../runners/flink/src/test/resources \
@@ -274,13 +280,13 @@ if [[ "$RUNNER" == "flink" || "$RUNNER" == "spark" ||
"$RUNNER" == "samza" || "$
--expansion-port 0 \
--artifact-port 0 &
elif [[ "$RUNNER" == "samza" ]]; then
- java \
+ "$JAVA_CMD" \
-jar $SAMZA_JOB_SERVER_JAR \
--job-port $JOB_PORT \
--expansion-port 0 \
--artifact-port 0 &
elif [[ "$RUNNER" == "spark" ]]; then
- java \
+ "$JAVA_CMD" \
-jar $SPARK_JOB_SERVER_JAR \
--spark-master-url local \
--job-port $JOB_PORT \
@@ -311,28 +317,28 @@ if [[ "$RUNNER" != "direct" ]]; then
EXPANSION_PORT=$(python3 -c "$SOCKET_SCRIPT")
TEST_EXPANSION_ADDR="localhost:$EXPANSION_PORT"
echo "No test expansion address specified; starting a new test expansion
server on $TEST_EXPANSION_ADDR"
- java -jar $TEST_EXPANSION_JAR $EXPANSION_PORT &
+ "$JAVA_CMD" -jar $TEST_EXPANSION_JAR $EXPANSION_PORT &
TEST_EXPANSION_PID=$!
fi
if [[ -z "$IO_EXPANSION_ADDR" && -n "$IO_EXPANSION_JAR" ]]; then
EXPANSION_PORT=$(python3 -c "$SOCKET_SCRIPT")
IO_EXPANSION_ADDR="localhost:$EXPANSION_PORT"
echo "No IO expansion address specified; starting a new IO expansion
server on $IO_EXPANSION_ADDR"
- java -jar $IO_EXPANSION_JAR $EXPANSION_PORT &
+ "$JAVA_CMD" -jar $IO_EXPANSION_JAR $EXPANSION_PORT &
IO_EXPANSION_PID=$!
fi
if [[ -z "$SCHEMAIO_EXPANSION_ADDR" && -n "$SCHEMAIO_EXPANSION_JAR" ]]; then
EXPANSION_PORT=$(python3 -c "$SOCKET_SCRIPT")
SCHEMAIO_EXPANSION_ADDR="localhost:$EXPANSION_PORT"
echo "No SchemaIO expansion address specified; starting a new SchemaIO
expansion server on $SCHEMAIO_EXPANSION_ADDR"
- java -jar $SCHEMAIO_EXPANSION_JAR $EXPANSION_PORT &
+ "$JAVA_CMD" -jar $SCHEMAIO_EXPANSION_JAR $EXPANSION_PORT &
SCHEMAIO_EXPANSION_PID=$!
fi
if [[ -z "$DEBEZIUMIO_EXPANSION_ADDR" && -n "$DEBEZIUMIO_EXPANSION_JAR" ]];
then
EXPANSION_PORT=$(python3 -c "$SOCKET_SCRIPT")
DEBEZIUMIO_EXPANSION_ADDR="localhost:$EXPANSION_PORT"
echo "No DebeziumIO expansion address specified; starting a new
DebeziumIO expansion server on $DEBEZIUMIO_EXPANSION_ADDR"
- java -jar $DEBEZIUMIO_EXPANSION_JAR $EXPANSION_PORT &
+ "$JAVA_CMD" -jar $DEBEZIUMIO_EXPANSION_JAR $EXPANSION_PORT &
DEBEZIUMIO_EXPANSION_PID=$!
fi
fi
diff --git a/sdks/python/scripts/run_job_server.sh
b/sdks/python/scripts/run_job_server.sh
index 03e9244d434..ce2ed9d08bc 100755
--- a/sdks/python/scripts/run_job_server.sh
+++ b/sdks/python/scripts/run_job_server.sh
@@ -27,6 +27,11 @@ END
JOB_PORT=8099
ARTIFACT_PORT=8098
+if [[ -n "$JAVA_HOME" ]]; then
+ JAVA_CMD="$JAVA_HOME/bin/java"
+else
+ JAVA_CMD="java"
+fi
while [[ $# -gt 0 ]]; do
key="$1"
@@ -102,7 +107,7 @@ case $STARTSTOP in
fi
echo "Launching job server @ $JOB_PORT ..."
- java -jar $JOB_SERVER_JAR --job-port=$JOB_PORT
--artifact-port=$ARTIFACT_PORT --expansion-port=0 $ADDITIONAL_ARGS
>$TEMP_DIR/$FILE_BASE.log 2>&1 </dev/null &
+ "$JAVA_CMD" -jar $JOB_SERVER_JAR --job-port=$JOB_PORT
--artifact-port=$ARTIFACT_PORT --expansion-port=0 $ADDITIONAL_ARGS
>$TEMP_DIR/$FILE_BASE.log 2>&1 </dev/null &
mypid=$!
if kill -0 $mypid >/dev/null 2>&1; then
echo $mypid >> $pid
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 856ca46f64a..c6aadaeaae6 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -307,6 +307,7 @@ commands =
bash {toxinidir}/scripts/pytest_validates_runner.sh {envname}
{toxinidir}/apache_beam/runners/portability/flink_runner_test.py {posargs}
[testenv:samza-runner-test]
+passenv = JAVA_HOME
extras = test
commands =
bash {toxinidir}/scripts/pytest_validates_runner.sh {envname}
{toxinidir}/apache_beam/runners/portability/samza_runner_test.py {posargs}
