This is an automated email from the ASF dual-hosted git repository.
shunping pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 720fa2fcb5f Track ReplaceTokens values as processResources inputs
(#40133)
720fa2fcb5f is described below
commit 720fa2fcb5f8f08b7c0385de4279b0e3f871c74e
Author: Shunping Huang <[email protected]>
AuthorDate: Wed Sep 16 09:48:50 2026 -0400
Track ReplaceTokens values as processResources inputs (#40133)
Gradle treats the `tokens:` map passed to the ReplaceTokens content
filter as a filter argument, not a task input, so it affects neither
the up-to-date check nor the build cache key. The filtered sources
contain only unsubstituted placeholders and are byte identical
regardless of what the tokens expand to, so nothing invalidates
processResources when a token value changes.
In a shared checkout where build/ survives branch switches, the task
stays UP-TO-DATE and the previous branch's substituted resource is
left behind. Switching master -> a former release branch left
dataflow.properties holding master's container version, so pipelines
ran against a wrong version of beam-master SDK harness. The same
staleness also poisons the build cache, since both branches compute
the same key for different correct outputs, so `clean` is not a
reliable workaround.
This can be fixed by declaring the token map as an input property.
---
runners/google-cloud-dataflow-java/arm/build.gradle | 5 ++++-
runners/google-cloud-dataflow-java/build.gradle | 5 ++++-
sdks/java/extensions/sql/jdbc/build.gradle | 5 ++++-
sdks/java/maven-archetypes/examples/build.gradle | 5 ++++-
sdks/java/maven-archetypes/gcp-bom-examples/build.gradle | 5 ++++-
sdks/java/maven-archetypes/starter/build.gradle | 5 ++++-
6 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/runners/google-cloud-dataflow-java/arm/build.gradle
b/runners/google-cloud-dataflow-java/arm/build.gradle
index 531300bb63e..b1e1ae59e9b 100644
--- a/runners/google-cloud-dataflow-java/arm/build.gradle
+++ b/runners/google-cloud-dataflow-java/arm/build.gradle
@@ -52,11 +52,14 @@
evaluationDependsOn(":runners:google-cloud-dataflow-java:worker")
evaluationDependsOn(":sdks:java:container:java11")
processResources {
- filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
+ // Filter arguments are not tracked by Gradle, so declare the tokens as an
input property.
+ def resourceTokens = [
'dataflow.fnapi_environment_major_version' :
project(':runners:google-cloud-dataflow-java').ext.dataflowFnapiEnvironmentMajorVersion,
'dataflow.fnapi_container_version' :
project(':runners:google-cloud-dataflow-java').ext.dataflowFnapiContainerVersion,
'dataflow.container_base_repository' :
project(':runners:google-cloud-dataflow-java').ext.dataflowContainerBaseRepository,
]
+ inputs.property('resourceTokens', resourceTokens)
+ filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens
}
// Exclude tests that need a runner
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index b513b1aa207..fae0ab53c70 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -57,13 +57,16 @@ ext.dataflowFnapiContainerVersion = 'beam-master-20260909'
ext.dataflowContainerBaseRepository = 'gcr.io/cloud-dataflow/v1beta3'
processResources {
- filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
+ // Filter arguments are not tracked by Gradle, so declare the tokens as an
input property.
+ def resourceTokens = [
'dataflow.legacy_environment_major_version' :
project.ext.dataflowFnapiEnvironmentMajorVersion,
'dataflow.fnapi_environment_major_version' :
project.ext.dataflowFnapiEnvironmentMajorVersion,
'dataflow.legacy_container_version' :
project.ext.dataflowLegacyContainerVersion,
'dataflow.fnapi_container_version' :
project.ext.dataflowFnapiContainerVersion,
'dataflow.container_base_repository' :
project.ext.dataflowContainerBaseRepository,
]
+ inputs.property('resourceTokens', resourceTokens)
+ filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens
}
// Exclude tests that need a runner
diff --git a/sdks/java/extensions/sql/jdbc/build.gradle
b/sdks/java/extensions/sql/jdbc/build.gradle
index 91c48b635d5..334da329308 100644
--- a/sdks/java/extensions/sql/jdbc/build.gradle
+++ b/sdks/java/extensions/sql/jdbc/build.gradle
@@ -63,9 +63,12 @@ dependencies {
}
processResources {
- filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
+ // Filter arguments are not tracked by Gradle, so declare the tokens as an
input property.
+ def resourceTokens = [
'beam.version': version,
]
+ inputs.property('resourceTokens', resourceTokens)
+ filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens
}
shadowJar {
diff --git a/sdks/java/maven-archetypes/examples/build.gradle
b/sdks/java/maven-archetypes/examples/build.gradle
index 1edb55a10f9..c3c203410c8 100644
--- a/sdks/java/maven-archetypes/examples/build.gradle
+++ b/sdks/java/maven-archetypes/examples/build.gradle
@@ -25,7 +25,8 @@ example pipelines from the Apache Beam Java SDK."""
processResources {
dependsOn 'generateSources'
- filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
+ // Filter arguments are not tracked by Gradle, so declare the tokens as an
input property.
+ def resourceTokens = [
'project.version': version,
'bigquery.version':
dependencies.create(project.library.java.google_api_services_bigquery).getVersion(),
'google-api-client.version':
dependencies.create(project.library.java.google_api_client).getVersion(),
@@ -49,6 +50,8 @@ processResources {
'maven-surefire-plugin.version':
dependencies.create(project.library.maven.maven_surefire_plugin).getVersion(),
'flink.artifact.name':
'beam-runners-flink-'.concat(project(":runners:flink:${project.ext.latestFlinkVersion}").getName()),
]
+ inputs.property('resourceTokens', resourceTokens)
+ filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens
}
/*
diff --git a/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle
b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle
index f9fabcfe19b..845cf5aa5c9 100644
--- a/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle
+++ b/sdks/java/maven-archetypes/gcp-bom-examples/build.gradle
@@ -26,7 +26,8 @@ using the Beam Google Cloud Platform BOM"""
processResources {
dependsOn 'generateSources'
- filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
+ // Filter arguments are not tracked by Gradle, so declare the tokens as an
input property.
+ def resourceTokens = [
'project.version': version,
'bigquery.version':
dependencies.create(project.library.java.google_api_services_bigquery).getVersion(),
'google-api-client.version':
dependencies.create(project.library.java.google_api_client).getVersion(),
@@ -48,6 +49,8 @@ processResources {
'maven-surefire-plugin.version':
dependencies.create(project.library.maven.maven_surefire_plugin).getVersion(),
'flink.artifact.name':
'beam-runners-flink-'.concat(project(":runners:flink:${project.ext.latestFlinkVersion}").getName()),
]
+ inputs.property('resourceTokens', resourceTokens)
+ filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens
}
/*
diff --git a/sdks/java/maven-archetypes/starter/build.gradle
b/sdks/java/maven-archetypes/starter/build.gradle
index 7c8c02bd559..3bb4a311fca 100644
--- a/sdks/java/maven-archetypes/starter/build.gradle
+++ b/sdks/java/maven-archetypes/starter/build.gradle
@@ -24,10 +24,13 @@ ext.summary = """A Maven archetype to create a simple
starter pipeline to
get started using the Apache Beam Java SDK."""
processResources {
- filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
+ // Filter arguments are not tracked by Gradle, so declare the tokens as an
input property.
+ def resourceTokens = [
'project.version': version,
'slf4j.version':
dependencies.create(project.library.java.slf4j_api).getVersion(),
'maven-compiler-plugin.version':
dependencies.create(project.library.maven.maven_compiler_plugin).getVersion(),
'maven-exec-plugin.version':
dependencies.create(project.library.maven.maven_exec_plugin).getVersion(),
]
+ inputs.property('resourceTokens', resourceTokens)
+ filter org.apache.tools.ant.filters.ReplaceTokens, tokens: resourceTokens
}
