This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new bd1f68a7bb Issue #8755 : Fix the integration test failures from build
2331 (#8759)
bd1f68a7bb is described below
commit bd1f68a7bb18236aaca5fe0d7763d0642017693f
Author: Matt Casters <[email protected]>
AuthorDate: Mon Oct 5 21:05:17 2026 +0200
Issue #8755 : Fix the integration test failures from build 2331 (#8759)
Ban Mask Fields on Beam by plugin id so the converter no longer loads a
class
that is not on the Beam plugin class loader. Build CastleMock 1.68 from the
Git tag for the HTTP tests, ignore the samples pipeline that needs
ollama-local, and give Marquez longer to index an OpenLineage dataset.
---
docker/integration-tests/castlemock/Dockerfile | 45 ++++++++++++++++++++++
.../integration-tests/integration-tests-http.yaml | 6 ++-
.../openlineage/main-0003-openlineage-writers.hwf | 4 +-
.../samples/read-samples-build-hop-run.hpl | 4 ++
integration-tests/scripts/run-tests-docker.sh | 5 +++
plugins/engines/beam/pom.xml | 6 ---
.../hop/beam/engines/BeamPipelineEngine.java | 13 ++++++-
.../HopPipelineMetaToBeamPipelineConverter.java | 33 +++++++++++++---
.../java/org/apache/hop/beam/util/BeamConst.java | 3 ++
.../engines/BeamPipelineEngineSupportsTest.java | 14 ++++---
10 files changed, 113 insertions(+), 20 deletions(-)
diff --git a/docker/integration-tests/castlemock/Dockerfile
b/docker/integration-tests/castlemock/Dockerfile
new file mode 100644
index 0000000000..38f71c6d89
--- /dev/null
+++ b/docker/integration-tests/castlemock/Dockerfile
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# CastleMock 1.68 for the HTTP integration tests.
+#
+# Docker Hub removed castlemock/castlemock, and the v1.68 GitHub release no
longer
+# publishes castlemock.war. Build that war from the v1.68 tag and deploy it
the same
+# way the archived official image did: Tomcat 11, context /castlemock, port
8080,
+# default user admin/admin. The HTTP tests import a project at
+# http://castlemock:8080/castlemock/api/rest/core/project/soap/import.
+#
+
+FROM maven:3.9.11-eclipse-temurin-21 AS build
+
+WORKDIR /src
+ADD https://github.com/castlemock/castlemock/archive/refs/tags/v1.68.tar.gz
/tmp/castlemock.tar.gz
+RUN tar -xzf /tmp/castlemock.tar.gz -C /src --strip-components=1 \
+ && mvn -B -DskipTests -pl deploy/deploy-tomcat/deploy-tomcat-war -am
package
+
+FROM tomcat:11.0.5-jdk21-temurin-noble
+
+RUN rm -rf /usr/local/tomcat/webapps/ROOT \
+ /usr/local/tomcat/webapps/docs \
+ /usr/local/tomcat/webapps/examples \
+ /usr/local/tomcat/webapps/manager \
+ /usr/local/tomcat/webapps/host-manager
+COPY --from=build
/src/deploy/deploy-tomcat/deploy-tomcat-war/target/castlemock.war \
+ /usr/local/tomcat/webapps/castlemock.war
+
+EXPOSE 8080
diff --git a/docker/integration-tests/integration-tests-http.yaml
b/docker/integration-tests/integration-tests-http.yaml
index 43a1679b4b..9fd38c9a87 100644
--- a/docker/integration-tests/integration-tests-http.yaml
+++ b/docker/integration-tests/integration-tests-http.yaml
@@ -98,8 +98,12 @@ services:
- ./resource/http/server.xml:/usr/local/tomcat/conf/server.xml
- ./resource/http/.keystore:/usr/local/tomcat/conf/.keystore
+ # Built from the CastleMock v1.68 tag. The Docker Hub image and the
published war are gone.
castlemock:
- image: castlemock/castlemock
+ build:
+ context: ./castlemock
+ dockerfile: Dockerfile
+ image: hop-it-castlemock:1.68
hostname: castlemock
# A paginating API, one endpoint per RestPaginationType. Every endpoint
serves the same six
diff --git a/integration-tests/openlineage/main-0003-openlineage-writers.hwf
b/integration-tests/openlineage/main-0003-openlineage-writers.hwf
index b759977886..6e69abd3ee 100644
--- a/integration-tests/openlineage/main-0003-openlineage-writers.hwf
+++ b/integration-tests/openlineage/main-0003-openlineage-writers.hwf
@@ -179,10 +179,10 @@ limitations under the License.
</action>
<action>
<name>Wait for delivery and ingestion</name>
- <description/>
+ <description>Marquez accepts the lineage POST and then indexes the
dataset. The following GET can 404 until that index exists.</description>
<type>DELAY</type>
<attributes/>
- <maximumTimeout>15</maximumTimeout>
+ <maximumTimeout>45</maximumTimeout>
<scaletime>0</scaletime>
<parallel>N</parallel>
<xloc>720</xloc>
diff --git a/integration-tests/samples/read-samples-build-hop-run.hpl
b/integration-tests/samples/read-samples-build-hop-run.hpl
index 1e8ec084d0..0079094b00 100644
--- a/integration-tests/samples/read-samples-build-hop-run.hpl
+++ b/integration-tests/samples/read-samples-build-hop-run.hpl
@@ -571,6 +571,10 @@ var hop_run_cmd = './hop-run.sh -j samples -r ' +
run_config + ' -f ' + filename
<item>ssh-pri-key.hpl</item>
<item>requires SSH private key and reachable SSH host</item>
</line>
+ <line>
+ <item>structured-extract-basic.hpl</item>
+ <item>requires an AI provider named ollama-local</item>
+ </line>
</data>
<distribute>Y</distribute>
<copies>1</copies>
diff --git a/integration-tests/scripts/run-tests-docker.sh
b/integration-tests/scripts/run-tests-docker.sh
index 76d64e7ddd..adbc1c5cf1 100755
--- a/integration-tests/scripts/run-tests-docker.sh
+++ b/integration-tests/scripts/run-tests-docker.sh
@@ -486,6 +486,11 @@ for d in "${CURRENT_DIR}"/../${PROJECT_NAME}/; do
PROJECT_NAME=${PROJECT_NAME} TEST_FILTER=${TEST_FILTER}
SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS} \
docker compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up --build
--abort-on-container-exit \
|| COMPOSE_EXIT=$?
+ elif [ "${PROJECT_NAME}" = "http" ]; then
+ echo "Rebuilding http images so CastleMock comes from the v1.68 source
tag"
+ PROJECT_NAME=${PROJECT_NAME} TEST_FILTER=${TEST_FILTER}
SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS} \
+ docker compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up --build
--abort-on-container-exit \
+ || COMPOSE_EXIT=$?
else
PROJECT_NAME=${PROJECT_NAME} TEST_FILTER=${TEST_FILTER}
SKIP_GOOGLE_SHEETS=${SKIP_GOOGLE_SHEETS} \
docker compose -f
${DOCKER_FILES_DIR}/integration-tests-${PROJECT_NAME}.yaml up
--abort-on-container-exit \
diff --git a/plugins/engines/beam/pom.xml b/plugins/engines/beam/pom.xml
index 7e42328e8b..902ee473ab 100644
--- a/plugins/engines/beam/pom.xml
+++ b/plugins/engines/beam/pom.xml
@@ -1611,12 +1611,6 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hop</groupId>
- <artifactId>hop-transform-maskfields</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-memgroupby</artifactId>
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/BeamPipelineEngine.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/BeamPipelineEngine.java
index 95824d021e..0ef0856791 100644
---
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/BeamPipelineEngine.java
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/BeamPipelineEngine.java
@@ -1755,6 +1755,9 @@ public abstract class BeamPipelineEngine extends Variables
* the converter does, so the answer at design time matches what actually
happens at run time.
*
* <ol>
+ * <li>Plugin id on the {@link
HopPipelineMetaToBeamPipelineConverter#HARD_BANNED_PLUGIN_IDS}
+ * list → UNSUPPORTED with the canonical user-facing reason. This path
does not load the
+ * transform class.
* <li>Meta class on the {@link
HopPipelineMetaToBeamPipelineConverter#HARD_BANNED_META_TYPES}
* list → UNSUPPORTED with the canonical user-facing reason.
* <li>Plugin id in {@link
HopPipelineMetaToBeamPipelineConverter#EXPLICIT_HANDLER_PLUGIN_IDS} →
@@ -1775,6 +1778,15 @@ public abstract class BeamPipelineEngine extends
Variables
if (transformPlugin == null) {
return EngineCompatibility.unknown();
}
+ String[] ids = transformPlugin.getIds();
+ if (ids != null) {
+ for (String id : ids) {
+ String ban =
HopPipelineMetaToBeamPipelineConverter.HARD_BANNED_PLUGIN_IDS.get(id);
+ if (ban != null) {
+ return EngineCompatibility.unsupported(ban);
+ }
+ }
+ }
Class<?> mainType = transformPlugin.getMainType();
if (mainType != null) {
String banReason =
@@ -1786,7 +1798,6 @@ public abstract class BeamPipelineEngine extends Variables
return EngineCompatibility.supported();
}
}
- String[] ids = transformPlugin.getIds();
if (ids != null) {
for (String id : ids) {
if
(HopPipelineMetaToBeamPipelineConverter.EXPLICIT_HANDLER_PLUGIN_IDS.contains(id))
{
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
index ccd2c7129e..e85dd60e68 100644
---
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
+++
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/HopPipelineMetaToBeamPipelineConverter.java
@@ -67,7 +67,6 @@ import org.apache.hop.pipeline.transform.ITransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.groupby.GroupByMeta;
import org.apache.hop.pipeline.transforms.joinrows.JoinRowsMeta;
-import org.apache.hop.pipeline.transforms.maskfields.MaskFieldsMeta;
import org.apache.hop.pipeline.transforms.sort.SortRowsMeta;
import org.apache.hop.pipeline.transforms.uniquerows.UniqueRowsMeta;
import
org.apache.hop.pipeline.transforms.uniquerowsbyhashset.UniqueRowsByHashSetMeta;
@@ -95,6 +94,11 @@ public class HopPipelineMetaToBeamPipelineConverter {
* Transform meta classes that Beam refuses to run at all, mapped to the
user-facing reason. The
* runtime check in {@link #validateTransformBeamUsage} consults this map;
{@code
* BeamPipelineEngine.supports} surfaces the same reason at design time.
+ *
+ * <p>Only classes that {@code dependencies.xml} already puts on the Beam
plugin class loader
+ * belong here. A class literal is resolved when this class is initialized,
so a transform that is
+ * not on that loader (Mask fields) is banned by plugin id in {@link
#HARD_BANNED_PLUGIN_IDS}
+ * instead.
*/
public static final Map<Class<?>, String> HARD_BANNED_META_TYPES =
Map.of(
@@ -107,8 +111,16 @@ public class HopPipelineMetaToBeamPipelineConverter {
UniqueRowsByHashSetMeta.class,
"Unique Rows By Hashset is not supported on Beam. Every worker
keeps its own hash set, so duplicates spread over different workers would
survive. Use a Memory Group By to get distinct rows.",
JoinRowsMeta.class,
- "Join Rows is not supported on Beam. A cartesian product needs
every row of every input in one place, but every worker would only combine the
rows it happens to hold, so combinations would go missing. Add the same
constant field to both inputs and use a Merge Join on that field instead.",
- MaskFieldsMeta.class,
+ "Join Rows is not supported on Beam. A cartesian product needs
every row of every input in one place, but every worker would only combine the
rows it happens to hold, so combinations would go missing. Add the same
constant field to both inputs and use a Merge Join on that field instead.");
+
+ /**
+ * Plugin ids Beam refuses to run, mapped to the user-facing reason. Used
for transforms that are
+ * not on the Beam plugin class loader. Keep in lockstep with {@code
BeamPipelineEngine.supports}
+ * and {@link #validateTransformBeamUsage}.
+ */
+ public static final Map<String, String> HARD_BANNED_PLUGIN_IDS =
+ Map.of(
+ BeamConst.STRING_MASK_FIELDS_PLUGIN_ID,
"Mask fields is not supported on Beam. Each worker keeps its own
mapping and sequence, so the same source value would not stay the same token.");
protected final String runConfigName;
@@ -533,7 +545,7 @@ public class HopPipelineMetaToBeamPipelineConverter {
// Generic transform
//
- validateTransformBeamUsage(transformMeta.getTransform());
+ validateTransformBeamUsage(transformMeta);
// Lookup all the previous transforms for this one, excluding info
transforms like
// StreamLookup...
@@ -648,7 +660,18 @@ public class HopPipelineMetaToBeamPipelineConverter {
}
}
- private void validateTransformBeamUsage(ITransformMeta meta) throws
HopException {
+ private void validateTransformBeamUsage(TransformMeta transformMeta) throws
HopException {
+ if (transformMeta == null) {
+ return;
+ }
+ String pluginId = transformMeta.getTransformPluginId();
+ if (pluginId != null) {
+ String pluginBan = HARD_BANNED_PLUGIN_IDS.get(pluginId);
+ if (pluginBan != null) {
+ throw new HopException(pluginBan);
+ }
+ }
+ ITransformMeta meta = transformMeta.getTransform();
if (meta == null) {
return;
}
diff --git
a/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java
b/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java
index 71e6044b63..a89f5c3546 100644
--- a/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java
+++ b/plugins/engines/beam/src/main/java/org/apache/hop/beam/util/BeamConst.java
@@ -45,6 +45,9 @@ public class BeamConst {
public static final String STRING_BEAM_KAFKA_PRODUCE_PLUGIN_ID =
"BeamKafkaProduce";
public static final String STRING_BEAM_ROW_GENERATOR_PLUGIN_ID =
"RowGenerator";
+ /** Mask fields. Banned by plugin id so the Beam engine does not have to
load the transform. */
+ public static final String STRING_MASK_FIELDS_PLUGIN_ID = "MaskFields";
+
public static final String STRING_HOP_BEAM = "Hop Beam";
public static final String STRING_TRANSFORM_FLAG_BATCH = "Batch";
diff --git
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/BeamPipelineEngineSupportsTest.java
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/BeamPipelineEngineSupportsTest.java
index a2c48d9dec..10d432d189 100644
---
a/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/BeamPipelineEngineSupportsTest.java
+++
b/plugins/engines/beam/src/test/java/org/apache/hop/beam/engines/BeamPipelineEngineSupportsTest.java
@@ -32,7 +32,6 @@ import org.apache.hop.core.plugins.IPlugin;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transforms.groupby.GroupByMeta;
import org.apache.hop.pipeline.transforms.joinrows.JoinRowsMeta;
-import org.apache.hop.pipeline.transforms.maskfields.MaskFieldsMeta;
import org.apache.hop.pipeline.transforms.sort.SortRowsMeta;
import org.apache.hop.pipeline.transforms.uniquerows.UniqueRowsMeta;
import
org.apache.hop.pipeline.transforms.uniquerowsbyhashset.UniqueRowsByHashSetMeta;
@@ -74,11 +73,16 @@ class BeamPipelineEngineSupportsTest {
}
@Test
- void maskFieldsMetaIsHardBanned() {
- EngineCompatibility verdict =
engine.supports(pluginWithMainType(MaskFieldsMeta.class));
- assertTrue(verdict.isUnsupported(), "MaskFieldsMeta should be
UNSUPPORTED");
+ void maskFieldsPluginIdIsHardBanned() {
+ // Ban by plugin id. Referencing MaskFieldsMeta here would put that class
on the Beam engine
+ // class loader, which is what broke every Beam run after the transform
was added.
+ EngineCompatibility verdict =
+ engine.supports(
+ pluginWithIdAndMainType(BeamConst.STRING_MASK_FIELDS_PLUGIN_ID,
Object.class));
+ assertTrue(verdict.isUnsupported(), "MaskFields should be UNSUPPORTED");
assertEquals(
-
HopPipelineMetaToBeamPipelineConverter.HARD_BANNED_META_TYPES.get(MaskFieldsMeta.class),
+ HopPipelineMetaToBeamPipelineConverter.HARD_BANNED_PLUGIN_IDS.get(
+ BeamConst.STRING_MASK_FIELDS_PLUGIN_ID),
verdict.getReason());
}
