This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 7fa61fcfe2 Issue #8417 : Use the unit test database replacement in
Database Lookup (#8520)
7fa61fcfe2 is described below
commit 7fa61fcfe2c6dc2dc8d64e9d8f703d0bf3264d80
Author: Matt Casters <[email protected]>
AuthorDate: Tue Sep 22 10:03:47 2026 +0200
Issue #8417 : Use the unit test database replacement in Database Lookup
(#8520)
---
.../ROOT/pages/pipeline/pipeline-unit-testing.adoc | 2 +-
plugins/misc/testing/pom.xml | 18 ++
.../hop/testing/xp/PipelineMetaModifier.java | 48 +++++-
.../hop/testing/xp/PipelineMetaModifierTest.java | 186 +++++++++++++++++++++
.../transforms/addsequence/AddSequenceMeta.java | 4 +-
.../gettablenames/GetTableNamesMeta.java | 4 +-
.../transforms/jdbcmetadata/JdbcMetadataMeta.java | 4 +-
.../SynchronizeAfterMergeMeta.java | 4 +-
8 files changed, 260 insertions(+), 10 deletions(-)
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-unit-testing.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-unit-testing.adoc
index d60c787d58..5094c5ec5b 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-unit-testing.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/pipeline-unit-testing.adoc
@@ -145,7 +145,7 @@ By default, you should see the active pipeline filename
here.
|Test pipeline filename (Optional)|the filename to use for this unit test.
|Base test path (or use HOP_UNIT_TESTS_FOLDER)|the folder to store this unit
test to.
|Select this test automatically|default: false
-|Replace a database connection with another|specify a list of database
connections for this pipeline that you'd like to swap out for a test-specific
connection.
+|Replace a database connection with another|specify a list of database
connections for this pipeline that you'd like to swap out for a test-specific
connection. The swap rewrites those relational connection references on the
in-memory test copy, including lookups, and does not change the project
connection.
|Variables|a list of variables to use in testing.
|===
diff --git a/plugins/misc/testing/pom.xml b/plugins/misc/testing/pom.xml
index fc3b98b2f1..ff401fd372 100644
--- a/plugins/misc/testing/pom.xml
+++ b/plugins/misc/testing/pom.xml
@@ -47,6 +47,24 @@
<artifactId>commons-csv</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-databases-generic</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-databases-h2</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hop</groupId>
+ <artifactId>hop-transform-databaselookup</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/PipelineMetaModifier.java
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/PipelineMetaModifier.java
index fbfed30317..8b811642b0 100644
---
a/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/PipelineMetaModifier.java
+++
b/plugins/misc/testing/src/main/java/org/apache/hop/testing/xp/PipelineMetaModifier.java
@@ -20,7 +20,9 @@ package org.apache.hop.testing.xp;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import org.apache.hop.core.database.DatabaseMeta;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.ILogChannel;
@@ -29,7 +31,9 @@ import org.apache.hop.core.plugins.TransformPluginType;
import org.apache.hop.core.row.IRowMeta;
import org.apache.hop.core.row.IValueMeta;
import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.util.HopMetadataPropertyWalker;
import org.apache.hop.pipeline.PipelineHopMeta;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
@@ -76,8 +80,11 @@ public class PipelineMetaModifier {
//
copyPipelineMeta.setMetadataProvider(pipelineMeta.getMetadataProvider());
- // Replace certain connections with another
+ // Connections are metadata loaded by name. Rewriting the name on the test
copy is what
+ // transforms such as Database Lookup read at runtime. Do not call
DatabaseMeta.replaceMeta():
+ // findDatabase() returns either a throwaway load or the shared metadata
object.
//
+ Map<String, String> connectionReplacements = new LinkedHashMap<>();
for (PipelineUnitTestDatabaseReplacement dbReplacement :
unitTest.getDatabaseReplacements()) {
String sourceDatabaseName =
variables.resolve(dbReplacement.getOriginalDatabaseName());
String replacementDatabaseName =
@@ -99,16 +106,33 @@ public class PipelineMetaModifier {
+ replacementDatabaseName
+ "', can not be used to replace");
}
+ connectionReplacements.put(sourceDatabaseName, replacementDatabaseName);
+ }
+
+ int replacedFields = 0;
+ if (!connectionReplacements.isEmpty()) {
+ for (TransformMeta transformMeta : copyPipelineMeta.getTransforms()) {
+ replacedFields +=
+ HopMetadataPropertyWalker.rewriteStrings(
+ transformMeta.getTransform(),
+ HopMetadataPropertyType.RDBMS_CONNECTION,
+ value -> replacementFor(variables, connectionReplacements,
value));
+ }
+ }
- if (log.isDetailed()) {
+ if (log.isDetailed()) {
+ for (Map.Entry<String, String> replacement :
connectionReplacements.entrySet()) {
log.logDetailed(
"Replaced database connection '"
- + sourceDatabaseName
+ + replacement.getKey()
+ "' with connection '"
- + replacementDatabaseName
+ + replacement.getValue()
+ "'");
}
- sourceDatabaseMeta.replaceMeta(replacementDatabaseMeta);
+ if (!connectionReplacements.isEmpty()) {
+ log.logDetailed(
+ "Rewrote " + replacedFields + " relational connection reference(s)
for this unit test");
+ }
}
// Replace all transforms with an Input Data Set marker with an Injector
@@ -160,6 +184,20 @@ public class PipelineMetaModifier {
return copyPipelineMeta;
}
+ /**
+ * Map a connection name to its replacement once. A later replacement of the
replacement name does
+ * not chain. Variable expressions are resolved before matching, and a match
is stored as the
+ * literal replacement name.
+ */
+ private static String replacementFor(
+ IVariables variables, Map<String, String> connectionReplacements, String
value) {
+ if (value == null) {
+ return null;
+ }
+ String replacement = connectionReplacements.get(variables.resolve(value));
+ return replacement == null ? value : replacement;
+ }
+
private void handleInputDataSet(
ILogChannel log,
PipelineUnitTestSetLocation inputLocation,
diff --git
a/plugins/misc/testing/src/test/java/org/apache/hop/testing/xp/PipelineMetaModifierTest.java
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/xp/PipelineMetaModifierTest.java
new file mode 100644
index 0000000000..db099ac6c6
--- /dev/null
+++
b/plugins/misc/testing/src/test/java/org/apache/hop/testing/xp/PipelineMetaModifierTest.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.testing.xp;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.database.DatabaseMeta;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.databases.generic.GenericDatabaseMeta;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.databaselookup.DatabaseLookupMeta;
+import org.apache.hop.testing.PipelineUnitTest;
+import org.apache.hop.testing.PipelineUnitTestDatabaseReplacement;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class PipelineMetaModifierTest {
+
+ private static final String SQL_SERVER_DRIVER =
"com.microsoft.sqlserver.jdbc.SQLServerDriver";
+ private static final String H2_DRIVER = "org.h2.Driver";
+
+ @BeforeAll
+ static void initHop() throws HopException {
+ HopEnvironment.init();
+ }
+
+ @Test
+ void replacesDatabaseLookupConnectionOnTheTestCopy() throws HopException {
+ IVariables variables = new Variables();
+ variables.setVariable("DB", "source-db");
+ MemoryMetadataProvider provider = new MemoryMetadataProvider();
+ DatabaseMeta source = genericConnection("source-db", "source.example",
"source_db");
+ DatabaseMeta test = h2Connection("test-h2", "mem:lookup_data");
+ save(provider, source, test, genericConnection("other-db",
"other.example", "other_db"));
+
+ PipelineMeta pipeline =
+ pipeline(
+ provider,
+ lookup("lookup", "source-db"),
+ lookup("other", "other-db"),
+ lookup("variable-lookup", "${DB}"));
+ PipelineUnitTest unitTest = new PipelineUnitTest();
+ unitTest
+ .getDatabaseReplacements()
+ .add(new PipelineUnitTestDatabaseReplacement("source-db", "test-h2"));
+
+ PipelineMeta copy =
+ new PipelineMetaModifier(variables, pipeline, unitTest)
+ .getTestPipeline(LogChannel.GENERAL, variables, provider);
+
+ assertEquals("test-h2", connection(copy, "lookup"));
+ DatabaseMeta resolved = copy.findDatabase(connection(copy, "lookup"),
variables);
+ assertEquals("H2", resolved.getPluginId());
+ assertEquals("mem:lookup_data", resolved.getDatabaseName());
+ assertEquals(H2_DRIVER, resolved.getDriverClass(variables));
+
+ assertEquals("other-db", connection(copy, "other"));
+ assertEquals("test-h2", connection(copy, "variable-lookup"));
+ assertEquals("source-db", connection(pipeline, "lookup"));
+
+ DatabaseMeta sourceAfter =
provider.getSerializer(DatabaseMeta.class).load("source-db");
+ assertEquals("source-db", sourceAfter.getName());
+ assertEquals("GENERIC", sourceAfter.getPluginId());
+ assertEquals(SQL_SERVER_DRIVER, sourceAfter.getDriverClass(variables));
+ assertEquals("source.example", sourceAfter.getHostname());
+ }
+
+ @Test
+ void replacementsDoNotChain() throws HopException {
+ IVariables variables = new Variables();
+ MemoryMetadataProvider provider = new MemoryMetadataProvider();
+ save(
+ provider,
+ genericConnection("source-db", "source.example", "source_db"),
+ h2Connection("test-h2", "mem:lookup_data"),
+ h2Connection("other-h2", "mem:other"));
+
+ PipelineMeta pipeline = pipeline(provider, lookup("lookup", "source-db"));
+ PipelineUnitTest unitTest = new PipelineUnitTest();
+ unitTest
+ .getDatabaseReplacements()
+ .add(new PipelineUnitTestDatabaseReplacement("source-db", "test-h2"));
+ unitTest
+ .getDatabaseReplacements()
+ .add(new PipelineUnitTestDatabaseReplacement("test-h2", "other-h2"));
+
+ PipelineMeta copy =
+ new PipelineMetaModifier(variables, pipeline, unitTest)
+ .getTestPipeline(LogChannel.GENERAL, variables, provider);
+
+ assertEquals("test-h2", connection(copy, "lookup"));
+ }
+
+ @Test
+ void missingReplacementDoesNotChangeTheSourceConnection() throws
HopException {
+ IVariables variables = new Variables();
+ MemoryMetadataProvider provider = new MemoryMetadataProvider();
+ DatabaseMeta source = genericConnection("source-db", "source.example",
"source_db");
+ save(provider, source);
+ PipelineMeta pipeline = pipeline(provider, lookup("lookup", "source-db"));
+ PipelineUnitTest unitTest = new PipelineUnitTest();
+ unitTest
+ .getDatabaseReplacements()
+ .add(new PipelineUnitTestDatabaseReplacement("source-db", "test-h2"));
+
+ HopException exception =
+ assertThrows(
+ HopException.class,
+ () ->
+ new PipelineMetaModifier(variables, pipeline, unitTest)
+ .getTestPipeline(LogChannel.GENERAL, variables, provider));
+
+ assertTrue(exception.getMessage().contains("test-h2"));
+ assertEquals("source-db", source.getName());
+ assertEquals("GENERIC", source.getPluginId());
+ assertEquals(SQL_SERVER_DRIVER, source.getDriverClass(variables));
+ assertEquals("source-db", connection(pipeline, "lookup"));
+ }
+
+ private static PipelineMeta pipeline(MemoryMetadataProvider provider,
TransformMeta... transforms)
+ throws HopException {
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.setName("lookup-test");
+ pipelineMeta.setMetadataProvider(provider);
+ for (TransformMeta transform : transforms) {
+ pipelineMeta.addTransform(transform);
+ }
+ return pipelineMeta;
+ }
+
+ private static TransformMeta lookup(String name, String connection) {
+ DatabaseLookupMeta lookupMeta = new DatabaseLookupMeta();
+ lookupMeta.setConnection(connection);
+ return new TransformMeta("DBLookup", name, lookupMeta);
+ }
+
+ private static String connection(PipelineMeta pipelineMeta, String
transformName) {
+ return ((DatabaseLookupMeta)
pipelineMeta.findTransform(transformName).getTransform())
+ .getConnection();
+ }
+
+ private static void save(MemoryMetadataProvider provider, DatabaseMeta...
connections)
+ throws HopException {
+ IHopMetadataSerializer<DatabaseMeta> serializer =
provider.getSerializer(DatabaseMeta.class);
+ for (DatabaseMeta connection : connections) {
+ serializer.save(connection);
+ }
+ }
+
+ private static DatabaseMeta genericConnection(String name, String host,
String database) {
+ DatabaseMeta databaseMeta =
+ new DatabaseMeta(name, "GENERIC", "Native", host, database, "1433",
"sa", "secret");
+ databaseMeta
+ .getAttributes()
+ .put(GenericDatabaseMeta.ATTRIBUTE_CUSTOM_DRIVER_CLASS,
SQL_SERVER_DRIVER);
+ return databaseMeta;
+ }
+
+ private static DatabaseMeta h2Connection(String name, String database) {
+ return new DatabaseMeta(name, "H2", "Native", "localhost", database, "",
"sa", "");
+ }
+}
diff --git
a/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java
b/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java
index 6cfe71782b..7038ea3873 100644
---
a/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java
+++
b/plugins/transforms/addsequence/src/main/java/org/apache/hop/pipeline/transforms/addsequence/AddSequenceMeta.java
@@ -34,6 +34,7 @@ import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
@@ -66,7 +67,8 @@ public class AddSequenceMeta extends
BaseTransformMeta<AddSequence, AddSequenceD
@HopMetadataProperty(
key = "connection",
- injectionKeyDescription = "AddSequenceMeta.Injection.Connection")
+ injectionKeyDescription = "AddSequenceMeta.Injection.Connection",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
private String connection;
@HopMetadataProperty(
diff --git
a/plugins/transforms/gettablenames/src/main/java/org/apache/hop/pipeline/transforms/gettablenames/GetTableNamesMeta.java
b/plugins/transforms/gettablenames/src/main/java/org/apache/hop/pipeline/transforms/gettablenames/GetTableNamesMeta.java
index bbd7989d3b..6163d1ed13 100644
---
a/plugins/transforms/gettablenames/src/main/java/org/apache/hop/pipeline/transforms/gettablenames/GetTableNamesMeta.java
+++
b/plugins/transforms/gettablenames/src/main/java/org/apache/hop/pipeline/transforms/gettablenames/GetTableNamesMeta.java
@@ -32,6 +32,7 @@ import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.PipelineMeta;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
@@ -55,7 +56,8 @@ public class GetTableNamesMeta extends
BaseTransformMeta<GetTableNames, GetTable
@HopMetadataProperty(
key = "connection",
injectionKey = "CONNECTIONNAME",
- injectionKeyDescription = "GetTableNames.Injection.CONNECTION_NAME")
+ injectionKeyDescription = "GetTableNames.Injection.CONNECTION_NAME",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
private String connection;
@HopMetadataProperty(
diff --git
a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/hop/pipeline/transforms/jdbcmetadata/JdbcMetadataMeta.java
b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/hop/pipeline/transforms/jdbcmetadata/JdbcMetadataMeta.java
index e20c65088c..539d6c0441 100644
---
a/plugins/transforms/jdbc-metadata/src/main/java/org/apache/hop/pipeline/transforms/jdbcmetadata/JdbcMetadataMeta.java
+++
b/plugins/transforms/jdbc-metadata/src/main/java/org/apache/hop/pipeline/transforms/jdbcmetadata/JdbcMetadataMeta.java
@@ -34,6 +34,7 @@ import org.apache.hop.core.row.value.ValueMetaInteger;
import org.apache.hop.core.row.value.ValueMetaString;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.transform.BaseTransformMeta;
import org.apache.hop.pipeline.transform.TransformMeta;
@@ -381,7 +382,8 @@ public class JdbcMetadataMeta extends
BaseTransformMeta<JdbcMetadata, JdbcMetada
outputFields.add(new OutputField(CONST_TABLE_CAT, CONST_TABLE_CAT));
}
- @HopMetadataProperty private String connection;
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.RDBMS_CONNECTION)
+ private String connection;
public String getConnection() {
return connection;
diff --git
a/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeMeta.java
b/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeMeta.java
index 61b029d01d..8f875b7cbe 100644
---
a/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeMeta.java
+++
b/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeMeta.java
@@ -38,6 +38,7 @@ import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.DatabaseImpact;
import org.apache.hop.pipeline.PipelineMeta;
@@ -63,7 +64,8 @@ public class SynchronizeAfterMergeMeta
@HopMetadataProperty(
key = "connection",
injectionKey = "CONNECTION_NAME",
- injectionKeyDescription =
"SynchronizeAfterMerge.Injection.CONNECTION_NAME")
+ injectionKeyDescription =
"SynchronizeAfterMerge.Injection.CONNECTION_NAME",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
private String connection;
/** Commit size for inserts/updates */
