This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 4c836794b5 Issue #2955 : Warn when a referenced database connection
does not exist (#8245)
4c836794b5 is described below
commit 4c836794b5d4ccd7f04c006b7b51fbdfeadfe7f1
Author: Matt Casters <[email protected]>
AuthorDate: Thu Sep 3 15:55:29 2026 +0200
Issue #2955 : Warn when a referenced database connection does not exist
(#8245)
Warn on save and Verify when a transform or action names a relational
database connection that is not in project metadata. Saving is never
blocked. Names that still contain a variable after resolving the current
environment are skipped, and the check does not open a JDBC connection.
---
.../java/org/apache/hop/core/util/StringUtil.java | 18 ++
.../metadata/util/HopMetadataPropertyWalker.java | 159 ++++++++++++++
.../org/apache/hop/core/util/StringUtilTest.java | 12 ++
.../util/HopMetadataPropertyWalkerTest.java | 124 +++++++++++
...n-validate-database-connections-when-saving.png | Bin 0 -> 118099 bytes
...database-connections-on-save-warning-dialog.png | Bin 0 -> 62130 bytes
.../pages/hop-gui/perspective-configuration.adoc | 7 +
.../pages/metadata-types/rdbms-connection.adoc | 4 +
.../ROOT/pages/pipeline/create-pipeline.adoc | 4 +
.../ROOT/pages/workflow/create-workflow.adoc | 4 +
.../ReferencedDatabaseConnectionChecker.java | 208 ++++++++++++++++++
.../java/org/apache/hop/pipeline/PipelineMeta.java | 7 +
.../java/org/apache/hop/workflow/WorkflowMeta.java | 4 +
.../validation/messages/messages_en_US.properties | 20 ++
.../ReferencedDatabaseConnectionCheckerTest.java | 235 +++++++++++++++++++++
.../actions/waitforsql/ActionWaitForSql.java | 13 +-
.../actions/mssqlbulkload/ActionMssqlBulkLoad.java | 13 +-
.../actions/mysqlbulkload/ActionMysqlBulkLoad.java | 13 +-
.../actions/snowflake/WarehouseManager.java | 5 +-
.../sqlfileoutput/SQLFileOutputMeta.java | 11 +-
.../file/config/FileValidationConfigPlugin.java | 148 +++++++++++++
.../hopgui/file/pipeline/HopGuiPipelineGraph.java | 11 +
.../shared/ReferencedConnectionSaveValidator.java | 117 ++++++++++
.../hopgui/file/workflow/HopGuiWorkflowGraph.java | 11 +
.../file/config/messages/messages_en_US.properties | 23 ++
25 files changed, 1158 insertions(+), 13 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/util/StringUtil.java
b/core/src/main/java/org/apache/hop/core/util/StringUtil.java
index 0668ec52c5..c7d1c584a5 100644
--- a/core/src/main/java/org/apache/hop/core/util/StringUtil.java
+++ b/core/src/main/java/org/apache/hop/core/util/StringUtil.java
@@ -579,6 +579,24 @@ public class StringUtil {
|| variable.startsWith(HEX_OPEN) && variable.endsWith(HEX_CLOSE);
}
+ /**
+ * Whether {@code value} still contains a Hop variable delimiter after
substitution. Used to skip
+ * design-time checks that cannot be decided when a name or URL still holds
{@code ${...}}, {@code
+ * %%...%%}, {@code $[...]} or {@code #{...}}.
+ *
+ * @param value the string to inspect, may be null
+ * @return true when a variable token is still present
+ */
+ public static boolean containsVariableToken(String value) {
+ if (value == null) {
+ return false;
+ }
+ return value.contains(UNIX_OPEN)
+ || value.contains(WINDOWS_OPEN)
+ || value.contains(HEX_OPEN)
+ || value.contains(RESOLVER_OPEN);
+ }
+
/**
* Calls the {@link String#toLowerCase()} method on the {@link String}
returned by a call to
* {@code obj.toString()}, guarding against {@link NullPointerException}s.
diff --git
a/core/src/main/java/org/apache/hop/metadata/util/HopMetadataPropertyWalker.java
b/core/src/main/java/org/apache/hop/metadata/util/HopMetadataPropertyWalker.java
new file mode 100644
index 0000000000..699166be99
--- /dev/null
+++
b/core/src/main/java/org/apache/hop/metadata/util/HopMetadataPropertyWalker.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.metadata.util;
+
+import java.lang.reflect.Array;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+
+/**
+ * Walks {@link HopMetadataProperty} fields on a metadata object, including
nested objects and
+ * collections, and collects string values of a given {@link
HopMetadataPropertyType}.
+ *
+ * <p>Failures to read a field are skipped. Cycles are broken. This is a
design-time helper: it must
+ * not throw because of a broken plugin class.
+ */
+public final class HopMetadataPropertyWalker {
+
+ private static final int MAX_DEPTH = 8;
+
+ private HopMetadataPropertyWalker() {}
+
+ /**
+ * A string property found on a metadata object.
+ *
+ * @param type the annotated property type
+ * @param key the serialised key, or the field name when no key is set
+ * @param value the raw (unresolved) string value, never null
+ */
+ public record StringProperty(HopMetadataPropertyType type, String key,
String value) {}
+
+ /**
+ * Collect every string field annotated with {@code type} under {@code root}.
+ *
+ * @param root the object to walk, may be null
+ * @param type the property type to collect
+ * @return the matching properties, possibly empty
+ */
+ public static List<StringProperty> collectStrings(Object root,
HopMetadataPropertyType type) {
+ List<StringProperty> collected = new ArrayList<>();
+ if (root == null || type == null) {
+ return collected;
+ }
+ walk(
+ root,
+ type,
+ collected,
+ 0,
+ java.util.Collections.newSetFromMap(new IdentityHashMap<Object,
Boolean>()));
+ return collected;
+ }
+
+ private static void walk(
+ Object node,
+ HopMetadataPropertyType type,
+ List<StringProperty> collected,
+ int depth,
+ Set<Object> visited) {
+ if (node == null || depth > MAX_DEPTH || !isMetadataObject(node) ||
!visited.add(node)) {
+ return;
+ }
+ for (Field field : ReflectionUtil.findAllFields(node.getClass())) {
+ if (Modifier.isStatic(field.getModifiers())) {
+ continue;
+ }
+ HopMetadataProperty property =
field.getAnnotation(HopMetadataProperty.class);
+ if (property == null) {
+ continue;
+ }
+ Object value = readField(field, node);
+ if (value == null) {
+ continue;
+ }
+ if (property.hopMetadataPropertyType() == type && value instanceof
String stringValue) {
+ collected.add(new StringProperty(type, serialisedKey(property, field),
stringValue));
+ }
+ descend(value, type, collected, depth, visited);
+ }
+ }
+
+ private static void descend(
+ Object value,
+ HopMetadataPropertyType type,
+ List<StringProperty> collected,
+ int depth,
+ Set<Object> visited) {
+ if (value instanceof Collection<?> collection) {
+ for (Object element : collection) {
+ walk(element, type, collected, depth + 1, visited);
+ }
+ return;
+ }
+ if (value instanceof Map<?, ?> map) {
+ for (Object element : map.values()) {
+ walk(element, type, collected, depth + 1, visited);
+ }
+ return;
+ }
+ if (value.getClass().isArray()) {
+ int length = Array.getLength(value);
+ for (int i = 0; i < length; i++) {
+ walk(Array.get(value, i), type, collected, depth + 1, visited);
+ }
+ return;
+ }
+ walk(value, type, collected, depth + 1, visited);
+ }
+
+ private static String serialisedKey(HopMetadataProperty property, Field
field) {
+ if (property.key() != null && !property.key().isEmpty()) {
+ return property.key();
+ }
+ return field.getName();
+ }
+
+ /** Only descends into Hop's own metadata classes, never into JDK or
third-party types. */
+ static boolean isMetadataObject(Object value) {
+ if (value == null) {
+ return false;
+ }
+ Class<?> type = value.getClass();
+ if (type.isPrimitive() || type.isEnum() || type.isArray()) {
+ return false;
+ }
+ Package pkg = type.getPackage();
+ return pkg != null &&
pkg.getName().toLowerCase(Locale.ROOT).startsWith("org.apache.hop");
+ }
+
+ private static Object readField(Field field, Object target) {
+ try {
+ field.setAccessible(true);
+ return field.get(target);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+}
diff --git a/core/src/test/java/org/apache/hop/core/util/StringUtilTest.java
b/core/src/test/java/org/apache/hop/core/util/StringUtilTest.java
index 41013a8a23..b030f117e7 100644
--- a/core/src/test/java/org/apache/hop/core/util/StringUtilTest.java
+++ b/core/src/test/java/org/apache/hop/core/util/StringUtilTest.java
@@ -162,6 +162,18 @@ class StringUtilTest {
assertFalse(StringUtil.isVariable(null));
}
+ @Test
+ void testContainsVariableToken() {
+ assertTrue(StringUtil.containsVariableToken("${CONNECTION}"));
+ assertTrue(StringUtil.containsVariableToken("db_${ENV}"));
+ assertTrue(StringUtil.containsVariableToken("%%WINDOWS%%"));
+ assertTrue(StringUtil.containsVariableToken("$[hex]"));
+ assertTrue(StringUtil.containsVariableToken("#{resolver}"));
+ assertFalse(StringUtil.containsVariableToken("sales-db"));
+ assertFalse(StringUtil.containsVariableToken(null));
+ assertFalse(StringUtil.containsVariableToken(""));
+ }
+
@Test
void testSafeToLowerCase() {
assertNull(StringUtil.safeToLowerCase(null));
diff --git
a/core/src/test/java/org/apache/hop/metadata/util/HopMetadataPropertyWalkerTest.java
b/core/src/test/java/org/apache/hop/metadata/util/HopMetadataPropertyWalkerTest.java
new file mode 100644
index 0000000000..f6a51c91a1
--- /dev/null
+++
b/core/src/test/java/org/apache/hop/metadata/util/HopMetadataPropertyWalkerTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.metadata.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.util.HopMetadataPropertyWalker.StringProperty;
+import org.junit.jupiter.api.Test;
+
+class HopMetadataPropertyWalkerTest {
+
+ static class SimpleMeta {
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.RDBMS_CONNECTION)
+ String connection = "warehouse";
+
+ @HopMetadataProperty(key = "sql")
+ String sql = "SELECT 1";
+
+ String unannotated = "ignored";
+ }
+
+ static class NestedItem {
+ @HopMetadataProperty(
+ key = "name",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
+ String name;
+
+ NestedItem(String name) {
+ this.name = name;
+ }
+ }
+
+ static class NestedMeta {
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.RDBMS_CONNECTION)
+ String connection = "primary";
+
+ @HopMetadataProperty
+ List<NestedItem> items = List.of(new NestedItem("second"), new
NestedItem("third"));
+ }
+
+ static class TwoConnectionsMeta {
+ @HopMetadataProperty(
+ key = "referenceConnection",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
+ String reference = "ref-db";
+
+ @HopMetadataProperty(
+ key = "compareConnection",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
+ String compare = "cmp-db";
+ }
+
+ static class UnannotatedConnectionMeta {
+ @HopMetadataProperty(key = "connection")
+ String connection = "hidden";
+ }
+
+ @Test
+ void collectsAnnotatedConnectionStrings() {
+ List<StringProperty> found =
+ HopMetadataPropertyWalker.collectStrings(
+ new SimpleMeta(), HopMetadataPropertyType.RDBMS_CONNECTION);
+
+ assertEquals(1, found.size());
+ assertEquals("connection", found.get(0).key());
+ assertEquals("warehouse", found.get(0).value());
+ }
+
+ @Test
+ void descendsIntoNestedLists() {
+ List<StringProperty> found =
+ HopMetadataPropertyWalker.collectStrings(
+ new NestedMeta(), HopMetadataPropertyType.RDBMS_CONNECTION);
+
+ assertEquals(3, found.size());
+ assertEquals(
+ List.of("primary", "second", "third"),
found.stream().map(StringProperty::value).toList());
+ }
+
+ @Test
+ void collectsTwoConnectionFieldsOnOneObject() {
+ List<StringProperty> found =
+ HopMetadataPropertyWalker.collectStrings(
+ new TwoConnectionsMeta(),
HopMetadataPropertyType.RDBMS_CONNECTION);
+
+ assertEquals(2, found.size());
+ assertTrue(found.stream().anyMatch(p ->
"referenceConnection".equals(p.key())));
+ assertTrue(found.stream().anyMatch(p ->
"compareConnection".equals(p.key())));
+ }
+
+ @Test
+ void ignoresConnectionFieldsWithoutThePropertyType() {
+ List<StringProperty> found =
+ HopMetadataPropertyWalker.collectStrings(
+ new UnannotatedConnectionMeta(),
HopMetadataPropertyType.RDBMS_CONNECTION);
+
+ assertTrue(found.isEmpty());
+ }
+
+ @Test
+ void nullRootYieldsNothing() {
+ assertTrue(
+ HopMetadataPropertyWalker.collectStrings(null,
HopMetadataPropertyType.RDBMS_CONNECTION)
+ .isEmpty());
+ }
+}
diff --git
a/docs/hop-user-manual/modules/ROOT/assets/images/hop-gui/configuration-perspective-file-validation-validate-database-connections-when-saving.png
b/docs/hop-user-manual/modules/ROOT/assets/images/hop-gui/configuration-perspective-file-validation-validate-database-connections-when-saving.png
new file mode 100644
index 0000000000..68426548e3
Binary files /dev/null and
b/docs/hop-user-manual/modules/ROOT/assets/images/hop-gui/configuration-perspective-file-validation-validate-database-connections-when-saving.png
differ
diff --git
a/docs/hop-user-manual/modules/ROOT/assets/images/hop-gui/validate-database-connections-on-save-warning-dialog.png
b/docs/hop-user-manual/modules/ROOT/assets/images/hop-gui/validate-database-connections-on-save-warning-dialog.png
new file mode 100644
index 0000000000..5b8457d1f8
Binary files /dev/null and
b/docs/hop-user-manual/modules/ROOT/assets/images/hop-gui/validate-database-connections-on-save-warning-dialog.png
differ
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/hop-gui/perspective-configuration.adoc
b/docs/hop-user-manual/modules/ROOT/pages/hop-gui/perspective-configuration.adoc
index b9c2504fe6..1809db1faa 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/hop-gui/perspective-configuration.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/hop-gui/perspective-configuration.adoc
@@ -138,6 +138,13 @@
image::hop-gui/configuration-perspective-open-help-pages-in.png[Open help pages
* xref:vfs/google-drive-vfs.adoc#_configuration[Google Drive] VFS
configuration options.
* xref:projects/index.adoc[Project] configuration options
* Welcome Dialog: specify whether to show or hide the welcome dialog when Hop
GUI starts.
+* **File validation**: when enabled (the default), saving a pipeline or
workflow warns if a transform or action references a relational database
connection that is not in the project metadata. You can still save. Connection
names that still contain a variable such as `'${CONNECTION}'` after the current
environment is applied are skipped, because the name cannot be decided at
design time. The same check also runs when you Verify a pipeline or workflow.
Hop does not try to open a JDBC conn [...]
++
+image::hop-gui/configuration-perspective-file-validation-validate-database-connections-when-saving.png[Validate
database connections when saving,width="90%"]
++
+The warning lists the missing connections. *Yes* saves anyway, *No* cancels
the save. Check *Don't run this check when saving* to turn the option off; you
can turn it back on here.
++
+image::hop-gui/validate-database-connections-on-save-warning-dialog.png[Database
connections save warning,width="90%"]
=== System Variables
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rdbms-connection.adoc
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rdbms-connection.adoc
index 65d0434959..65dc288261 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rdbms-connection.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rdbms-connection.adoc
@@ -30,6 +30,10 @@ Hop supports tens of relational databases out of the box. If
your specific datab
Check the list of xref:database/databases.adoc[databases] for more details.
+Transforms and actions store the *name* of a relational database connection.
When you save or Verify a pipeline or workflow, Hop warns if that name is not
in the project metadata. You can still save. If the name still contains a
variable such as `'${CONNECTION}'` after the current environment is applied,
the check is skipped because the name cannot be decided at design time. This
does not open a JDBC connection; it only looks the name up in metadata. The
warning dialog can turn the check [...]
+
+image::hop-gui/validate-database-connections-on-save-warning-dialog.png[Database
connections save warning,width="90%"]
+
== Related Plugins
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/create-pipeline.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/create-pipeline.adoc
index dedd4e9825..cb435efbf1 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/create-pipeline.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/create-pipeline.adoc
@@ -61,6 +61,10 @@ When you are finished with your pipeline, save it.
This can be done via the File menu, the icons or using CTLR s or Command s.
For new pipelines a file browser is displayed to navigate towards the location
you want to store the file.
+By default Hop warns if a transform references a relational database
connection that is not in the project metadata. You can still save. Connection
names that still contain a variable such as `'${CONNECTION}'` are skipped.
Check *Don't run this check when saving* on the dialog, or turn the option off
under xref:hop-gui/perspective-configuration.adoc[Configuration] → Plugins →
File validation.
+
+image::hop-gui/validate-database-connections-on-save-warning-dialog.png[Database
connections save warning,width="90%"]
+
== Add Transform to your pipelines
Click anywhere in the pipeline canvas, the area where you'll see the image
below.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/workflow/create-workflow.adoc
b/docs/hop-user-manual/modules/ROOT/pages/workflow/create-workflow.adoc
index cef658ac67..65af6017fa 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/workflow/create-workflow.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/workflow/create-workflow.adoc
@@ -58,6 +58,10 @@ When you are finished with your workflow, save it.
This can be done via the File menu, the icons or using CTLR s or Command s.
For new workflows a file browser is displayed to navigate towards the location
you want to store the file.
+By default Hop warns if an action references a relational database connection
that is not in the project metadata. You can still save. Connection names that
still contain a variable such as `'${CONNECTION}'` are skipped. Check *Don't
run this check when saving* on the dialog, or turn the option off under
xref:hop-gui/perspective-configuration.adoc[Configuration] → Plugins → File
validation.
+
+image::hop-gui/validate-database-connections-on-save-warning-dialog.png[Database
connections save warning,width="90%"]
+
== Add Action to your workflow
Add the following actions to your workflow and create the hops to connect them:
diff --git
a/engine/src/main/java/org/apache/hop/metadata/validation/ReferencedDatabaseConnectionChecker.java
b/engine/src/main/java/org/apache/hop/metadata/validation/ReferencedDatabaseConnectionChecker.java
new file mode 100644
index 0000000000..2d451f3e01
--- /dev/null
+++
b/engine/src/main/java/org/apache/hop/metadata/validation/ReferencedDatabaseConnectionChecker.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.metadata.validation;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.CheckResult;
+import org.apache.hop.core.ICheckResult;
+import org.apache.hop.core.ICheckResultSource;
+import org.apache.hop.core.database.DatabaseMeta;
+import org.apache.hop.core.util.StringUtil;
+import org.apache.hop.core.util.Utils;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.util.HopMetadataPropertyWalker;
+import org.apache.hop.metadata.util.HopMetadataPropertyWalker.StringProperty;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.workflow.WorkflowMeta;
+import org.apache.hop.workflow.action.ActionMeta;
+
+/**
+ * Warns when a transform or action references a relational database
connection that is not in
+ * project metadata.
+ *
+ * <p>This is an existence check only. It never opens a JDBC connection. Names
that still contain a
+ * variable token after resolving the current {@link IVariables} are skipped,
because the name
+ * cannot be decided at design time.
+ */
+public final class ReferencedDatabaseConnectionChecker {
+
+ public static final String ERROR_NOT_ASSIGNED = "CONNECTION_NOT_ASSIGNED";
+ public static final String ERROR_DOES_NOT_EXIST =
"CONNECTION_DOES_NOT_EXIST";
+
+ private static final Class<?> PKG =
ReferencedDatabaseConnectionChecker.class;
+
+ private ReferencedDatabaseConnectionChecker() {}
+
+ public static List<ICheckResult> checkPipeline(
+ PipelineMeta pipelineMeta, IVariables variables, IHopMetadataProvider
metadataProvider) {
+ List<ICheckResult> remarks = new ArrayList<>();
+ if (pipelineMeta == null) {
+ return remarks;
+ }
+ for (TransformMeta transformMeta : pipelineMeta.getTransforms()) {
+ remarks.addAll(checkTransform(transformMeta, variables,
metadataProvider));
+ }
+ return remarks;
+ }
+
+ public static List<ICheckResult> checkWorkflow(
+ WorkflowMeta workflowMeta, IVariables variables, IHopMetadataProvider
metadataProvider) {
+ List<ICheckResult> remarks = new ArrayList<>();
+ if (workflowMeta == null) {
+ return remarks;
+ }
+ for (ActionMeta actionMeta : workflowMeta.getActions()) {
+ remarks.addAll(checkAction(actionMeta, variables, metadataProvider));
+ }
+ return remarks;
+ }
+
+ public static List<ICheckResult> checkTransform(
+ TransformMeta transformMeta, IVariables variables, IHopMetadataProvider
metadataProvider) {
+ if (transformMeta == null || transformMeta.getTransform() == null) {
+ return List.of();
+ }
+ return checkObject(
+ transformMeta.getTransform(),
+ BaseMessages.getString(PKG,
"ReferencedDatabaseConnectionChecker.Kind.Transform"),
+ transformMeta.getName(),
+ transformMeta,
+ variables,
+ metadataProvider);
+ }
+
+ public static List<ICheckResult> checkAction(
+ ActionMeta actionMeta, IVariables variables, IHopMetadataProvider
metadataProvider) {
+ if (actionMeta == null || actionMeta.getAction() == null) {
+ return List.of();
+ }
+ return checkObject(
+ actionMeta.getAction(),
+ BaseMessages.getString(PKG,
"ReferencedDatabaseConnectionChecker.Kind.Action"),
+ actionMeta.getName(),
+ actionMeta.getAction(),
+ variables,
+ metadataProvider);
+ }
+
+ /**
+ * Check one metadata object for {@link
HopMetadataPropertyType#RDBMS_CONNECTION} fields.
+ *
+ * @param metadataObject the transform or action metadata, or a nested POJO
used in tests
+ * @param ownerKind "Transform" or "Action" (already translated)
+ * @param ownerName the transform or action name
+ * @param source the check-result source, may be null
+ */
+ public static List<ICheckResult> checkObject(
+ Object metadataObject,
+ String ownerKind,
+ String ownerName,
+ ICheckResultSource source,
+ IVariables variables,
+ IHopMetadataProvider metadataProvider) {
+ List<ICheckResult> remarks = new ArrayList<>();
+ if (metadataObject == null || metadataProvider == null) {
+ return remarks;
+ }
+
+ IHopMetadataSerializer<DatabaseMeta> serializer;
+ try {
+ serializer = metadataProvider.getSerializer(DatabaseMeta.class);
+ } catch (Exception e) {
+ return remarks;
+ }
+ if (serializer == null) {
+ return remarks;
+ }
+
+ for (StringProperty property :
+ HopMetadataPropertyWalker.collectStrings(
+ metadataObject, HopMetadataPropertyType.RDBMS_CONNECTION)) {
+ ICheckResult remark =
+ checkConnectionName(
+ property.value(), ownerKind, ownerName, source, variables,
serializer);
+ if (remark != null) {
+ remarks.add(remark);
+ }
+ }
+ return remarks;
+ }
+
+ private static ICheckResult checkConnectionName(
+ String rawName,
+ String ownerKind,
+ String ownerName,
+ ICheckResultSource source,
+ IVariables variables,
+ IHopMetadataSerializer<DatabaseMeta> serializer) {
+ if (Utils.isEmpty(rawName)) {
+ return new CheckResult(
+ ICheckResult.TYPE_RESULT_WARNING,
+ ERROR_NOT_ASSIGNED,
+ BaseMessages.getString(
+ PKG, "ReferencedDatabaseConnectionChecker.NotAssigned",
ownerKind, ownerName),
+ source);
+ }
+
+ String resolved = variables != null ? variables.resolve(rawName) : rawName;
+ if (Utils.isEmpty(resolved)) {
+ return new CheckResult(
+ ICheckResult.TYPE_RESULT_WARNING,
+ ERROR_NOT_ASSIGNED,
+ BaseMessages.getString(
+ PKG, "ReferencedDatabaseConnectionChecker.NotAssigned",
ownerKind, ownerName),
+ source);
+ }
+ if (StringUtil.containsVariableToken(resolved)) {
+ return null;
+ }
+
+ try {
+ if (serializer.exists(resolved)) {
+ return null;
+ }
+ } catch (Exception e) {
+ return new CheckResult(
+ ICheckResult.TYPE_RESULT_WARNING,
+ ERROR_DOES_NOT_EXIST,
+ BaseMessages.getString(
+ PKG,
+ "ReferencedDatabaseConnectionChecker.DoesNotExist",
+ resolved,
+ ownerKind,
+ ownerName),
+ source);
+ }
+
+ return new CheckResult(
+ ICheckResult.TYPE_RESULT_WARNING,
+ ERROR_DOES_NOT_EXIST,
+ BaseMessages.getString(
+ PKG,
+ "ReferencedDatabaseConnectionChecker.DoesNotExist",
+ resolved,
+ ownerKind,
+ ownerName),
+ source);
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/pipeline/PipelineMeta.java
b/engine/src/main/java/org/apache/hop/pipeline/PipelineMeta.java
index 7db7e7d582..f0f0834355 100644
--- a/engine/src/main/java/org/apache/hop/pipeline/PipelineMeta.java
+++ b/engine/src/main/java/org/apache/hop/pipeline/PipelineMeta.java
@@ -77,6 +77,7 @@ import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil;
+import org.apache.hop.metadata.validation.ReferencedDatabaseConnectionChecker;
import org.apache.hop.partition.PartitionSchema;
import org.apache.hop.pipeline.analysis.BufferDeadlockRisk;
import org.apache.hop.pipeline.analysis.PipelineBufferDeadlockAnalyzer;
@@ -2818,6 +2819,12 @@ public class PipelineMeta extends AbstractMeta
risk.reconvergence()));
}
+ for (TransformMeta transformMeta : transformsToCheck) {
+ remarks.addAll(
+ ReferencedDatabaseConnectionChecker.checkTransform(
+ transformMeta, variables, metadataProvider));
+ }
+
ExtensionPointHandler.callExtensionPoint(
LogChannel.GENERAL,
variables,
diff --git a/engine/src/main/java/org/apache/hop/workflow/WorkflowMeta.java
b/engine/src/main/java/org/apache/hop/workflow/WorkflowMeta.java
index e9d8ae417b..82ab7eb53c 100644
--- a/engine/src/main/java/org/apache/hop/workflow/WorkflowMeta.java
+++ b/engine/src/main/java/org/apache/hop/workflow/WorkflowMeta.java
@@ -64,6 +64,7 @@ import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil;
+import org.apache.hop.metadata.validation.ReferencedDatabaseConnectionChecker;
import org.apache.hop.resource.IResourceExport;
import org.apache.hop.resource.IResourceNaming;
import org.apache.hop.resource.ResourceDefinition;
@@ -1593,6 +1594,9 @@ public class WorkflowMeta extends AbstractMeta
if (action != null) {
checkAction(remarks, monitor, variables, metadataProvider,
actionMeta, action);
}
+ remarks.addAll(
+ ReferencedDatabaseConnectionChecker.checkAction(
+ actionMeta, variables, metadataProvider));
// Progress bar...
monitor.worked(worked++);
}
diff --git
a/engine/src/main/resources/org/apache/hop/metadata/validation/messages/messages_en_US.properties
b/engine/src/main/resources/org/apache/hop/metadata/validation/messages/messages_en_US.properties
new file mode 100644
index 0000000000..73b2d5a586
--- /dev/null
+++
b/engine/src/main/resources/org/apache/hop/metadata/validation/messages/messages_en_US.properties
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+ReferencedDatabaseConnectionChecker.Kind.Transform=Transform
+ReferencedDatabaseConnectionChecker.Kind.Action=Action
+ReferencedDatabaseConnectionChecker.NotAssigned=No database connection is
assigned on {0} ''{1}''
+ReferencedDatabaseConnectionChecker.DoesNotExist=Database connection ''{0}''
assigned on {1} ''{2}'' does not exist
diff --git
a/engine/src/test/java/org/apache/hop/metadata/validation/ReferencedDatabaseConnectionCheckerTest.java
b/engine/src/test/java/org/apache/hop/metadata/validation/ReferencedDatabaseConnectionCheckerTest.java
new file mode 100644
index 0000000000..590c6d2d4b
--- /dev/null
+++
b/engine/src/test/java/org/apache/hop/metadata/validation/ReferencedDatabaseConnectionCheckerTest.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.metadata.validation;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import org.apache.hop.core.ICheckResult;
+import org.apache.hop.core.database.DatabaseMeta;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.dummy.DummyMeta;
+import org.apache.hop.workflow.WorkflowMeta;
+import org.apache.hop.workflow.action.ActionBase;
+import org.apache.hop.workflow.action.ActionMeta;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class ReferencedDatabaseConnectionCheckerTest {
+
+ private IHopMetadataProvider metadataProvider;
+ private Variables variables;
+
+ @BeforeEach
+ @SuppressWarnings("unchecked")
+ void setUp() throws Exception {
+ variables = new Variables();
+ metadataProvider = mock(IHopMetadataProvider.class);
+ IHopMetadataSerializer<DatabaseMeta> serializer =
mock(IHopMetadataSerializer.class);
+
when(metadataProvider.getSerializer(DatabaseMeta.class)).thenReturn(serializer);
+ when(serializer.exists(anyString())).thenAnswer(inv ->
"sales-db".equals(inv.getArgument(0)));
+ }
+
+ static class ConnMeta {
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.RDBMS_CONNECTION)
+ String connection;
+
+ ConnMeta(String connection) {
+ this.connection = connection;
+ }
+ }
+
+ static class NestedItem {
+ @HopMetadataProperty(
+ key = "name",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
+ String name;
+
+ NestedItem(String name) {
+ this.name = name;
+ }
+ }
+
+ static class NestedMeta {
+ @HopMetadataProperty List<NestedItem> items;
+
+ NestedMeta(String... names) {
+ items = java.util.Arrays.stream(names).map(NestedItem::new).toList();
+ }
+ }
+
+ static class TwoConnectionsMeta {
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.RDBMS_CONNECTION)
+ String referenceConnection;
+
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.RDBMS_CONNECTION)
+ String compareConnection;
+ }
+
+ static class UnannotatedMeta {
+ @HopMetadataProperty(key = "connection")
+ String connection = "missing";
+ }
+
+ static class TestAction extends ActionBase {
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.RDBMS_CONNECTION)
+ String connection;
+
+ TestAction(String name, String connection) {
+ super(name, "");
+ this.connection = connection;
+ }
+
+ @Override
+ public org.apache.hop.core.Result execute(org.apache.hop.core.Result
prevResult, int nr) {
+ return prevResult;
+ }
+ }
+
+ private List<ICheckResult> check(Object meta, String ownerName) {
+ return ReferencedDatabaseConnectionChecker.checkObject(
+ meta, "Transform", ownerName, null, variables, metadataProvider);
+ }
+
+ @Test
+ void missingLiteralNameIsAWarning() {
+ List<ICheckResult> remarks = check(new ConnMeta("missing-db"), "Read
sales");
+
+ assertEquals(1, remarks.size());
+ assertEquals(
+ ReferencedDatabaseConnectionChecker.ERROR_DOES_NOT_EXIST,
remarks.get(0).getErrorCode());
+ assertTrue(remarks.get(0).getText().contains("missing-db"));
+ assertTrue(remarks.get(0).getText().contains("Read sales"));
+ }
+
+ @Test
+ void existingLiteralNameIsSilent() {
+ assertTrue(check(new ConnMeta("sales-db"), "Read sales").isEmpty());
+ }
+
+ @Test
+ void unresolvedVariableIsSkipped() {
+ assertTrue(check(new ConnMeta("${CONNECTION}"), "Read sales").isEmpty());
+ }
+
+ @Test
+ void resolvedVariableToExistingNameIsSilent() {
+ variables.setVariable("CONNECTION", "sales-db");
+ assertTrue(check(new ConnMeta("${CONNECTION}"), "Read sales").isEmpty());
+ }
+
+ @Test
+ void resolvedVariableToMissingNameWarns() {
+ variables.setVariable("CONNECTION", "missing-db");
+ List<ICheckResult> remarks = check(new ConnMeta("${CONNECTION}"), "Read
sales");
+
+ assertEquals(1, remarks.size());
+ assertEquals(
+ ReferencedDatabaseConnectionChecker.ERROR_DOES_NOT_EXIST,
remarks.get(0).getErrorCode());
+ assertTrue(remarks.get(0).getText().contains("missing-db"));
+ }
+
+ @Test
+ void mixedUnresolvedVariableIsSkipped() {
+ assertTrue(check(new ConnMeta("db_${ENV}"), "Read sales").isEmpty());
+ }
+
+ @Test
+ void emptyConnectionIsAWarning() {
+ List<ICheckResult> remarks = check(new ConnMeta(""), "Read sales");
+
+ assertEquals(1, remarks.size());
+ assertEquals(
+ ReferencedDatabaseConnectionChecker.ERROR_NOT_ASSIGNED,
remarks.get(0).getErrorCode());
+ }
+
+ @Test
+ void nestedListConnectionsAreChecked() {
+ List<ICheckResult> remarks =
+ check(new NestedMeta("sales-db", "missing-db"), "Check connections");
+
+ assertEquals(1, remarks.size());
+ assertTrue(remarks.get(0).getText().contains("missing-db"));
+ }
+
+ @Test
+ void twoConnectionFieldsOnOneObject() {
+ TwoConnectionsMeta meta = new TwoConnectionsMeta();
+ meta.referenceConnection = "sales-db";
+ meta.compareConnection = "other-db";
+
+ List<ICheckResult> remarks = check(meta, "Compare tables");
+
+ assertEquals(1, remarks.size());
+ assertTrue(remarks.get(0).getText().contains("other-db"));
+ }
+
+ @Test
+ void unannotatedConnectionFieldIsIgnored() {
+ assertTrue(check(new UnannotatedMeta(), "Legacy").isEmpty());
+ }
+
+ @Test
+ void checksAPipelineTransform() {
+ ConnTransformMeta transform = new ConnTransformMeta();
+ transform.connection = "missing-db";
+ TransformMeta transformMeta = new TransformMeta("Read sales", transform);
+
+ PipelineMeta pipelineMeta = new PipelineMeta();
+ pipelineMeta.addTransform(transformMeta);
+
+ List<ICheckResult> remarks =
+ ReferencedDatabaseConnectionChecker.checkPipeline(
+ pipelineMeta, variables, metadataProvider);
+
+ assertEquals(1, remarks.size());
+ assertEquals(transformMeta, remarks.get(0).getSourceInfo());
+ }
+
+ @Test
+ void checksAWorkflowAction() {
+ TestAction action = new TestAction("Run SQL", "missing-db");
+ ActionMeta actionMeta = new ActionMeta(action);
+
+ WorkflowMeta workflowMeta = new WorkflowMeta();
+ workflowMeta.addAction(actionMeta);
+
+ List<ICheckResult> remarks =
+ ReferencedDatabaseConnectionChecker.checkWorkflow(
+ workflowMeta, variables, metadataProvider);
+
+ assertEquals(1, remarks.size());
+ assertEquals(
+ ReferencedDatabaseConnectionChecker.ERROR_DOES_NOT_EXIST,
remarks.get(0).getErrorCode());
+ }
+
+ /** A Dummy transform that also carries an annotated connection name, for
pipeline-level tests. */
+ static class ConnTransformMeta extends DummyMeta {
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.RDBMS_CONNECTION)
+ String connection;
+ }
+}
diff --git
a/plugins/actions/waitforsql/src/main/java/org/apache/hop/workflow/actions/waitforsql/ActionWaitForSql.java
b/plugins/actions/waitforsql/src/main/java/org/apache/hop/workflow/actions/waitforsql/ActionWaitForSql.java
index 10a7624a08..147c22f935 100644
---
a/plugins/actions/waitforsql/src/main/java/org/apache/hop/workflow/actions/waitforsql/ActionWaitForSql.java
+++
b/plugins/actions/waitforsql/src/main/java/org/apache/hop/workflow/actions/waitforsql/ActionWaitForSql.java
@@ -33,6 +33,7 @@ import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IEnumHasCode;
import org.apache.hop.metadata.api.IEnumHasCodeAndDescription;
import org.apache.hop.metadata.api.IHopMetadataProvider;
@@ -72,13 +73,19 @@ public class ActionWaitForSql extends ActionBase implements
Cloneable, IAction {
@HopMetadataProperty(key = "custom_sql")
private String customSql;
- @HopMetadataProperty(key = "connection")
+ @HopMetadataProperty(
+ key = "connection",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
private String connection;
- @HopMetadataProperty(key = "tablename")
+ @HopMetadataProperty(
+ key = "tablename",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_TABLE)
private String tableName;
- @HopMetadataProperty(key = "schemaname")
+ @HopMetadataProperty(
+ key = "schemaname",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_SCHEMA)
private String schemaName;
/** Maximum timeout in seconds */
diff --git
a/plugins/databases/mssqlnative/src/main/java/org/apache/hop/workflow/actions/mssqlbulkload/ActionMssqlBulkLoad.java
b/plugins/databases/mssqlnative/src/main/java/org/apache/hop/workflow/actions/mssqlbulkload/ActionMssqlBulkLoad.java
index fb7bae774c..9d952f0252 100644
---
a/plugins/databases/mssqlnative/src/main/java/org/apache/hop/workflow/actions/mssqlbulkload/ActionMssqlBulkLoad.java
+++
b/plugins/databases/mssqlnative/src/main/java/org/apache/hop/workflow/actions/mssqlbulkload/ActionMssqlBulkLoad.java
@@ -39,6 +39,7 @@ import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.resource.ResourceEntry;
import org.apache.hop.resource.ResourceEntry.ResourceType;
@@ -65,10 +66,14 @@ import
org.apache.hop.workflow.action.validator.ValidatorContext;
public class ActionMssqlBulkLoad extends ActionBase {
private static final Class<?> PKG = ActionMssqlBulkLoad.class;
- @HopMetadataProperty(key = "schemaname")
+ @HopMetadataProperty(
+ key = "schemaname",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_SCHEMA)
private String schemaName;
- @HopMetadataProperty(key = "tablename")
+ @HopMetadataProperty(
+ key = "tablename",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_TABLE)
private String tableName;
@HopMetadataProperty(key = "filename")
@@ -140,7 +145,9 @@ public class ActionMssqlBulkLoad extends ActionBase {
@HopMetadataProperty(key = "truncate")
private boolean truncate;
- @HopMetadataProperty(key = "connection")
+ @HopMetadataProperty(
+ key = "connection",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
private String connection;
public ActionMssqlBulkLoad(String n) {
diff --git
a/plugins/databases/mysql/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoad.java
b/plugins/databases/mysql/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoad.java
index 513f896eed..9a7f62668e 100644
---
a/plugins/databases/mysql/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoad.java
+++
b/plugins/databases/mysql/src/main/java/org/apache/hop/workflow/actions/mysqlbulkload/ActionMysqlBulkLoad.java
@@ -38,6 +38,7 @@ import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.resource.ResourceEntry;
import org.apache.hop.resource.ResourceEntry.ResourceType;
@@ -65,10 +66,14 @@ import
org.apache.hop.workflow.action.validator.ValidatorContext;
public class ActionMysqlBulkLoad extends ActionBase {
private static final Class<?> PKG = ActionMysqlBulkLoad.class;
- @HopMetadataProperty(key = "schemaname")
+ @HopMetadataProperty(
+ key = "schemaname",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_SCHEMA)
private String schemaName;
- @HopMetadataProperty(key = "tablename")
+ @HopMetadataProperty(
+ key = "tablename",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_TABLE)
private String tableName;
@HopMetadataProperty(key = "filename")
@@ -107,7 +112,9 @@ public class ActionMysqlBulkLoad extends ActionBase {
@HopMetadataProperty(key = "addfiletoresult")
private boolean addFileToResult;
- @HopMetadataProperty(key = "connection")
+ @HopMetadataProperty(
+ key = "connection",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
private String connection;
public ActionMysqlBulkLoad(String n) {
diff --git
a/plugins/databases/snowflake/src/main/java/org/apache/hop/workflow/actions/snowflake/WarehouseManager.java
b/plugins/databases/snowflake/src/main/java/org/apache/hop/workflow/actions/snowflake/WarehouseManager.java
index 0442a4c64d..e0ff861075 100644
---
a/plugins/databases/snowflake/src/main/java/org/apache/hop/workflow/actions/snowflake/WarehouseManager.java
+++
b/plugins/databases/snowflake/src/main/java/org/apache/hop/workflow/actions/snowflake/WarehouseManager.java
@@ -36,6 +36,7 @@ import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.workflow.WorkflowMeta;
import org.apache.hop.workflow.action.ActionBase;
@@ -96,7 +97,9 @@ public class WarehouseManager extends ActionBase implements
Cloneable, IAction {
public static final String CONST_COMMIT = ";\ncommit;";
/** The database to connect to. */
- @HopMetadataProperty(key = CONNECTION)
+ @HopMetadataProperty(
+ key = CONNECTION,
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
private String connection;
/** The management action to perform. */
diff --git
a/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMeta.java
b/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMeta.java
index 477759cb6c..0365295844 100644
---
a/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMeta.java
+++
b/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMeta.java
@@ -42,6 +42,7 @@ import org.apache.hop.core.vfs.HopVfs;
import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.DatabaseImpact;
import org.apache.hop.pipeline.PipelineMeta;
@@ -66,13 +67,17 @@ public class SQLFileOutputMeta extends
BaseTransformMeta<SQLFileOutput, SQLFileO
private static final String CONST_SPACE = " ";
private static final String CONST_SPACE_SHORT = " ";
- @HopMetadataProperty(key = "connection")
+ @HopMetadataProperty(
+ key = "connection",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_CONNECTION)
private String connection;
- @HopMetadataProperty(key = "schema")
+ @HopMetadataProperty(
+ key = "schema",
+ hopMetadataPropertyType = HopMetadataPropertyType.RDBMS_SCHEMA)
private String schemaName;
- @HopMetadataProperty(key = "table")
+ @HopMetadataProperty(key = "table", hopMetadataPropertyType =
HopMetadataPropertyType.RDBMS_TABLE)
private String tableName;
@HopMetadataProperty(key = "truncate")
diff --git
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/config/FileValidationConfigPlugin.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/config/FileValidationConfigPlugin.java
new file mode 100644
index 0000000000..0ce254db6c
--- /dev/null
+++
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/config/FileValidationConfigPlugin.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.ui.hopgui.file.config;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.hop.core.config.HopConfig;
+import org.apache.hop.core.config.plugin.ConfigPlugin;
+import org.apache.hop.core.config.plugin.IConfigOptions;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.gui.plugin.GuiElementType;
+import org.apache.hop.core.gui.plugin.GuiPlugin;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHasHopMetadataProvider;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
+import org.apache.hop.ui.core.gui.IGuiPluginCompositeWidgetsListener;
+import
org.apache.hop.ui.hopgui.perspective.configuration.tabs.ConfigPluginOptionsTab;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Control;
+import picocli.CommandLine;
+
+/**
+ * Options that run when a pipeline or workflow file is saved. Surfaced in the
Configuration
+ * perspective Plugins tab.
+ */
+@ConfigPlugin(
+ id = "file-validation-config",
+ description = "Validate referenced database connections when saving a
file",
+ category = ConfigPlugin.CATEGORY_CONFIG)
+@GuiPlugin(description = "i18n::FileValidationConfigPlugin.Description")
+public class FileValidationConfigPlugin
+ implements IConfigOptions, IGuiPluginCompositeWidgetsListener {
+
+ public static final String KEY_VALIDATE_DB_CONNECTIONS_ON_SAVE =
"ValidateDbConnectionsOnSave";
+
+ private static final String WIDGET_VALIDATE_DB_CONNECTIONS_ON_SAVE =
+ "file-validation-validate-db-connections-on-save";
+
+ @GuiWidgetElement(
+ id = WIDGET_VALIDATE_DB_CONNECTIONS_ON_SAVE,
+ parentId = ConfigPluginOptionsTab.GUI_WIDGETS_PARENT_ID,
+ type = GuiElementType.CHECKBOX,
+ label =
"i18n::FileValidationConfigPlugin.ValidateDbConnectionsOnSave.Label",
+ toolTip =
"i18n::FileValidationConfigPlugin.ValidateDbConnectionsOnSave.ToolTip")
+ @CommandLine.Option(
+ names = {"--validate-db-connections-on-save"},
+ description =
+ "Warn when saving a pipeline or workflow if a referenced database
connection does not exist (default: true)")
+ private Boolean validateDbConnectionsOnSave;
+
+ public FileValidationConfigPlugin() {
+ loadFromHopConfig();
+ }
+
+ public static FileValidationConfigPlugin getInstance() {
+ return new FileValidationConfigPlugin();
+ }
+
+ private void loadFromHopConfig() {
+ try {
+ validateDbConnectionsOnSave =
+ HopConfig.readOptionBoolean(KEY_VALIDATE_DB_CONNECTIONS_ON_SAVE,
true);
+ } catch (Exception e) {
+ validateDbConnectionsOnSave = true;
+ }
+ }
+
+ /**
+ * JavaBeans getter used by {@code GuiCompositeWidgets}. Null (never
persisted) is treated as
+ * enabled, which is the default.
+ */
+ public Boolean getValidateDbConnectionsOnSave() {
+ return isValidateDbConnectionsOnSave();
+ }
+
+ /** JavaBeans setter used by {@code GuiCompositeWidgets}. */
+ public void setValidateDbConnectionsOnSave(Boolean
validateDbConnectionsOnSave) {
+ this.validateDbConnectionsOnSave = validateDbConnectionsOnSave;
+ }
+
+ public boolean isValidateDbConnectionsOnSave() {
+ return validateDbConnectionsOnSave == null || validateDbConnectionsOnSave;
+ }
+
+ @Override
+ public void widgetsCreated(GuiCompositeWidgets compositeWidgets) {
+ // Widgets are filled from this instance, loaded in the constructor.
+ }
+
+ @Override
+ public void widgetsPopulated(GuiCompositeWidgets compositeWidgets) {
+ // Nothing to do.
+ }
+
+ @Override
+ public void widgetModified(
+ GuiCompositeWidgets compositeWidgets, Control changedWidget, String
widgetId) {
+ persistContents(compositeWidgets);
+ }
+
+ @Override
+ public void persistContents(GuiCompositeWidgets compositeWidgets) {
+ Control control =
compositeWidgets.getWidgetsMap().get(WIDGET_VALIDATE_DB_CONNECTIONS_ON_SAVE);
+ if (control instanceof Button button) {
+ validateDbConnectionsOnSave = button.getSelection();
+ }
+ saveToHopConfig();
+ }
+
+ public Map<String, Object> saveToHopConfig() {
+ Map<String, Object> options = new HashMap<>();
+ if (validateDbConnectionsOnSave != null) {
+ options.put(KEY_VALIDATE_DB_CONNECTIONS_ON_SAVE,
validateDbConnectionsOnSave);
+ HopConfig.saveOptions(options);
+ }
+ return options;
+ }
+
+ @Override
+ public boolean handleOption(
+ ILogChannel log, IHasHopMetadataProvider metadataProvider, IVariables
variables)
+ throws HopException {
+ if (validateDbConnectionsOnSave == null) {
+ return false;
+ }
+ saveToHopConfig();
+ log.logBasic(
+ "Validate database connections on save is now "
+ + (isValidateDbConnectionsOnSave() ? "enabled" : "disabled"));
+ return true;
+ }
+}
diff --git
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
index a97f718051..31bdae5841 100644
---
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
+++
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/pipeline/HopGuiPipelineGraph.java
@@ -109,6 +109,7 @@ import org.apache.hop.execution.IExecutionInfoLocation;
import org.apache.hop.history.AuditManager;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.laf.BasePropertyHandler;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.api.IHopMetadataSerializer;
import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
import org.apache.hop.pipeline.DatabaseImpact;
@@ -200,6 +201,7 @@ import
org.apache.hop.ui.hopgui.file.shared.HopGuiGraphSnapshotUndo;
import org.apache.hop.ui.hopgui.file.shared.HopGuiTooltipExtension;
import org.apache.hop.ui.hopgui.file.shared.ISnapshotUndoSupport;
import org.apache.hop.ui.hopgui.file.shared.PipelineRowSamplerHelper;
+import org.apache.hop.ui.hopgui.file.shared.ReferencedConnectionSaveValidator;
import org.apache.hop.ui.hopgui.palette.GraphPalette;
import org.apache.hop.ui.hopgui.palette.GraphPaletteTree;
import org.apache.hop.ui.hopgui.palette.IGraphPaletteHost;
@@ -5141,6 +5143,15 @@ public class HopGuiPipelineGraph extends
HopGuiAbstractGraph
throw new HopException("No filename: please specify a filename for
this pipeline");
}
+ IHopMetadataProvider saveMetadataProvider =
pipelineMeta.getMetadataProvider();
+ if (saveMetadataProvider == null) {
+ saveMetadataProvider = hopGui.getMetadataProvider();
+ }
+ if (!ReferencedConnectionSaveValidator.confirmSave(
+ hopShell(), pipelineMeta, variables, saveMetadataProvider)) {
+ return;
+ }
+
// Keep track of save
//
AuditManager.registerEvent(
diff --git
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/ReferencedConnectionSaveValidator.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/ReferencedConnectionSaveValidator.java
new file mode 100644
index 0000000000..1fdd52b5df
--- /dev/null
+++
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/shared/ReferencedConnectionSaveValidator.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.ui.hopgui.file.shared;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.hop.core.ICheckResult;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.validation.ReferencedDatabaseConnectionChecker;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.ui.core.dialog.MessageDialogWithToggle;
+import org.apache.hop.ui.hopgui.file.config.FileValidationConfigPlugin;
+import org.apache.hop.workflow.WorkflowMeta;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.widgets.Shell;
+
+/**
+ * Optional warning when saving a pipeline or workflow that references a
missing database
+ * connection. Saving is never blocked: the user can continue or cancel.
+ */
+public final class ReferencedConnectionSaveValidator {
+
+ private static final Class<?> PKG = FileValidationConfigPlugin.class;
+ private static final int MAX_LISTED = 15;
+
+ private ReferencedConnectionSaveValidator() {}
+
+ public static boolean confirmSave(
+ Shell shell,
+ PipelineMeta pipelineMeta,
+ IVariables variables,
+ IHopMetadataProvider metadataProvider) {
+ if
(!FileValidationConfigPlugin.getInstance().isValidateDbConnectionsOnSave()) {
+ return true;
+ }
+ return confirmRemarks(
+ shell,
+ ReferencedDatabaseConnectionChecker.checkPipeline(
+ pipelineMeta, variables, metadataProvider));
+ }
+
+ public static boolean confirmSave(
+ Shell shell,
+ WorkflowMeta workflowMeta,
+ IVariables variables,
+ IHopMetadataProvider metadataProvider) {
+ if
(!FileValidationConfigPlugin.getInstance().isValidateDbConnectionsOnSave()) {
+ return true;
+ }
+ return confirmRemarks(
+ shell,
+ ReferencedDatabaseConnectionChecker.checkWorkflow(
+ workflowMeta, variables, metadataProvider));
+ }
+
+ static boolean confirmRemarks(Shell shell, List<ICheckResult> remarks) {
+ if (remarks == null || remarks.isEmpty()) {
+ return true;
+ }
+ if (shell == null || shell.isDisposed()) {
+ return true;
+ }
+
+ String listed =
+ remarks.stream()
+ .limit(MAX_LISTED)
+ .map(ICheckResult::getText)
+ .collect(Collectors.joining("\n"));
+ if (remarks.size() > MAX_LISTED) {
+ listed =
+ listed
+ + "\n"
+ + BaseMessages.getString(
+ PKG,
+ "ReferencedConnectionSaveValidator.Dialog.More",
+ Integer.toString(remarks.size() - MAX_LISTED));
+ }
+
+ MessageDialogWithToggle dialog =
+ new MessageDialogWithToggle(
+ shell,
+ BaseMessages.getString(PKG,
"ReferencedConnectionSaveValidator.Dialog.Title"),
+ BaseMessages.getString(PKG,
"ReferencedConnectionSaveValidator.Dialog.Message", listed),
+ SWT.ICON_WARNING,
+ new String[] {
+ BaseMessages.getString(PKG, "System.Button.Yes"),
+ BaseMessages.getString(PKG, "System.Button.No")
+ },
+ BaseMessages.getString(PKG,
"ReferencedConnectionSaveValidator.Dialog.Toggle"),
+ false);
+ int answer = dialog.open();
+
+ if (dialog.getToggleState()) {
+ FileValidationConfigPlugin config =
FileValidationConfigPlugin.getInstance();
+ config.setValidateDbConnectionsOnSave(false);
+ config.saveToHopConfig();
+ }
+
+ return answer == 0;
+ }
+}
diff --git
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/HopGuiWorkflowGraph.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/HopGuiWorkflowGraph.java
index d61339984a..a35b218104 100644
---
a/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/HopGuiWorkflowGraph.java
+++
b/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/HopGuiWorkflowGraph.java
@@ -98,6 +98,7 @@ import org.apache.hop.execution.IExecutionInfoLocation;
import org.apache.hop.history.AuditManager;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.laf.BasePropertyHandler;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.api.IHopMetadataSerializer;
import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
import org.apache.hop.pipeline.PipelinePainter;
@@ -141,6 +142,7 @@ import
org.apache.hop.ui.hopgui.file.shared.HopGuiAbstractGraph;
import org.apache.hop.ui.hopgui.file.shared.HopGuiGraphSnapshotUndo;
import org.apache.hop.ui.hopgui.file.shared.HopGuiTooltipExtension;
import org.apache.hop.ui.hopgui.file.shared.ISnapshotUndoSupport;
+import org.apache.hop.ui.hopgui.file.shared.ReferencedConnectionSaveValidator;
import
org.apache.hop.ui.hopgui.file.workflow.context.HopGuiWorkflowActionContext;
import org.apache.hop.ui.hopgui.file.workflow.context.HopGuiWorkflowContext;
import org.apache.hop.ui.hopgui.file.workflow.context.HopGuiWorkflowHopContext;
@@ -4673,6 +4675,15 @@ public class HopGuiWorkflowGraph extends
HopGuiAbstractGraph
throw new HopException("No filename: please specify a filename for
this workflow");
}
+ IHopMetadataProvider saveMetadataProvider =
workflowMeta.getMetadataProvider();
+ if (saveMetadataProvider == null) {
+ saveMetadataProvider = hopGui.getMetadataProvider();
+ }
+ if (!ReferencedConnectionSaveValidator.confirmSave(
+ hopShell(), workflowMeta, variables, saveMetadataProvider)) {
+ return;
+ }
+
// Keep track of save
//
AuditManager.registerEvent(
diff --git
a/ui/src/main/resources/org/apache/hop/ui/hopgui/file/config/messages/messages_en_US.properties
b/ui/src/main/resources/org/apache/hop/ui/hopgui/file/config/messages/messages_en_US.properties
new file mode 100644
index 0000000000..cbb4f206bc
--- /dev/null
+++
b/ui/src/main/resources/org/apache/hop/ui/hopgui/file/config/messages/messages_en_US.properties
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FileValidationConfigPlugin.Description=File validation
+FileValidationConfigPlugin.ValidateDbConnectionsOnSave.Label=Validate database
connections when saving
+FileValidationConfigPlugin.ValidateDbConnectionsOnSave.ToolTip=When saving a
pipeline or workflow, warn if a transform or action references a database
connection that is not in the project metadata. Names that still contain a
variable such as '${CONNECTION}' after resolving the current environment are
skipped. Saving is never blocked.
+ReferencedConnectionSaveValidator.Dialog.Title=Database connections
+ReferencedConnectionSaveValidator.Dialog.Message=The following database
connection problems were found. Save anyway?\n\n{0}
+ReferencedConnectionSaveValidator.Dialog.More=...and {0} more
+ReferencedConnectionSaveValidator.Dialog.Toggle=Don''t run this check when
saving. Configure the option in the configuration perspective
