This is an automated email from the ASF dual-hosted git repository.
bamaer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new e487fe70f5 Issue #2727 : Align Kettle import connection names to a
case-sensitive scheme (#8317)
e487fe70f5 is described below
commit e487fe70f58a235701e5148ed3544b77d7fcc88e
Author: Matt Casters <[email protected]>
AuthorDate: Fri Sep 11 13:41:33 2026 +0200
Issue #2727 : Align Kettle import connection names to a case-sensitive
scheme (#8317)
Kettle treated relational connection names as case-insensitive. Hop metadata
does not, so mixed-case references from a PDI folder import failed to
resolve.
During import, collapse those spellings to one name (shared.xml wins, then
frequency), optionally apply a hop-metadata naming scheme, and rewrite every
RDBMS_CONNECTION field. The import dialog keeps new metadata in memory until
Import, then writes only the selected run configurations and naming scheme.
---
.../hop/core/extension/HopExtensionPoint.java | 4 +
.../metadata/util/HopMetadataPropertyWalker.java | 75 +-
.../util/HopMetadataPropertyWalkerTest.java | 87 +++
.../assets/images/hop-import/import-dialog.png | Bin 46470 -> 0 bytes
.../kettle-import-dialog-metadata-tab.png | Bin 0 -> 72833 bytes
.../hop-import/kettle-import-dialog-source-tab.png | Bin 0 -> 90657 bytes
.../hop-import/kettle-import-dialog-target-tab.png | Bin 0 -> 77277 bytes
.../modules/ROOT/pages/hop-tools/hop-import.adoc | 8 +
.../hop-vs-kettle/import-kettle-projects.adoc | 43 +-
.../ROOT/pages/metadata-types/naming-scheme.adoc | 9 +
.../java/org/apache/hop/imp/ConnectionNameMap.java | 181 +++++
.../main/java/org/apache/hop/imp/HopImport.java | 14 +
.../java/org/apache/hop/imp/HopImportBase.java | 75 +-
.../main/java/org/apache/hop/imp/IHopImport.java | 14 +
.../apache/hop/imp/ImportedConnectionRewriter.java | 283 ++++++++
.../org/apache/hop/imp/ConnectionNameMapTest.java | 97 +++
.../hop/imp/ImportedConnectionRewriterTest.java | 118 ++++
.../apache/hop/imports/kettle/KettleImport.java | 84 ++-
.../hop/imports/kettle/KettleImportDialog.java | 760 +++++++++++----------
.../kettle/messages/messages_en_US.properties | 7 +-
.../hop/imports/kettle/KettleImportDialogTest.java | 39 +-
.../hop/imports/kettle/KettleImportTest.java | 25 +
.../naming/gui/NamingSchemeImportExtension.java | 89 +++
.../gui/NamingSchemeImportExtensionTest.java | 148 ++++
.../hop/ui/core/widget/MetaSelectionLine.java | 17 +
25 files changed, 1767 insertions(+), 410 deletions(-)
diff --git
a/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java
b/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java
index 74be646186..a57675a18f 100644
--- a/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java
+++ b/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java
@@ -192,6 +192,10 @@ public enum HopExtensionPoint {
HopGuiSearchMarketplace("Open the marketplace, searching for a plugin id
(String)"),
HopImportStart("Executed at the start of the 'hop-import' command line
tool"),
+ HopImportTargetMetadataReady(
+ "The import target metadata provider has been created (HopImportBase)"),
+ HopImportRewriteMetadata(
+ "Imported files and connections have been written; rewrite metadata
names (HopImportBase)"),
HopImportEnd("Executed at the end of the 'hop-import' command line tool"),
;
diff --git
a/core/src/main/java/org/apache/hop/metadata/util/HopMetadataPropertyWalker.java
b/core/src/main/java/org/apache/hop/metadata/util/HopMetadataPropertyWalker.java
index 699166be99..d1f0d93801 100644
---
a/core/src/main/java/org/apache/hop/metadata/util/HopMetadataPropertyWalker.java
+++
b/core/src/main/java/org/apache/hop/metadata/util/HopMetadataPropertyWalker.java
@@ -26,12 +26,13 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.function.UnaryOperator;
import org.apache.hop.metadata.api.HopMetadataProperty;
import org.apache.hop.metadata.api.HopMetadataPropertyType;
/**
* Walks {@link HopMetadataProperty} fields on a metadata object, including
nested objects and
- * collections, and collects string values of a given {@link
HopMetadataPropertyType}.
+ * collections, and collects or rewrites string values of a given {@link
HopMetadataPropertyType}.
*
* <p>Failures to read a field are skipped. Cycles are broken. This is a
design-time helper: it must
* not throw because of a broken plugin class.
@@ -66,16 +67,60 @@ public final class HopMetadataPropertyWalker {
walk(
root,
type,
- collected,
+ (field, node, property, value) ->
+ collected.add(new StringProperty(type, serialisedKey(property,
field), value)),
0,
java.util.Collections.newSetFromMap(new IdentityHashMap<Object,
Boolean>()));
return collected;
}
+ /**
+ * Rewrite every string field annotated with {@code type} under {@code root}
by applying {@code
+ * mapper}. Fields whose mapped value is {@code null} or equal to the
current value are left
+ * unchanged. Failures to read or write a field are skipped.
+ *
+ * @param root the object to walk, may be null
+ * @param type the property type to rewrite
+ * @param mapper replacement for each matching string, may be null
+ * @return the number of fields whose value changed
+ */
+ public static int rewriteStrings(
+ Object root, HopMetadataPropertyType type, UnaryOperator<String> mapper)
{
+ if (root == null || type == null || mapper == null) {
+ return 0;
+ }
+ int[] changed = new int[1];
+ walk(
+ root,
+ type,
+ (field, node, property, value) -> {
+ String mapped;
+ try {
+ mapped = mapper.apply(value);
+ } catch (Exception e) {
+ return;
+ }
+ if (mapped == null || mapped.equals(value)) {
+ return;
+ }
+ if (writeField(field, node, mapped)) {
+ changed[0]++;
+ }
+ },
+ 0,
+ java.util.Collections.newSetFromMap(new IdentityHashMap<Object,
Boolean>()));
+ return changed[0];
+ }
+
+ @FunctionalInterface
+ private interface StringFieldHandler {
+ void handle(Field field, Object node, HopMetadataProperty property, String
value);
+ }
+
private static void walk(
Object node,
HopMetadataPropertyType type,
- List<StringProperty> collected,
+ StringFieldHandler handler,
int depth,
Set<Object> visited) {
if (node == null || depth > MAX_DEPTH || !isMetadataObject(node) ||
!visited.add(node)) {
@@ -94,38 +139,38 @@ public final class HopMetadataPropertyWalker {
continue;
}
if (property.hopMetadataPropertyType() == type && value instanceof
String stringValue) {
- collected.add(new StringProperty(type, serialisedKey(property, field),
stringValue));
+ handler.handle(field, node, property, stringValue);
}
- descend(value, type, collected, depth, visited);
+ descend(value, type, handler, depth, visited);
}
}
private static void descend(
Object value,
HopMetadataPropertyType type,
- List<StringProperty> collected,
+ StringFieldHandler handler,
int depth,
Set<Object> visited) {
if (value instanceof Collection<?> collection) {
for (Object element : collection) {
- walk(element, type, collected, depth + 1, visited);
+ walk(element, type, handler, depth + 1, visited);
}
return;
}
if (value instanceof Map<?, ?> map) {
for (Object element : map.values()) {
- walk(element, type, collected, depth + 1, visited);
+ walk(element, type, handler, depth + 1, visited);
}
return;
}
if (value.getClass().isArray()) {
int length = Array.getLength(value);
for (int i = 0; i < length; i++) {
- walk(Array.get(value, i), type, collected, depth + 1, visited);
+ walk(Array.get(value, i), type, handler, depth + 1, visited);
}
return;
}
- walk(value, type, collected, depth + 1, visited);
+ walk(value, type, handler, depth + 1, visited);
}
private static String serialisedKey(HopMetadataProperty property, Field
field) {
@@ -156,4 +201,14 @@ public final class HopMetadataPropertyWalker {
return null;
}
}
+
+ private static boolean writeField(Field field, Object target, String value) {
+ try {
+ field.setAccessible(true);
+ field.set(target, value);
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ }
}
diff --git
a/core/src/test/java/org/apache/hop/metadata/util/HopMetadataPropertyWalkerTest.java
b/core/src/test/java/org/apache/hop/metadata/util/HopMetadataPropertyWalkerTest.java
index f6a51c91a1..bb5665df13 100644
---
a/core/src/test/java/org/apache/hop/metadata/util/HopMetadataPropertyWalkerTest.java
+++
b/core/src/test/java/org/apache/hop/metadata/util/HopMetadataPropertyWalkerTest.java
@@ -73,6 +73,14 @@ class HopMetadataPropertyWalkerTest {
String connection = "hidden";
}
+ static class SftpAndRdbmsMeta {
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.RDBMS_CONNECTION)
+ String rdbms = "Warehouse";
+
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.VFS_SFTP_CONNECTION)
+ String sftp = "sftp-server";
+ }
+
@Test
void collectsAnnotatedConnectionStrings() {
List<StringProperty> found =
@@ -121,4 +129,83 @@ class HopMetadataPropertyWalkerTest {
HopMetadataPropertyWalker.collectStrings(null,
HopMetadataPropertyType.RDBMS_CONNECTION)
.isEmpty());
}
+
+ @Test
+ void rewriteChangesAnnotatedStrings() {
+ SimpleMeta meta = new SimpleMeta();
+
+ int changed =
+ HopMetadataPropertyWalker.rewriteStrings(
+ meta, HopMetadataPropertyType.RDBMS_CONNECTION,
String::toUpperCase);
+
+ assertEquals(1, changed);
+ assertEquals("WAREHOUSE", meta.connection);
+ assertEquals("SELECT 1", meta.sql);
+ assertEquals("ignored", meta.unannotated);
+ }
+
+ @Test
+ void rewriteDescendsIntoNestedLists() {
+ NestedMeta meta = new NestedMeta();
+
+ int changed =
+ HopMetadataPropertyWalker.rewriteStrings(
+ meta, HopMetadataPropertyType.RDBMS_CONNECTION, value -> value +
"-x");
+
+ assertEquals(3, changed);
+ assertEquals("primary-x", meta.connection);
+ assertEquals("second-x", meta.items.get(0).name);
+ assertEquals("third-x", meta.items.get(1).name);
+ }
+
+ @Test
+ void rewriteLeavesUnannotatedConnectionFields() {
+ UnannotatedConnectionMeta meta = new UnannotatedConnectionMeta();
+
+ int changed =
+ HopMetadataPropertyWalker.rewriteStrings(
+ meta, HopMetadataPropertyType.RDBMS_CONNECTION,
String::toUpperCase);
+
+ assertEquals(0, changed);
+ assertEquals("hidden", meta.connection);
+ }
+
+ @Test
+ void rewriteSkipsWhenMapperReturnsTheSameValue() {
+ SimpleMeta meta = new SimpleMeta();
+
+ int changed =
+ HopMetadataPropertyWalker.rewriteStrings(
+ meta, HopMetadataPropertyType.RDBMS_CONNECTION, value -> value);
+
+ assertEquals(0, changed);
+ assertEquals("warehouse", meta.connection);
+ }
+
+ @Test
+ void rewriteDoesNotTouchNonRdbmsConnectionFields() {
+ SftpAndRdbmsMeta meta = new SftpAndRdbmsMeta();
+
+ int changed =
+ HopMetadataPropertyWalker.rewriteStrings(
+ meta, HopMetadataPropertyType.RDBMS_CONNECTION,
String::toLowerCase);
+
+ assertEquals(1, changed);
+ assertEquals("warehouse", meta.rdbms);
+ assertEquals("sftp-server", meta.sftp);
+ }
+
+ @Test
+ void rewriteNullMapperOrRootDoesNothing() {
+ SimpleMeta meta = new SimpleMeta();
+ assertEquals(
+ 0,
+ HopMetadataPropertyWalker.rewriteStrings(
+ meta, HopMetadataPropertyType.RDBMS_CONNECTION, null));
+ assertEquals(
+ 0,
+ HopMetadataPropertyWalker.rewriteStrings(
+ null, HopMetadataPropertyType.RDBMS_CONNECTION,
String::toUpperCase));
+ assertEquals("warehouse", meta.connection);
+ }
}
diff --git
a/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/import-dialog.png
b/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/import-dialog.png
deleted file mode 100644
index 0719339ac9..0000000000
Binary files
a/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/import-dialog.png
and /dev/null differ
diff --git
a/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/kettle-import-dialog-metadata-tab.png
b/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/kettle-import-dialog-metadata-tab.png
new file mode 100644
index 0000000000..d8de0a6b44
Binary files /dev/null and
b/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/kettle-import-dialog-metadata-tab.png
differ
diff --git
a/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/kettle-import-dialog-source-tab.png
b/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/kettle-import-dialog-source-tab.png
new file mode 100644
index 0000000000..44251ab1ea
Binary files /dev/null and
b/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/kettle-import-dialog-source-tab.png
differ
diff --git
a/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/kettle-import-dialog-target-tab.png
b/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/kettle-import-dialog-target-tab.png
new file mode 100644
index 0000000000..013263fdfa
Binary files /dev/null and
b/docs/hop-user-manual/modules/ROOT/assets/images/hop-import/kettle-import-dialog-target-tab.png
differ
diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-import.adoc
b/docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-import.adoc
index 80dd415131..0bf1c5cc46 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-import.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/hop-tools/hop-import.adoc
@@ -59,6 +59,12 @@ Usage: <main class> [-efhlp] [-c=<targetConfigFilename>]
[-i=<inputFolderName>]
-s, --shared-xml=<sharedXmlFilename>
The shared.xml file to read from
-t, --type=<type> The type of import plugin to use (e.g. kettle)
+ -n, --naming-scheme=<name>
+ Naming scheme metadata name in the target folder
+ to apply to relational connections
+ --no-apply-naming-schemes
+ Do not apply a naming scheme; connection names are
+ still aligned to one spelling
-v, --version Print version information and exit
----
@@ -66,6 +72,8 @@ Usage: <main class> [-efhlp] [-c=<targetConfigFilename>]
[-i=<inputFolderName>]
+
+Place naming-scheme JSON files in `<output>/metadata/naming-scheme/` before
running the command (or create them in the target project in Hop Gui). The
unique *Hop metadata* scheme, or a unique *General* scheme, is applied to
imported relational connection names. Use `--naming-scheme` to pick one by name.
+
== Examples
Import a set of Kettle files and folders into a project stored on Amazon AWS
S3:
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/hop-vs-kettle/import-kettle-projects.adoc
b/docs/hop-user-manual/modules/ROOT/pages/hop-vs-kettle/import-kettle-projects.adoc
index 4ffa436011..236fe347e1 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/hop-vs-kettle/import-kettle-projects.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/hop-vs-kettle/import-kettle-projects.adoc
@@ -40,7 +40,7 @@ Compatibility with Kettle/PDI was never a goal for Apache
Hop, but since a lot o
== Known limitations
-* no connection cleanup: only 1 copy of database connections with the same
name but different configurations is kept.
+* no connection cleanup: only 1 copy of database connections with the same
name but different configurations is kept. Names that only differ by case
(Kettle was case-insensitive) are collapsed to one spelling.
* no metastore import
== Usage
@@ -49,33 +49,52 @@ To import your Kettle/PDI projects in Hop, select `File ->
Import from Kettle/PD
image:hop-import/menu-import.png[File --> Import from Kettle/PDI]
-Add you import sources and target in the pop-up dialog you'll be presented
with:
+The dialog has three tabs: *Source*, *Target*, and *Metadata*. Import and
Cancel stay at the bottom of the window.
-image:hop-import/import-dialog.png[Import Dialog]
+=== Source
-The options in this dialog are:
+image::hop-import/kettle-import-dialog-source-tab.png[Kettle import dialog
Source tab,width="90%"]
[options="header",width=90%]
|===
|Option|Description|Default
|Import from|The folder to import Kettle/PDI jobs and transformations from|-
-|Import in existing project|Check to import into an existing project, uncheck
to import into a folder|Selected
-|Import in project|Dropdown list of available projects to import the
Kettle/PDI project into|-
-|Import to folder|Path to import the Kettle/PDI project to.
-All imported items will be imported into a Hop project in this folder.
-Only available when *Import in existing project* is unchecked.|-
|Path to kettle.properties|Path to a kettle.properties file.
All properties in this file will be imported as variables in the Hop project.|-
|Path to shared.xml|Path to a shared.xml file.
All database connections in this file will be imported as Hop relational
database connection metadata objects in the specified Hop project or folder.|-
|Path to jdbc.properties|Path to a jdbc.properties file.
All Kettle/PDI JNDI database connections in this file will be imported as Hop
(generic) relational database connection metadata objects in the specified Hop
project or folder.|-
-|Skip existing target files?|Skip files that are already present in the target
folder|Selected
|Skip hidden files and folders?|Exclude hidden files and folders such as
`.git` and `.gitignore` from the import|Selected
|Skip folders in the source?|Exclude the sub-folders of the source folder from
the import|Selected
+|===
+
+=== Target
+
+image::hop-import/kettle-import-dialog-target-tab.png[Kettle import dialog
Target tab,width="90%"]
+
+[options="header",width=90%]
+|===
+|Option|Description|Default
+|Import in existing project|Check to import into an existing project, uncheck
to import into a folder|Selected
+|Import in project|Dropdown list of available projects to import the
Kettle/PDI project into|-
+|Import to folder|Path to import the Kettle/PDI project to.
+All imported items will be imported into a Hop project in this folder.
+Only available when *Import in existing project* is unchecked.|-
+|Skip existing target files?|Skip files that are already present in the target
folder|Selected
|Target environment variables config file|Name of the environment
configuration file the variables from `kettle.properties` are written
to|`imported-env-conf.json`
-|Pipeline default run configuration|The run configuration set as the default
on imported pipelines|-
-|Workflow default run configuration|The run configuration set as the default
on imported workflows|-
+|===
+
+=== Metadata
+
+image::hop-import/kettle-import-dialog-metadata-tab.png[Kettle import dialog
Metadata tab,width="90%"]
+
+[options="header",width=90%]
+|===
+|Option|Description|Default
+|Pipeline default run configuration|The run configuration set as the default
on imported pipelines. The line lists objects from the target folder and the
current project. New and Edit stay in memory until you click Import, which
writes the selected object to the target project's `metadata/` folder if it is
not there yet.|-
+|Workflow default run configuration|The run configuration set as the default
on imported workflows. Same in-memory-until-Import behaviour as the pipeline
run configuration.|-
+|Naming scheme for connections|Optional naming scheme applied to imported
relational connection names (the connection metadata and every
`@HopMetadataProperty` of type `RDBMS_CONNECTION`). New and Edit stay in memory
until Import. Leave empty to use the unique Hop-metadata (or General) scheme in
the target folder. Connection names that only differ by case are always
aligned, even without a scheme.|-
|===
After entering your import details, click the 'Import' button.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/naming-scheme.adoc
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/naming-scheme.adoc
index 6b20e7e13e..0afcc002ae 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/naming-scheme.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/naming-scheme.adoc
@@ -134,6 +134,15 @@ The *Type* combo lists every registered naming-scheme
kind. Built-in kinds inclu
Other plugins can add kinds without changing Hop. For example hop-data-vault
can register `dv-hub`, `dv-satellite`, `dv-link`, `dm-dimension`, `dm-fact`,
and so on. Create one Naming Scheme metadata object per kind (or one General
scheme). Use one scheme per type in a project so CI has a single expected
spelling.
+== Kettle / PDI import
+
+When you import a Kettle/PDI folder, Hop looks for naming schemes in the
*target* project's `metadata/naming-scheme/` folder (the same Json metadata
provider the import writes connections into).
+
+* In Hop Gui the import dialog has a *Naming scheme for connections* line on
the Metadata tab. New and Edit stay in memory until you click Import, which
writes the selected scheme to the target project's `metadata/naming-scheme/`
folder if it is not there yet. The line also lists schemes already in that
folder and in the current project.
+* From the command line, put the JSON files in
`<output>/metadata/naming-scheme/` first, or pass `--naming-scheme <name>`.
+
+The selected (or unique Hop-metadata / General) scheme is applied to every
imported relational connection name and to every transform/action field
annotated `RDBMS_CONNECTION`. Names that only differ by case are always
collapsed to one spelling, even when no scheme is present.
+
== Checking names in CI
A name is valid when applying the matching scheme leaves the string unchanged.
Empty values, `<null>`, and values that contain `${...}` are skipped.
diff --git a/engine/src/main/java/org/apache/hop/imp/ConnectionNameMap.java
b/engine/src/main/java/org/apache/hop/imp/ConnectionNameMap.java
new file mode 100644
index 0000000000..38ba1da50d
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/imp/ConnectionNameMap.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.imp;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.UnaryOperator;
+import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Maps every observed spelling of a relational connection name onto one
target name.
+ *
+ * <p>Kettle treated names as case-insensitive, so {@code Database}, {@code
database} and {@code
+ * DATABASE} are one group. A group collapses to a preferred spelling
(shared.xml, then the most
+ * frequent, then first seen). An optional mapper (a naming scheme) is then
applied to that
+ * canonical spelling. Distinct groups that land on the same target are
recorded as collisions; they
+ * still share the target name.
+ */
+@Getter
+public final class ConnectionNameMap {
+
+ private final Map<String, String> oldToNew = new LinkedHashMap<>();
+ private final List<Collision> collisions = new ArrayList<>();
+
+ public record Collision(String leftOriginal, String rightOriginal, String
targetName) {}
+
+ private ConnectionNameMap() {}
+
+ public static ConnectionNameMap empty() {
+ return new ConnectionNameMap();
+ }
+
+ /**
+ * Names that cannot be rewritten: empty, table null markers, and values
that contain a variable
+ * expression.
+ */
+ public static boolean shouldSkip(String value) {
+ if (StringUtils.isEmpty(value) || "<null>".equals(value)) {
+ return true;
+ }
+ return value.contains("${");
+ }
+
+ /**
+ * @param names every spelling found on connections and in pipeline/workflow
references
+ * @param preferredNames spellings that win inside a case-insensitive group
(typically shared.xml)
+ * @param mapper applied to the canonical original; {@code null} or identity
leaves it as-is
+ */
+ public static ConnectionNameMap build(
+ Collection<String> names, Collection<String> preferredNames,
UnaryOperator<String> mapper) {
+ Map<String, List<String>> groups = new LinkedHashMap<>();
+ Map<String, Integer> frequency = new LinkedHashMap<>();
+ if (names != null) {
+ for (String name : names) {
+ if (shouldSkip(name)) {
+ continue;
+ }
+ String key = name.toLowerCase(Locale.ROOT);
+ groups.computeIfAbsent(key, k -> new ArrayList<>()).add(name);
+ frequency.merge(name, 1, Integer::sum);
+ }
+ }
+
+ Map<String, String> preferredByLower = new LinkedHashMap<>();
+ if (preferredNames != null) {
+ for (String preferred : preferredNames) {
+ if (shouldSkip(preferred)) {
+ continue;
+ }
+ preferredByLower.putIfAbsent(preferred.toLowerCase(Locale.ROOT),
preferred);
+ }
+ }
+
+ UnaryOperator<String> effective = mapper != null ? mapper :
UnaryOperator.identity();
+ ConnectionNameMap result = new ConnectionNameMap();
+ Map<String, String> targetOwner = new LinkedHashMap<>();
+
+ for (Map.Entry<String, List<String>> group : groups.entrySet()) {
+ String canonical =
+ pickCanonical(group.getValue(),
preferredByLower.get(group.getKey()), frequency);
+ String target = canonical;
+ try {
+ String mapped = effective.apply(canonical);
+ if (!shouldSkip(mapped)) {
+ target = mapped;
+ }
+ } catch (Exception e) {
+ // Keep the canonical original when the mapper fails.
+ }
+
+ String targetKey = target.toLowerCase(Locale.ROOT);
+ String owner = targetOwner.putIfAbsent(targetKey, canonical);
+ if (owner != null && !owner.equalsIgnoreCase(canonical)) {
+ result.collisions.add(new Collision(owner, canonical, target));
+ }
+
+ Set<String> unique = new LinkedHashSet<>(group.getValue());
+ for (String spelling : unique) {
+ result.oldToNew.put(spelling, target);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Look up the target name for {@code name}. Exact match wins, then
case-insensitive. Unknown
+ * names (and skipped values) are returned unchanged.
+ */
+ public String targetFor(String name) {
+ if (shouldSkip(name)) {
+ return name;
+ }
+ String exact = oldToNew.get(name);
+ if (exact != null) {
+ return exact;
+ }
+ for (Map.Entry<String, String> entry : oldToNew.entrySet()) {
+ if (entry.getKey().equalsIgnoreCase(name)) {
+ return entry.getValue();
+ }
+ }
+ return name;
+ }
+
+ public boolean isEmpty() {
+ return oldToNew.isEmpty();
+ }
+
+ /** How many spellings actually change. */
+ public int changedCount() {
+ int count = 0;
+ for (Map.Entry<String, String> entry : oldToNew.entrySet()) {
+ if (!entry.getKey().equals(entry.getValue())) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ private static String pickCanonical(
+ List<String> spellings, String preferred, Map<String, Integer>
frequency) {
+ if (preferred != null) {
+ return preferred;
+ }
+ String best = spellings.get(0);
+ int bestCount = frequency.getOrDefault(best, 0);
+ Set<String> seen = new LinkedHashSet<>();
+ for (String spelling : spellings) {
+ if (!seen.add(spelling)) {
+ continue;
+ }
+ int count = frequency.getOrDefault(spelling, 0);
+ if (count > bestCount) {
+ best = spelling;
+ bestCount = count;
+ }
+ }
+ return best;
+ }
+}
diff --git a/engine/src/main/java/org/apache/hop/imp/HopImport.java
b/engine/src/main/java/org/apache/hop/imp/HopImport.java
index 29c8ff2be2..ef027eec35 100644
--- a/engine/src/main/java/org/apache/hop/imp/HopImport.java
+++ b/engine/src/main/java/org/apache/hop/imp/HopImport.java
@@ -124,6 +124,18 @@ public class HopImport implements Runnable,
IHasHopMetadataProvider, IHopCommand
description = "Print version information and exit")
private boolean versionRequested;
+ @Option(
+ names = {"-n", "--naming-scheme"},
+ description =
+ "Naming scheme metadata name in the target folder to apply to
relational connections")
+ private String namingSchemeName;
+
+ @Option(
+ names = {"--no-apply-naming-schemes"},
+ description =
+ "Do not apply a naming scheme; connection names are still aligned to
one spelling")
+ private boolean noApplyNamingSchemes;
+
private MultiMetadataProvider metadataProvider;
private IVariables variables;
private CommandLine cmd;
@@ -195,6 +207,8 @@ public class HopImport implements Runnable,
IHasHopMetadataProvider, IHopCommand
hopImport.setSkippingFolders(skippingFolders);
}
hopImport.setTargetConfigFilename(targetConfigFilename);
+ hopImport.setApplyNamingSchemes(!noApplyNamingSchemes);
+ hopImport.setNamingSchemeName(namingSchemeName);
// Allow plugins to modify the elements loaded so far, before a pipeline
or workflow is even
// loaded
diff --git a/engine/src/main/java/org/apache/hop/imp/HopImportBase.java
b/engine/src/main/java/org/apache/hop/imp/HopImportBase.java
index 36aa958abf..15bfdf4cc1 100644
--- a/engine/src/main/java/org/apache/hop/imp/HopImportBase.java
+++ b/engine/src/main/java/org/apache/hop/imp/HopImportBase.java
@@ -21,17 +21,24 @@ import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Properties;
+import java.util.Set;
import java.util.TreeMap;
-import java.util.stream.Collectors;
+import java.util.function.UnaryOperator;
import javax.xml.transform.dom.DOMSource;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.IProgressMonitor;
import org.apache.hop.core.database.DatabaseMeta;
import org.apache.hop.core.encryption.Encr;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.exception.HopFileException;
+import org.apache.hop.core.extension.ExtensionPointHandler;
+import org.apache.hop.core.extension.HopExtensionPoint;
import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.variables.IVariables;
@@ -71,6 +78,14 @@ public abstract class HopImportBase implements IHopImport {
protected MultiMetadataProvider metadataProvider;
protected IProgressMonitor monitor;
protected String metadataTargetFolder;
+ @Getter @Setter protected boolean collectingFromSharedXml;
+ @Getter @Setter protected Set<String> sharedConnectionNames;
+ @Getter @Setter protected List<String> writtenHopFileNames;
+ @Getter @Setter protected boolean applyNamingSchemes;
+ @Getter @Setter protected String namingSchemeName;
+ @Getter @Setter protected String appliedNamingSchemeName;
+ @Setter protected UnaryOperator<String> connectionNameMapper;
+ @Getter @Setter protected ImportedConnectionRewriter.Result
connectionRewriteResult;
public HopImportBase() {
this.variables = new Variables();
@@ -81,6 +96,10 @@ public abstract class HopImportBase implements IHopImport {
connectionFileMap = new TreeMap<>();
migratedFilesMap = new HashMap<>();
collectedVariables = new Variables();
+ sharedConnectionNames = new LinkedHashSet<>();
+ writtenHopFileNames = new ArrayList<>();
+ applyNamingSchemes = true;
+ connectionNameMapper = UnaryOperator.identity();
}
@Override
@@ -96,7 +115,7 @@ public abstract class HopImportBase implements IHopImport {
// Create a new metadata provider for the target folder...
//
if (metadataProvider == null) {
- this.metadataTargetFolder = outputFolder.getName().getURI() +
"/metadata";
+ this.metadataTargetFolder =
metadataFolderFor(outputFolder.getName().getURI());
metadataProvider =
new MultiMetadataProvider(
Encr.getEncoder(),
@@ -105,6 +124,11 @@ public abstract class HopImportBase implements IHopImport {
Encr.getEncoder(), this.metadataTargetFolder,
variables)),
variables);
}
+ if (StringUtils.isEmpty(metadataTargetFolder) && outputFolder != null) {
+ this.metadataTargetFolder =
metadataFolderFor(outputFolder.getName().getURI());
+ }
+ ExtensionPointHandler.callExtensionPoint(
+ log, variables, HopExtensionPoint.HopImportTargetMetadataReady.id,
this);
if (monitor != null) {
monitor.setTaskName("Finding files to import");
}
@@ -126,6 +150,17 @@ public abstract class HopImportBase implements IHopImport {
monitor.setTaskName("Importing connections");
}
importConnections();
+ if (monitor != null) {
+ if (monitor.isCanceled()) {
+ return;
+ }
+ monitor.worked(1);
+ monitor.setTaskName("Normalizing connection names");
+ }
+ ExtensionPointHandler.callExtensionPoint(
+ log, variables, HopExtensionPoint.HopImportRewriteMetadata.id, this);
+ connectionRewriteResult = ImportedConnectionRewriter.rewrite(this);
+ afterConnectionRewrite();
if (monitor != null) {
if (monitor.isCanceled()) {
return;
@@ -172,11 +207,16 @@ public abstract class HopImportBase implements IHopImport
{
// build a list of all jobs, transformations with their connections
connectionFileMap.put(filename, databaseMeta.getName());
- // only add new connection names to the list
+ if (collectingFromSharedXml && databaseMeta.getName() != null) {
+ sharedConnectionNames.add(databaseMeta.getName());
+ }
+
+ // Kettle names are case-insensitive: keep the first spelling we saw.
if (connectionsList.stream()
- .filter(dbMeta -> dbMeta.getName().equals(databaseMeta.getName()))
- .collect(Collectors.toList())
- .isEmpty()) {
+ .noneMatch(
+ dbMeta ->
+ dbMeta.getName() != null
+ &&
dbMeta.getName().equalsIgnoreCase(databaseMeta.getName()))) {
connectionsList.add(databaseMeta);
connectionCounter++;
}
@@ -574,6 +614,29 @@ public abstract class HopImportBase implements IHopImport {
this.skippingFolders = skippingFolders;
}
+ public UnaryOperator<String> getConnectionNameMapper() {
+ return connectionNameMapper != null ? connectionNameMapper :
UnaryOperator.identity();
+ }
+
+ /**
+ * {@code <folder>/metadata}, with trailing slashes and backslashes
normalized so VFS URIs do not
+ * pick up a double slash.
+ */
+ public static String metadataFolderFor(String targetFolder) {
+ if (StringUtils.isBlank(targetFolder)) {
+ return null;
+ }
+ return StringUtils.removeEnd(targetFolder.replace('\\', '/'), "/") +
"/metadata";
+ }
+
+ /**
+ * Called after imported connection names have been normalized. Override to
write reports that
+ * include the final names.
+ */
+ protected void afterConnectionRewrite() throws HopException {
+ // nothing
+ }
+
/**
* Generate a report with statistics and advice.
*
diff --git a/engine/src/main/java/org/apache/hop/imp/IHopImport.java
b/engine/src/main/java/org/apache/hop/imp/IHopImport.java
index 4d7a259b74..ca8d55ba28 100644
--- a/engine/src/main/java/org/apache/hop/imp/IHopImport.java
+++ b/engine/src/main/java/org/apache/hop/imp/IHopImport.java
@@ -172,4 +172,18 @@ public interface IHopImport {
* @param metadataProvider The metadataProvider to set
*/
void setMetadataProvider(MultiMetadataProvider metadataProvider);
+
+ /** When true, a hop-metadata (or general) naming scheme in the target is
applied. */
+ default void setApplyNamingSchemes(boolean applyNamingSchemes) {}
+
+ default boolean isApplyNamingSchemes() {
+ return true;
+ }
+
+ /** Optional explicit naming-scheme metadata name to apply to relational
connections. */
+ default void setNamingSchemeName(String namingSchemeName) {}
+
+ default String getNamingSchemeName() {
+ return null;
+ }
}
diff --git
a/engine/src/main/java/org/apache/hop/imp/ImportedConnectionRewriter.java
b/engine/src/main/java/org/apache/hop/imp/ImportedConnectionRewriter.java
new file mode 100644
index 0000000000..965f4e0bdd
--- /dev/null
+++ b/engine/src/main/java/org/apache/hop/imp/ImportedConnectionRewriter.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.imp;
+
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.UnaryOperator;
+import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.database.DatabaseMeta;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.util.HopMetadataPropertyWalker;
+import org.apache.hop.metadata.util.HopMetadataPropertyWalker.StringProperty;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.workflow.WorkflowMeta;
+import org.apache.hop.workflow.action.ActionMeta;
+
+/**
+ * After a Kettle import has written pipelines, workflows and relational
connections, collapse
+ * case-insensitive connection names (and optionally a naming-scheme mapping)
onto one Hop name and
+ * rewrite every {@link HopMetadataPropertyType#RDBMS_CONNECTION} field to
match.
+ */
+public final class ImportedConnectionRewriter {
+
+ private ImportedConnectionRewriter() {}
+
+ @Getter
+ public static final class Result {
+ private int connectionsRenamed;
+ private int filesRewritten;
+ private int fieldsRewritten;
+ private ConnectionNameMap nameMap = ConnectionNameMap.empty();
+ }
+
+ /**
+ * Collect names, rename {@link DatabaseMeta} objects, and rewrite imported
{@code .hpl}/{@code
+ * .hwf} files. Failures to load a single file are logged and skipped.
+ */
+ public static Result rewrite(HopImportBase hopImport) throws HopException {
+ Result result = new Result();
+ if (hopImport == null) {
+ return result;
+ }
+ IHopMetadataProvider provider = hopImport.getMetadataProvider();
+ IVariables variables = hopImport.getVariables();
+ ILogChannel log = hopImport.getLog();
+ if (provider == null) {
+ return result;
+ }
+
+ List<String> names = new ArrayList<>();
+ if (hopImport.getConnectionsList() != null) {
+ for (DatabaseMeta databaseMeta : hopImport.getConnectionsList()) {
+ if (databaseMeta != null &&
!StringUtils.isEmpty(databaseMeta.getName())) {
+ names.add(databaseMeta.getName());
+ }
+ }
+ }
+ collectReferenceNames(hopImport, names, log);
+
+ UnaryOperator<String> mapper = hopImport.getConnectionNameMapper();
+ ConnectionNameMap nameMap =
+ ConnectionNameMap.build(names, hopImport.getSharedConnectionNames(),
mapper);
+ result.nameMap = nameMap;
+ if (nameMap.isEmpty() || nameMap.changedCount() == 0) {
+ return result;
+ }
+
+ result.connectionsRenamed = renameConnections(hopImport, nameMap, log);
+ RewriteCounts counts = rewriteWrittenFiles(hopImport, nameMap, provider,
variables, log);
+ result.filesRewritten = counts.files;
+ result.fieldsRewritten = counts.fields;
+ return result;
+ }
+
+ public static int rewriteObject(Object root, ConnectionNameMap nameMap) {
+ if (root == null || nameMap == null) {
+ return 0;
+ }
+ return HopMetadataPropertyWalker.rewriteStrings(
+ root, HopMetadataPropertyType.RDBMS_CONNECTION, nameMap::targetFor);
+ }
+
+ private static void collectReferenceNames(
+ HopImportBase hopImport, List<String> names, ILogChannel log) {
+ IHopMetadataProvider provider = hopImport.getMetadataProvider();
+ IVariables variables = hopImport.getVariables();
+ for (String filename : hopImport.getWrittenHopFileNames()) {
+ try {
+ Object graph = loadGraph(filename, provider, variables);
+ if (graph == null) {
+ continue;
+ }
+ walkGraph(
+ graph,
+ node -> {
+ for (StringProperty property :
+ HopMetadataPropertyWalker.collectStrings(
+ node, HopMetadataPropertyType.RDBMS_CONNECTION)) {
+ names.add(property.value());
+ }
+ });
+ } catch (Exception e) {
+ if (log != null) {
+ log.logError("Unable to scan connection names in imported file " +
filename, e);
+ }
+ }
+ }
+ }
+
+ private static int renameConnections(
+ HopImportBase hopImport, ConnectionNameMap nameMap, ILogChannel log)
throws HopException {
+ IHopMetadataSerializer<DatabaseMeta> serializer =
+ hopImport.getMetadataProvider().getSerializer(DatabaseMeta.class);
+ int renamed = 0;
+ List<DatabaseMeta> connections = hopImport.getConnectionsList();
+ if (connections == null) {
+ return 0;
+ }
+ for (DatabaseMeta databaseMeta : connections) {
+ if (databaseMeta == null || StringUtils.isEmpty(databaseMeta.getName()))
{
+ continue;
+ }
+ String oldName = databaseMeta.getName();
+ String newName = nameMap.targetFor(oldName);
+ if (StringUtils.isEmpty(newName) || newName.equals(oldName)) {
+ continue;
+ }
+ boolean deletedOld = false;
+ try {
+ if (serializer.exists(newName) && !oldName.equalsIgnoreCase(newName)) {
+ if (log != null) {
+ log.logError(
+ "Cannot rename connection '"
+ + oldName
+ + "' to '"
+ + newName
+ + "': that name already exists");
+ }
+ continue;
+ }
+ // Delete first so a case-only rename works on case-insensitive
filesystems.
+ if (serializer.exists(oldName)) {
+ serializer.delete(oldName);
+ deletedOld = true;
+ }
+ databaseMeta.setName(newName);
+ serializer.save(databaseMeta);
+ renamed++;
+ if (log != null) {
+ log.logBasic("Renamed imported connection '" + oldName + "' to '" +
newName + "'");
+ }
+ } catch (Exception e) {
+ databaseMeta.setName(oldName);
+ if (deletedOld) {
+ try {
+ serializer.save(databaseMeta);
+ } catch (Exception ignored) {
+ // already logging the rename failure
+ }
+ }
+ if (log != null) {
+ log.logError(
+ "Error renaming imported connection '" + oldName + "' to '" +
newName + "'", e);
+ }
+ }
+ }
+ return renamed;
+ }
+
+ private static RewriteCounts rewriteWrittenFiles(
+ HopImportBase hopImport,
+ ConnectionNameMap nameMap,
+ IHopMetadataProvider provider,
+ IVariables variables,
+ ILogChannel log) {
+ RewriteCounts counts = new RewriteCounts();
+ for (String filename : hopImport.getWrittenHopFileNames()) {
+ try {
+ Object graph = loadGraph(filename, provider, variables);
+ if (graph == null) {
+ continue;
+ }
+ int[] changed = new int[1];
+ walkGraph(graph, node -> changed[0] += rewriteObject(node, nameMap));
+ if (changed[0] == 0) {
+ continue;
+ }
+ saveGraph(filename, graph, variables);
+ counts.files++;
+ counts.fields += changed[0];
+ if (log != null) {
+ log.logBasic(
+ "Updated " + changed[0] + " relational connection reference(s)
in " + filename);
+ }
+ } catch (Exception e) {
+ if (log != null) {
+ log.logError("Unable to rewrite connection names in imported file "
+ filename, e);
+ }
+ }
+ }
+ return counts;
+ }
+
+ private static Object loadGraph(
+ String filename, IHopMetadataProvider provider, IVariables variables)
throws HopException {
+ if (StringUtils.isEmpty(filename)) {
+ return null;
+ }
+ String lower = filename.toLowerCase(Locale.ROOT);
+ if (lower.endsWith(".hpl")) {
+ return new PipelineMeta(filename, provider, variables);
+ }
+ if (lower.endsWith(".hwf")) {
+ return new WorkflowMeta(variables, filename, provider);
+ }
+ return null;
+ }
+
+ private static void saveGraph(String filename, Object graph, IVariables
variables)
+ throws HopException {
+ String xml;
+ if (graph instanceof PipelineMeta pipelineMeta) {
+ xml = pipelineMeta.getXml(variables);
+ } else if (graph instanceof WorkflowMeta workflowMeta) {
+ xml = workflowMeta.getXml(variables);
+ } else {
+ return;
+ }
+ try (OutputStream out = HopVfs.getOutputStream(filename, false)) {
+ out.write(xml.getBytes(StandardCharsets.UTF_8));
+ } catch (Exception e) {
+ throw new HopException("Error writing imported file " + filename, e);
+ }
+ }
+
+ private static void walkGraph(Object graph,
java.util.function.Consumer<Object> consumer) {
+ if (graph instanceof PipelineMeta pipelineMeta) {
+ for (TransformMeta transformMeta : pipelineMeta.getTransforms()) {
+ if (transformMeta != null && transformMeta.getTransform() != null) {
+ consumer.accept(transformMeta.getTransform());
+ }
+ }
+ return;
+ }
+ if (graph instanceof WorkflowMeta workflowMeta) {
+ for (ActionMeta actionMeta : workflowMeta.getActions()) {
+ if (actionMeta != null && actionMeta.getAction() != null) {
+ consumer.accept(actionMeta.getAction());
+ }
+ }
+ }
+ }
+
+ private static final class RewriteCounts {
+ private int files;
+ private int fields;
+ }
+}
diff --git a/engine/src/test/java/org/apache/hop/imp/ConnectionNameMapTest.java
b/engine/src/test/java/org/apache/hop/imp/ConnectionNameMapTest.java
new file mode 100644
index 0000000000..81d3c033c6
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/imp/ConnectionNameMapTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.imp;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import java.util.Locale;
+import org.junit.jupiter.api.Test;
+
+class ConnectionNameMapTest {
+
+ @Test
+ void caseVariantsCollapseToOneSpelling() {
+ ConnectionNameMap map =
+ ConnectionNameMap.build(List.of("Database", "database", "DATABASE"),
List.of(), null);
+
+ assertEquals("Database", map.targetFor("Database"));
+ assertEquals("Database", map.targetFor("database"));
+ assertEquals("Database", map.targetFor("DATABASE"));
+ }
+
+ @Test
+ void preferredSharedXmlSpellingWins() {
+ ConnectionNameMap map =
+ ConnectionNameMap.build(
+ List.of("WAREHOUSE", "warehouse", "Warehouse"),
List.of("Warehouse"), null);
+
+ assertEquals("Warehouse", map.targetFor("WAREHOUSE"));
+ assertEquals("Warehouse", map.targetFor("warehouse"));
+ }
+
+ @Test
+ void mostFrequentSpellingWinsWithoutPreference() {
+ ConnectionNameMap map =
+ ConnectionNameMap.build(List.of("Sales", "sales", "sales", "SALES"),
List.of(), null);
+
+ assertEquals("sales", map.targetFor("Sales"));
+ assertEquals("sales", map.targetFor("SALES"));
+ }
+
+ @Test
+ void mapperIsAppliedToTheCanonicalOriginal() {
+ ConnectionNameMap map =
+ ConnectionNameMap.build(
+ List.of("My DB", "MY DB"),
+ List.of(),
+ value -> value.toLowerCase(Locale.ROOT).replace(' ', '_'));
+
+ assertEquals("my_db", map.targetFor("My DB"));
+ assertEquals("my_db", map.targetFor("MY DB"));
+ }
+
+ @Test
+ void distinctGroupsThatMapToTheSameTargetAreCollisions() {
+ ConnectionNameMap map =
+ ConnectionNameMap.build(
+ List.of("My-DB", "My_DB"),
+ List.of(),
+ value -> value.toLowerCase(Locale.ROOT).replace('-', '_'));
+
+ assertEquals("my_db", map.targetFor("My-DB"));
+ assertEquals("my_db", map.targetFor("My_DB"));
+ assertEquals(1, map.getCollisions().size());
+ }
+
+ @Test
+ void variableNamesAreLeftAlone() {
+ ConnectionNameMap map =
+ ConnectionNameMap.build(List.of("${CONN}", "Warehouse"), List.of(),
String::toLowerCase);
+
+ assertEquals("${CONN}", map.targetFor("${CONN}"));
+ assertTrue(ConnectionNameMap.shouldSkip("${CONN}"));
+ assertEquals("warehouse", map.targetFor("Warehouse"));
+ }
+
+ @Test
+ void unknownNamesPassThrough() {
+ ConnectionNameMap map = ConnectionNameMap.build(List.of("Warehouse"),
List.of(), null);
+ assertEquals("Other", map.targetFor("Other"));
+ }
+}
diff --git
a/engine/src/test/java/org/apache/hop/imp/ImportedConnectionRewriterTest.java
b/engine/src/test/java/org/apache/hop/imp/ImportedConnectionRewriterTest.java
new file mode 100644
index 0000000000..e6023ddb6e
--- /dev/null
+++
b/engine/src/test/java/org/apache/hop/imp/ImportedConnectionRewriterTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.imp;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.util.List;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.database.DatabaseMeta;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.HopMetadataPropertyType;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class ImportedConnectionRewriterTest {
+
+ @BeforeAll
+ static void init() throws Exception {
+ HopClientEnvironment.init();
+ }
+
+ static class SampleMeta {
+ @HopMetadataProperty(hopMetadataPropertyType =
HopMetadataPropertyType.RDBMS_CONNECTION)
+ String connection = "DATABASE";
+
+ @HopMetadataProperty(key = "other")
+ String other = "leave";
+ }
+
+ @Test
+ void rewriteObjectUsesTheNameMap() {
+ SampleMeta meta = new SampleMeta();
+ ConnectionNameMap map =
+ ConnectionNameMap.build(List.of("Database", "DATABASE"),
List.of("Database"), null);
+
+ int changed = ImportedConnectionRewriter.rewriteObject(meta, map);
+
+ assertEquals(1, changed);
+ assertEquals("Database", meta.connection);
+ assertEquals("leave", meta.other);
+ }
+
+ @Test
+ void rewriteRenamesCaseOnlyConnectionMetadata() throws Exception {
+ MemoryMetadataProvider memory = new MemoryMetadataProvider();
+ IHopMetadataSerializer<DatabaseMeta> serializer =
memory.getSerializer(DatabaseMeta.class);
+ DatabaseMeta databaseMeta = new DatabaseMeta();
+ databaseMeta.setName("sales");
+ serializer.save(databaseMeta);
+
+ TestImport hopImport = new TestImport();
+ hopImport.setMetadataProvider(
+ new MultiMetadataProvider(null, List.of(memory),
hopImport.getVariables()));
+ hopImport.getConnectionsList().add(databaseMeta);
+ hopImport.getSharedConnectionNames().add("Sales");
+
+ ImportedConnectionRewriter.Result result =
ImportedConnectionRewriter.rewrite(hopImport);
+
+ assertEquals(1, result.getConnectionsRenamed());
+ assertNull(serializer.load("sales"));
+ assertEquals("Sales", serializer.load("Sales").getName());
+ }
+
+ @Test
+ void rewriteSkipsWorkWhenNamesAlreadyMatch() throws Exception {
+ MemoryMetadataProvider memory = new MemoryMetadataProvider();
+ DatabaseMeta databaseMeta = new DatabaseMeta();
+ databaseMeta.setName("Sales");
+ memory.getSerializer(DatabaseMeta.class).save(databaseMeta);
+
+ TestImport hopImport = new TestImport();
+ hopImport.setMetadataProvider(
+ new MultiMetadataProvider(null, List.of(memory),
hopImport.getVariables()));
+ hopImport.getConnectionsList().add(databaseMeta);
+
+ ImportedConnectionRewriter.Result result =
ImportedConnectionRewriter.rewrite(hopImport);
+
+ assertEquals(0, result.getConnectionsRenamed());
+ assertEquals("Sales",
memory.getSerializer(DatabaseMeta.class).load("Sales").getName());
+ }
+
+ private static final class TestImport extends HopImportBase {
+ @Override
+ public void importFiles() {}
+
+ @Override
+ public void findFilesToImport() {}
+
+ @Override
+ public void importConnections() {}
+
+ @Override
+ public void importVariables() {}
+
+ @Override
+ public String getImportReport() {
+ return "";
+ }
+ }
+}
diff --git
a/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImport.java
b/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImport.java
index c04b1f72af..142848bc91 100644
---
a/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImport.java
+++
b/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImport.java
@@ -63,6 +63,7 @@ import org.apache.hop.core.xml.XmlFormatter;
import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.core.xml.XmlParserFactoryProducer;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.imp.ConnectionNameMap;
import org.apache.hop.imp.HopImportBase;
import org.apache.hop.imp.IHopImport;
import org.apache.hop.imp.ImportPlugin;
@@ -315,6 +316,7 @@ public class KettleImport extends HopImportBase implements
IHopImport {
try (OutputStream fileStream =
HopVfs.getOutputStream(targetFilename, false)) {
fileStream.write(xml.getBytes(StandardCharsets.UTF_8));
}
+ writtenHopFileNames.add(targetFilename);
}
}
}
@@ -329,6 +331,10 @@ public class KettleImport extends HopImportBase implements
IHopImport {
collectConnectionsFromSharedXml();
collectConnectionsFromJdbcProperties();
importCollectedConnections();
+ }
+
+ @Override
+ protected void afterConnectionRewrite() throws HopException {
saveConnectionsReport();
}
@@ -338,11 +344,27 @@ public class KettleImport extends HopImportBase
implements IHopImport {
this.connectionsReportFileName = getOutputFolderName() +
"/connections.csv";
try (OutputStream outputStream =
HopVfs.getOutputStream(this.connectionsReportFileName, false)) {
+ writeCsvRow(outputStream, "file", "original_name", "target_name",
"note");
+ ConnectionNameMap nameMap =
+ connectionRewriteResult != null ?
connectionRewriteResult.getNameMap() : null;
for (Map.Entry<String, String> entry : connectionFileMap.entrySet()) {
- outputStream.write(entry.getKey().getBytes(StandardCharsets.UTF_8));
- outputStream.write(",".getBytes(StandardCharsets.UTF_8));
-
outputStream.write(entry.getValue().getBytes(StandardCharsets.UTF_8));
- outputStream.write(Const.CR.getBytes(StandardCharsets.UTF_8));
+ String original = entry.getValue();
+ String target = nameMap != null ? nameMap.targetFor(original) :
original;
+ writeCsvRow(outputStream, entry.getKey(), original, target, "");
+ }
+ if (nameMap != null) {
+ for (ConnectionNameMap.Collision collision :
nameMap.getCollisions()) {
+ writeCsvRow(
+ outputStream,
+ "",
+ "",
+ collision.targetName(),
+ "collision: '"
+ + collision.leftOriginal()
+ + "' and '"
+ + collision.rightOriginal()
+ + "'");
+ }
}
} catch (IOException e) {
throw new HopException("Error writing connections.csv file to
project", e);
@@ -350,6 +372,27 @@ public class KettleImport extends HopImportBase implements
IHopImport {
}
}
+ static void writeCsvRow(OutputStream outputStream, String... fields) throws
IOException {
+ for (int i = 0; i < fields.length; i++) {
+ if (i > 0) {
+ outputStream.write(',');
+ }
+ outputStream.write(csvField(fields[i]).getBytes(StandardCharsets.UTF_8));
+ }
+ outputStream.write(Const.CR.getBytes(StandardCharsets.UTF_8));
+ }
+
+ static String csvField(String value) {
+ String field = Const.NVL(value, "");
+ if (field.indexOf(',') >= 0
+ || field.indexOf('"') >= 0
+ || field.indexOf('\n') >= 0
+ || field.indexOf('\r') >= 0) {
+ return '"' + field.replace("\"", "\"\"") + '"';
+ }
+ return field;
+ }
+
private void importCollectedConnections() throws HopException {
// Simply add the collected connections to the metadata provider...
//
@@ -364,8 +407,13 @@ public class KettleImport extends HopImportBase implements
IHopImport {
if (StringUtils.isEmpty(sharedXmlFilename)) {
return;
}
- Document doc = getDocFromFile(HopVfs.getFileObject(sharedXmlFilename));
- importDbConnections(doc, HopVfs.getFileObject(sharedXmlFilename));
+ collectingFromSharedXml = true;
+ try {
+ Document doc = getDocFromFile(HopVfs.getFileObject(sharedXmlFilename));
+ importDbConnections(doc, HopVfs.getFileObject(sharedXmlFilename));
+ } finally {
+ collectingFromSharedXml = false;
+ }
}
public void collectConnectionsFromJdbcProperties() throws HopException {
@@ -1165,6 +1213,30 @@ public class KettleImport extends HopImportBase
implements IHopImport {
messageString +=
"Connections with the same name and different configurations have
only been saved once."
+ eol;
+ if (appliedNamingSchemeName != null) {
+ messageString +=
+ "Relational connection names were rewritten with naming scheme '"
+ + appliedNamingSchemeName
+ + "'."
+ + eol;
+ } else {
+ messageString +=
+ "Relational connection names were aligned to a single
case-sensitive spelling." + eol;
+ }
+ if (connectionRewriteResult != null) {
+ messageString +=
+ connectionRewriteResult.getConnectionsRenamed()
+ + " connection metadata object(s) renamed, "
+ + connectionRewriteResult.getFilesRewritten()
+ + " pipeline/workflow file(s) updated."
+ + eol;
+ if (!connectionRewriteResult.getNameMap().getCollisions().isEmpty()) {
+ messageString +=
+ connectionRewriteResult.getNameMap().getCollisions().size()
+ + " name collision(s) were recorded (distinct connections
mapping to the same target name)."
+ + eol;
+ }
+ }
messageString +=
"Check the following file for a list of connections that might need
extra attention: "
+ getConnectionsReportFileName();
diff --git
a/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImportDialog.java
b/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImportDialog.java
index bfea79558d..44071ce90d 100644
---
a/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImportDialog.java
+++
b/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleImportDialog.java
@@ -18,19 +18,27 @@
package org.apache.hop.imports.kettle;
import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import org.apache.commons.lang3.StringUtils;
import org.apache.hop.core.Const;
+import org.apache.hop.core.Props;
+import org.apache.hop.core.encryption.Encr;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.extension.ExtensionPointHandler;
-import org.apache.hop.core.extension.HopExtensionPoint;
import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.util.SingletonUtil;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.imp.HopImportBase;
+import org.apache.hop.metadata.api.IHopMetadata;
import org.apache.hop.metadata.api.IHopMetadataProvider;
-import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.serializer.json.JsonMetadataProvider;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
import org.apache.hop.pipeline.config.PipelineRunConfiguration;
import org.apache.hop.ui.core.PropsUi;
import org.apache.hop.ui.core.dialog.BaseDialog;
@@ -39,6 +47,7 @@ import org.apache.hop.ui.core.dialog.MessageBox;
import org.apache.hop.ui.core.dialog.ProgressMonitorDialog;
import org.apache.hop.ui.core.gui.GuiResource;
import org.apache.hop.ui.core.gui.WindowProperty;
+import org.apache.hop.ui.core.widget.MetaSelectionLine;
import org.apache.hop.ui.core.widget.TextVar;
import org.apache.hop.ui.hopgui.HopGui;
import org.apache.hop.ui.hopgui.shared.AuditManagerGuiUtil;
@@ -46,15 +55,19 @@ import
org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
import org.apache.hop.ui.util.EnvironmentUtils;
import org.apache.hop.workflow.config.WorkflowRunConfiguration;
import org.eclipse.swt.SWT;
+import org.eclipse.swt.custom.CTabFolder;
+import org.eclipse.swt.custom.CTabItem;
import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
import org.eclipse.swt.layout.FormLayout;
import org.eclipse.swt.widgets.Button;
import org.eclipse.swt.widgets.Combo;
+import org.eclipse.swt.widgets.Composite;
import org.eclipse.swt.widgets.Control;
import org.eclipse.swt.widgets.Dialog;
import org.eclipse.swt.widgets.Event;
import org.eclipse.swt.widgets.Label;
+import org.eclipse.swt.widgets.Listener;
import org.eclipse.swt.widgets.Shell;
public class KettleImportDialog extends Dialog {
@@ -76,6 +89,8 @@ public class KettleImportDialog extends Dialog {
"ImportPipelineRunConfiguration";
public static final String LAST_USED_IMPORT_WORKFLOW_RUN_CONFIGURATION =
"ImportWorkflowRunConfiguration";
+ public static final String LAST_USED_IMPORT_NAMING_SCHEME =
"ImportNamingScheme";
+ public static final String NAMING_SCHEME_METADATA_KEY = "naming-scheme";
public static final String CONST_FALSE = "false";
public static final String CONST_ALL_FILES = "All Files (*.*)";
public static final String CONST_KETTLE_IMPORT_DIALOG_BUTTON_BROWSE =
@@ -103,8 +118,15 @@ public class KettleImportDialog extends Dialog {
private TextVar wJdbcProps;
private TextVar wTargetConfigFile;
- private Combo wPipelineRunConfiguration;
- private Combo wWorkflowRunConfiguration;
+ private MetaSelectionLine<PipelineRunConfiguration>
wPipelineRunConfiguration;
+ private MetaSelectionLine<WorkflowRunConfiguration>
wWorkflowRunConfiguration;
+
+ @SuppressWarnings("rawtypes")
+ private MetaSelectionLine wNamingScheme;
+
+ private MemoryMetadataProvider scratchMetadata;
+ private IHopMetadataProvider dialogMetadataProvider;
+ private String boundMetadataFolder;
private Combo wImportProject;
private Button wImportInExisting;
private Button wbImportPath;
@@ -112,6 +134,9 @@ public class KettleImportDialog extends Dialog {
private Button wSkipHidden;
private Button wSkipFolders;
+ private int margin;
+ private int middle;
+
public KettleImportDialog(Shell parent, IVariables variables, KettleImport
kettleImport)
throws HopException {
super(parent, SWT.DIALOG_TRIM | SWT.APPLICATION_MODAL | SWT.RESIZE);
@@ -121,6 +146,9 @@ public class KettleImportDialog extends Dialog {
this.variables = variables;
this.kettleImport = kettleImport;
this.configuredSourceFolder = kettleImport.getInputFolderName();
+ this.scratchMetadata = new MemoryMetadataProvider(Encr.getEncoder(),
variables);
+ this.dialogMetadataProvider =
+ new MultiMetadataProvider(Encr.getEncoder(), List.of(scratchMetadata),
variables);
try {
projectNames =
@@ -140,8 +168,8 @@ public class KettleImportDialog extends Dialog {
shell.setImage(GuiResource.getInstance().getImageHopUi());
PropsUi.setLook(shell);
- int margin = PropsUi.getMargin() + 2;
- int middle = props.getMiddlePct();
+ margin = PropsUi.getMargin() + 2;
+ middle = props.getMiddlePct();
FormLayout formLayout = new FormLayout();
formLayout.marginWidth = PropsUi.getFormMargin();
@@ -149,379 +177,27 @@ public class KettleImportDialog extends Dialog {
shell.setLayout(formLayout);
shell.setText(BaseMessages.getString(PKG,
"KettleImportDialog.Shell.Name"));
- // Select folder to import from
- Label wlImportFrom = new Label(shell, SWT.RIGHT);
- PropsUi.setLook(wlImportFrom);
- wlImportFrom.setText(BaseMessages.getString(PKG,
"KettleImportDialog.Label.ImportFrom"));
- FormData fdlImportFrom = new FormData();
- fdlImportFrom.left = new FormAttachment(0, 0);
- fdlImportFrom.right = new FormAttachment(middle, 0);
- fdlImportFrom.top = new FormAttachment(0, margin);
- wlImportFrom.setLayoutData(fdlImportFrom);
-
- Button wbImportFrom = new Button(shell, SWT.PUSH);
- PropsUi.setLook(wbImportFrom);
- wbImportFrom.setText(BaseMessages.getString(PKG,
CONST_KETTLE_IMPORT_DIALOG_BUTTON_BROWSE));
- FormData fdbImportFrom = new FormData();
- fdbImportFrom.right = new FormAttachment(100, 0);
- fdbImportFrom.top = new FormAttachment(wlImportFrom, 0, SWT.CENTER);
- wbImportFrom.setLayoutData(fdbImportFrom);
- wbImportFrom.addListener(SWT.Selection, this::browseHomeFolder);
-
- wImportFrom = new TextVar(variables, shell, SWT.SINGLE | SWT.BORDER |
SWT.LEFT);
- PropsUi.setLook(wImportFrom);
- FormData fdImportFrom = new FormData();
- fdImportFrom.left = new FormAttachment(middle, margin);
- fdImportFrom.right = new FormAttachment(wbImportFrom, -margin);
- fdImportFrom.top = new FormAttachment(wlImportFrom, 0, SWT.CENTER);
- wImportFrom.setLayoutData(fdImportFrom);
- Control lastControl = wImportFrom;
-
- // Import in existing project?
- Label wlImportInExisting = new Label(shell, SWT.RIGHT);
- PropsUi.setLook(wlImportInExisting);
- wlImportInExisting.setText(
- BaseMessages.getString(PKG,
"KettleImportDialog.Label.ImportInExistingProject"));
- FormData fdlImportInExisting = new FormData();
- fdlImportInExisting.left = new FormAttachment(0, 0);
- fdlImportInExisting.right = new FormAttachment(middle, 0);
- fdlImportInExisting.top = new FormAttachment(lastControl, margin);
- wlImportInExisting.setLayoutData(fdlImportInExisting);
-
- wImportInExisting = new Button(shell, SWT.CHECK);
- wImportInExisting.setSelection(true);
- PropsUi.setLook(wImportInExisting);
- FormData fdcbImportInExisting = new FormData();
- fdcbImportInExisting.left = new FormAttachment(middle, margin);
- fdcbImportInExisting.right = new FormAttachment(100, 0);
- fdcbImportInExisting.top = new FormAttachment(wlImportInExisting, 0,
SWT.CENTER);
- wImportInExisting.setLayoutData(fdcbImportInExisting);
- wImportInExisting.setSelection(true);
- wImportInExisting.addListener(SWT.Selection, this::showHideProjectFields);
- lastControl = wlImportInExisting;
-
- // Import in project
- Label wlImportProject = new Label(shell, SWT.RIGHT);
- PropsUi.setLook(wlImportProject);
- wlImportProject.setText(
- BaseMessages.getString(PKG,
"KettleImportDialog.Label.ImportInProject"));
- FormData fdlImportProject = new FormData();
- fdlImportProject.left = new FormAttachment(0, 0);
- fdlImportProject.right = new FormAttachment(middle, 0);
- fdlImportProject.top = new FormAttachment(lastControl, margin);
- wlImportProject.setLayoutData(fdlImportProject);
-
- wImportProject = new Combo(shell, SWT.READ_ONLY);
- wImportProject.setItems(projectNames.toArray(new
String[projectNames.size()]));
- PropsUi.setLook(wImportProject);
- FormData fdImportProject = new FormData();
- fdImportProject.left = new FormAttachment(middle, margin);
- fdImportProject.right = new FormAttachment(100, 0);
- fdImportProject.top = new FormAttachment(wlImportProject, 0, SWT.CENTER);
- wImportProject.setLayoutData(fdImportProject);
- lastControl = wlImportProject;
-
- // Import in path
- Label wlImportPath = new Label(shell, SWT.RIGHT);
- PropsUi.setLook(wlImportPath);
- wlImportPath.setText(BaseMessages.getString(PKG,
"KettleImportDialog.Label.ImportToFolder"));
- FormData fdlImportPath = new FormData();
- fdlImportPath.left = new FormAttachment(0, 0);
- fdlImportPath.right = new FormAttachment(middle, 0);
- fdlImportPath.top = new FormAttachment(lastControl, margin);
- wlImportPath.setLayoutData(fdlImportPath);
-
- wbImportPath = new Button(shell, SWT.PUSH);
- PropsUi.setLook(wbImportPath);
- wbImportPath.setText(BaseMessages.getString(PKG,
CONST_KETTLE_IMPORT_DIALOG_BUTTON_BROWSE));
- FormData fdbImportPath = new FormData();
- fdbImportPath.right = new FormAttachment(100, 0);
- fdbImportPath.top = new FormAttachment(wlImportPath, 0, SWT.CENTER);
- wbImportPath.setLayoutData(fdbImportPath);
- wbImportPath.setEnabled(false);
- wbImportPath.addListener(SWT.Selection, this::browseTargetFolder);
-
- wImportPath = new TextVar(variables, shell, SWT.SINGLE | SWT.BORDER |
SWT.LEFT);
- PropsUi.setLook(wImportPath);
- FormData fdImportPath = new FormData();
- fdImportPath.left = new FormAttachment(middle, margin);
- fdImportPath.right = new FormAttachment(wbImportPath, -margin);
- fdImportPath.top = new FormAttachment(wlImportPath, 0, SWT.CENTER);
- wImportPath.setLayoutData(fdImportPath);
- wImportPath.setEditable(false);
- lastControl = wImportPath;
-
- // Kettle properties path
- Label wlKettleProps = new Label(shell, SWT.RIGHT);
- PropsUi.setLook(wlKettleProps);
- wlKettleProps.setText(
- BaseMessages.getString(PKG,
"KettleImportDialog.Label.PathToKettleProperties"));
- FormData fdlKettleProps = new FormData();
- fdlKettleProps.left = new FormAttachment(0, 0);
- fdlKettleProps.right = new FormAttachment(middle, 0);
- fdlKettleProps.top = new FormAttachment(lastControl, margin);
- wlKettleProps.setLayoutData(fdlKettleProps);
-
- Button wbKettleProps = new Button(shell, SWT.PUSH);
- PropsUi.setLook(wbKettleProps);
- wbKettleProps.setText(BaseMessages.getString(PKG,
CONST_KETTLE_IMPORT_DIALOG_BUTTON_BROWSE));
- FormData fdbKettleProps = new FormData();
- fdbKettleProps.right = new FormAttachment(100, 0);
- fdbKettleProps.top = new FormAttachment(wlKettleProps, 0, SWT.CENTER);
- wbKettleProps.setLayoutData(fdbKettleProps);
- wbKettleProps.addListener(SWT.Selection, this::browseKettlePropsFile);
-
- wKettleProps = new TextVar(variables, shell, SWT.SINGLE | SWT.BORDER |
SWT.LEFT);
- PropsUi.setLook(wKettleProps);
- FormData fdKettleProps = new FormData();
- fdKettleProps.left = new FormAttachment(middle, margin);
- fdKettleProps.right = new FormAttachment(wbKettleProps, -margin);
- fdKettleProps.top = new FormAttachment(wlKettleProps, 0, SWT.CENTER);
- wKettleProps.setLayoutData(fdKettleProps);
- lastControl = wKettleProps;
-
- // Shared.xml path
- Label wlShared = new Label(shell, SWT.RIGHT);
- PropsUi.setLook(wlShared);
- wlShared.setText(BaseMessages.getString(PKG,
"KettleImportDialog.Label.PathToSharedXml"));
- FormData fdlShared = new FormData();
- fdlShared.left = new FormAttachment(0, 0);
- fdlShared.right = new FormAttachment(middle, 0);
- fdlShared.top = new FormAttachment(lastControl, margin);
- wlShared.setLayoutData(fdlShared);
-
- Button wbShared = new Button(shell, SWT.PUSH);
- wbShared.setText(BaseMessages.getString(PKG,
CONST_KETTLE_IMPORT_DIALOG_BUTTON_BROWSE));
- FormData fdbShared = new FormData();
- fdbShared.right = new FormAttachment(100, 0);
- fdbShared.top = new FormAttachment(wlShared, 0, SWT.CENTER);
- wbShared.setLayoutData(fdbShared);
- wbShared.addListener(SWT.Selection, this::browseXmlFile);
-
- wShared = new TextVar(variables, shell, SWT.SINGLE | SWT.BORDER |
SWT.LEFT);
- PropsUi.setLook(wShared);
- FormData fdShared = new FormData();
- fdShared.left = new FormAttachment(middle, margin);
- fdShared.right = new FormAttachment(wbShared, -margin);
- fdShared.top = new FormAttachment(wlShared, 0, SWT.CENTER);
- wShared.setLayoutData(fdShared);
- lastControl = wShared;
-
- // Jdbc properties path
- Label wlJdbcProps = new Label(shell, SWT.RIGHT);
- PropsUi.setLook(wlJdbcProps);
- wlJdbcProps.setText(
- BaseMessages.getString(PKG,
"KettleImportDialog.Label.PathToJDBCProperties"));
- FormData fdlJdbcProps = new FormData();
- fdlJdbcProps.left = new FormAttachment(0, 0);
- fdlJdbcProps.right = new FormAttachment(middle, 0);
- fdlJdbcProps.top = new FormAttachment(lastControl, margin);
- wlJdbcProps.setLayoutData(fdlJdbcProps);
-
- Button wbJdbcProps = new Button(shell, SWT.PUSH);
- PropsUi.setLook(wbJdbcProps);
- wbJdbcProps.setText(BaseMessages.getString(PKG,
CONST_KETTLE_IMPORT_DIALOG_BUTTON_BROWSE));
- FormData fdbJdbcProps = new FormData();
- fdbJdbcProps.right = new FormAttachment(100, 0);
- fdbJdbcProps.top = new FormAttachment(wlJdbcProps, 0, SWT.CENTER);
- wbJdbcProps.setLayoutData(fdbJdbcProps);
- wbJdbcProps.addListener(SWT.Selection, this::browseJdbcPropsFile);
-
- wJdbcProps = new TextVar(variables, shell, SWT.SINGLE | SWT.BORDER |
SWT.LEFT);
- PropsUi.setLook(wJdbcProps);
- FormData fdJdbcProps = new FormData();
- fdJdbcProps.left = new FormAttachment(middle, margin);
- fdJdbcProps.right = new FormAttachment(wbJdbcProps, -margin);
- fdJdbcProps.top = new FormAttachment(wlJdbcProps, 0, SWT.CENTER);
- wJdbcProps.setLayoutData(fdJdbcProps);
- lastControl = wJdbcProps;
-
- // Skip existing target files?
- Label wlSkipExisting = new Label(shell, SWT.RIGHT);
- PropsUi.setLook(wlSkipExisting);
- wlSkipExisting.setText(
- BaseMessages.getString(PKG,
"KettleImportDialog.Label.SkipExistingTargetFiles"));
- FormData fdlSkipExisting = new FormData();
- fdlSkipExisting.left = new FormAttachment(0, 0);
- fdlSkipExisting.right = new FormAttachment(middle, 0);
- fdlSkipExisting.top = new FormAttachment(lastControl, margin);
- wlSkipExisting.setLayoutData(fdlSkipExisting);
-
- wSkipExisting = new Button(shell, SWT.CHECK);
- PropsUi.setLook(wSkipExisting);
- FormData fdSkipExisting = new FormData();
- fdSkipExisting.left = new FormAttachment(middle, margin);
- fdSkipExisting.right = new FormAttachment(100, 0);
- fdSkipExisting.top = new FormAttachment(wlSkipExisting, 0, SWT.CENTER);
- wSkipExisting.setLayoutData(fdSkipExisting);
- wSkipExisting.setSelection(true);
- lastControl = wlSkipExisting;
-
- // Skip existing target files?
- Label wlSkipHidden = new Label(shell, SWT.RIGHT);
- PropsUi.setLook(wlSkipHidden);
- wlSkipHidden.setText(BaseMessages.getString(PKG,
"KettleImportDialog.Label.SkipHiddenFiles"));
- FormData fdlSkipHidden = new FormData();
- fdlSkipHidden.left = new FormAttachment(0, 0);
- fdlSkipHidden.right = new FormAttachment(middle, 0);
- fdlSkipHidden.top = new FormAttachment(lastControl, margin);
- wlSkipHidden.setLayoutData(fdlSkipHidden);
-
- wSkipHidden = new Button(shell, SWT.CHECK);
- PropsUi.setLook(wSkipHidden);
- FormData fdSkipHidden = new FormData();
- fdSkipHidden.left = new FormAttachment(middle, margin);
- fdSkipHidden.right = new FormAttachment(100, 0);
- fdSkipHidden.top = new FormAttachment(wlSkipHidden, 0, SWT.CENTER);
- wSkipHidden.setLayoutData(fdSkipHidden);
- wSkipHidden.setSelection(true);
- lastControl = wlSkipHidden;
-
- // Skip existing target files?
- Label wlSkipFolders = new Label(shell, SWT.RIGHT);
- PropsUi.setLook(wlSkipFolders);
- wlSkipFolders.setText(BaseMessages.getString(PKG,
"KettleImportDialog.Label.SkipFolders"));
- FormData fdlSkipFolders = new FormData();
- fdlSkipFolders.left = new FormAttachment(0, 0);
- fdlSkipFolders.right = new FormAttachment(middle, 0);
- fdlSkipFolders.top = new FormAttachment(lastControl, margin);
- wlSkipFolders.setLayoutData(fdlSkipFolders);
-
- wSkipFolders = new Button(shell, SWT.CHECK);
- PropsUi.setLook(wSkipFolders);
- FormData fdSkipFolders = new FormData();
- fdSkipFolders.left = new FormAttachment(middle, margin);
- fdSkipFolders.right = new FormAttachment(100, 0);
- fdSkipFolders.top = new FormAttachment(wlSkipFolders, 0, SWT.CENTER);
- wSkipFolders.setLayoutData(fdSkipFolders);
- wSkipFolders.setSelection(true);
- wSkipFolders.addListener(SWT.Selection, this::showHideProjectFields);
- lastControl = wlSkipFolders;
-
- // Target environment configuration file
- Label wlTargetConfigFile = new Label(shell, SWT.RIGHT);
- PropsUi.setLook(wlTargetConfigFile);
- wlTargetConfigFile.setText(
- BaseMessages.getString(PKG,
"KettleImportDialog.Label.TargetConfigFile"));
- FormData fdlTargetConfigFile = new FormData();
- fdlTargetConfigFile.left = new FormAttachment(0, 0);
- fdlTargetConfigFile.right = new FormAttachment(middle, 0);
- fdlTargetConfigFile.top = new FormAttachment(lastControl, margin);
- wlTargetConfigFile.setLayoutData(fdlTargetConfigFile);
-
- wTargetConfigFile = new TextVar(variables, shell, SWT.SINGLE | SWT.BORDER
| SWT.LEFT);
- PropsUi.setLook(wTargetConfigFile);
- FormData fdTargetConfigFile = new FormData();
- fdTargetConfigFile.left = new FormAttachment(middle, margin);
- fdTargetConfigFile.right = new FormAttachment(100, 0);
- fdTargetConfigFile.top = new FormAttachment(wlTargetConfigFile, 0,
SWT.CENTER);
- wTargetConfigFile.setLayoutData(fdTargetConfigFile);
- wTargetConfigFile.setEditable(false);
-
- lastControl = wTargetConfigFile;
-
- Label wlPipelineRunConfiguration = new Label(shell, SWT.RIGHT);
- wlPipelineRunConfiguration.setText(
- BaseMessages.getString(PKG,
"KettleImportDialog.Pipeline.RunConfiguration.Label"));
- PropsUi.setLook(wlPipelineRunConfiguration);
- FormData fdlPipelineRunConfiguration = new FormData();
- fdlPipelineRunConfiguration.left = new FormAttachment(0, 0);
- fdlPipelineRunConfiguration.right = new FormAttachment(middle, 0);
- fdlPipelineRunConfiguration.top = new FormAttachment(lastControl, margin);
- wlPipelineRunConfiguration.setLayoutData(fdlPipelineRunConfiguration);
-
- wPipelineRunConfiguration = new Combo(shell, SWT.READ_ONLY);
- PropsUi.setLook(wlPipelineRunConfiguration);
- FormData fdPipelineRunConfiguration = new FormData();
- fdPipelineRunConfiguration.left = new FormAttachment(middle, margin);
- fdPipelineRunConfiguration.top = new
FormAttachment(wlPipelineRunConfiguration, 0, SWT.CENTER);
- fdPipelineRunConfiguration.right = new FormAttachment(100, 0);
- wPipelineRunConfiguration.setLayoutData(fdPipelineRunConfiguration);
- PropsUi.setLook(wPipelineRunConfiguration);
-
- HopGui hopGui = HopGui.getInstance();
- IHopMetadataProvider metadataProvider = hopGui.getMetadataProvider();
-
- try {
- List<String> runConfigurations =
-
metadataProvider.getSerializer(PipelineRunConfiguration.class).listObjectNames();
-
- try {
- ExtensionPointHandler.callExtensionPoint(
- HopGui.getInstance().getLog(),
- variables,
- HopExtensionPoint.HopGuiRunConfiguration.id,
- new Object[] {runConfigurations, PipelineMeta.XML_TAG});
- } catch (HopException e) {
- // Ignore errors
- }
-
- wPipelineRunConfiguration.setItems(runConfigurations.toArray(new
String[0]));
- } catch (Exception e) {
- LogChannel.UI.logError("Error getting pipeline run configurations", e);
- }
-
- lastControl = wPipelineRunConfiguration;
-
- Label wlWorkflowRunConfiguration = new Label(shell, SWT.RIGHT);
- wlWorkflowRunConfiguration.setText(
- BaseMessages.getString(PKG,
"KettleImportDialog.Workflow.RunConfiguration.Label"));
- PropsUi.setLook(wlWorkflowRunConfiguration);
- FormData fdlWorkflowRunConfiguration = new FormData();
- fdlWorkflowRunConfiguration.left = new FormAttachment(0, 0);
- fdlWorkflowRunConfiguration.right = new FormAttachment(middle, 0);
- fdlWorkflowRunConfiguration.top = new FormAttachment(lastControl, margin);
- wlWorkflowRunConfiguration.setLayoutData(fdlWorkflowRunConfiguration);
-
- wWorkflowRunConfiguration = new Combo(shell, SWT.READ_ONLY);
- PropsUi.setLook(wlWorkflowRunConfiguration);
- FormData fdWorkflowRunConfiguration = new FormData();
- fdWorkflowRunConfiguration.left = new FormAttachment(middle, margin);
- fdWorkflowRunConfiguration.top = new
FormAttachment(wlWorkflowRunConfiguration, 0, SWT.CENTER);
- fdWorkflowRunConfiguration.right = new FormAttachment(100, 0);
- wWorkflowRunConfiguration.setLayoutData(fdWorkflowRunConfiguration);
- PropsUi.setLook(wWorkflowRunConfiguration);
-
- try {
- List<String> runConfigurations =
-
metadataProvider.getSerializer(WorkflowRunConfiguration.class).listObjectNames();
-
- try {
- ExtensionPointHandler.callExtensionPoint(
- HopGui.getInstance().getLog(),
- variables,
- HopExtensionPoint.HopGuiRunConfiguration.id,
- new Object[] {runConfigurations, PipelineMeta.XML_TAG});
- } catch (HopException e) {
- // Ignore errors
- }
-
- wWorkflowRunConfiguration.setItems(runConfigurations.toArray(new
String[0]));
- } catch (Exception e) {
- LogChannel.UI.logError("Error getting workflow run configurations", e);
- }
-
- lastControl = wWorkflowRunConfiguration;
-
- Label separator = new Label(shell, SWT.HORIZONTAL | SWT.SEPARATOR);
- FormData fdLine = new FormData();
- fdLine.height = 5;
- fdLine.left = new FormAttachment(0, 0);
- fdLine.right = new FormAttachment(100, 0);
- fdLine.top = new FormAttachment(lastControl, margin);
- separator.setLayoutData(fdLine);
- lastControl = separator;
-
- // Buttons go at the bottom of the dialog
- //
Button wImport = new Button(shell, SWT.PUSH);
wImport.setText("Import");
wImport.addListener(SWT.Selection, event -> doImport());
Button wCancel = new Button(shell, SWT.PUSH);
wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel"));
wCancel.addListener(SWT.Selection, event -> dispose());
- BaseTransformDialog.positionBottomButtons(
- shell, new Button[] {wImport, wCancel}, margin, lastControl);
+ BaseTransformDialog.positionBottomButtons(shell, new Button[] {wImport,
wCancel}, margin, null);
+
+ CTabFolder wTabFolder = new CTabFolder(shell, SWT.BORDER);
+ PropsUi.setLook(wTabFolder, Props.WIDGET_STYLE_TAB);
+ FormData fdTabFolder = new FormData();
+ fdTabFolder.left = new FormAttachment(0, 0);
+ fdTabFolder.top = new FormAttachment(0, 0);
+ fdTabFolder.right = new FormAttachment(100, 0);
+ fdTabFolder.bottom = new FormAttachment(wImport, -margin);
+ wTabFolder.setLayoutData(fdTabFolder);
+
+ addSourceTab(wTabFolder);
+ addTargetTab(wTabFolder);
+ addMetadataTab(wTabFolder);
+ wTabFolder.setSelection(0);
// See if we need to remember previous settings...
//
@@ -564,6 +240,19 @@ public class KettleImportDialog extends Dialog {
!CONST_FALSE.equalsIgnoreCase(
AuditManagerGuiUtil.getLastUsedValue(LAST_USED_IMPORT_SKIP_FOLDERS)));
+ showHideProjectFields(null);
+ bindTargetMetadataProvider();
+ wPipelineRunConfiguration.setText(
+ Const.NVL(
+
AuditManagerGuiUtil.getLastUsedValue(LAST_USED_IMPORT_PIPELINE_RUN_CONFIGURATION),
""));
+ wWorkflowRunConfiguration.setText(
+ Const.NVL(
+
AuditManagerGuiUtil.getLastUsedValue(LAST_USED_IMPORT_WORKFLOW_RUN_CONFIGURATION),
""));
+ if (wNamingScheme != null) {
+ wNamingScheme.setText(
+
Const.NVL(AuditManagerGuiUtil.getLastUsedValue(LAST_USED_IMPORT_NAMING_SCHEME),
""));
+ }
+
wImportFrom.setFocus();
BaseDialog.defaultShellHandling(shell, c -> dispose(), c -> dispose());
@@ -587,6 +276,9 @@ public class KettleImportDialog extends Dialog {
LAST_USED_IMPORT_PIPELINE_RUN_CONFIGURATION,
wPipelineRunConfiguration.getText());
AuditManagerGuiUtil.addLastUsedValue(
LAST_USED_IMPORT_WORKFLOW_RUN_CONFIGURATION,
wWorkflowRunConfiguration.getText());
+ if (wNamingScheme != null) {
+ AuditManagerGuiUtil.addLastUsedValue(LAST_USED_IMPORT_NAMING_SCHEME,
wNamingScheme.getText());
+ }
AuditManagerGuiUtil.addLastUsedValue(
LAST_USED_IMPORT_SKIP_EXISTING, wSkipExisting.getSelection() ? "true"
: CONST_FALSE);
AuditManagerGuiUtil.addLastUsedValue(
@@ -676,6 +368,188 @@ public class KettleImportDialog extends Dialog {
true);
}
+ private void addSourceTab(CTabFolder folder) {
+ Composite parent = addTab(folder, "KettleImportDialog.Tab.Source");
+ wImportFrom =
+ addTextRow(
+ parent, null, "KettleImportDialog.Label.ImportFrom", true,
this::browseHomeFolder);
+ wKettleProps =
+ addTextRow(
+ parent,
+ wImportFrom,
+ "KettleImportDialog.Label.PathToKettleProperties",
+ true,
+ this::browseKettlePropsFile);
+ wShared =
+ addTextRow(
+ parent,
+ wKettleProps,
+ "KettleImportDialog.Label.PathToSharedXml",
+ true,
+ this::browseXmlFile);
+ wJdbcProps =
+ addTextRow(
+ parent,
+ wShared,
+ "KettleImportDialog.Label.PathToJDBCProperties",
+ true,
+ this::browseJdbcPropsFile);
+ wSkipHidden =
+ addCheckboxRow(parent, wJdbcProps,
"KettleImportDialog.Label.SkipHiddenFiles", true);
+ wSkipFolders =
+ addCheckboxRow(parent, wSkipHidden,
"KettleImportDialog.Label.SkipFolders", true);
+ }
+
+ private void addTargetTab(CTabFolder folder) {
+ Composite parent = addTab(folder, "KettleImportDialog.Tab.Target");
+ wImportInExisting =
+ addCheckboxRow(parent, null,
"KettleImportDialog.Label.ImportInExistingProject", true);
+ wImportInExisting.addListener(SWT.Selection, this::showHideProjectFields);
+
+ Label wlImportProject =
+ addLabel(parent, wImportInExisting,
"KettleImportDialog.Label.ImportInProject");
+ wImportProject = new Combo(parent, SWT.READ_ONLY);
+ wImportProject.setItems(projectNames.toArray(new String[0]));
+ PropsUi.setLook(wImportProject);
+ FormData fdImportProject = new FormData();
+ fdImportProject.left = new FormAttachment(middle, margin);
+ fdImportProject.right = new FormAttachment(100, 0);
+ fdImportProject.top = new FormAttachment(wlImportProject, 0, SWT.CENTER);
+ wImportProject.setLayoutData(fdImportProject);
+ wImportProject.addListener(SWT.Selection, event ->
bindTargetMetadataProvider());
+
+ Label wlImportPath =
+ addLabel(parent, wImportProject,
"KettleImportDialog.Label.ImportToFolder");
+ wbImportPath = addBrowseButton(parent, wlImportPath,
this::browseTargetFolder);
+ wbImportPath.setEnabled(false);
+ wImportPath = addTextVar(parent, wlImportPath, wbImportPath);
+ wImportPath.setEditable(false);
+ wImportPath.addModifyListener(event -> bindTargetMetadataProvider());
+
+ wSkipExisting =
+ addCheckboxRow(
+ parent, wImportPath,
"KettleImportDialog.Label.SkipExistingTargetFiles", true);
+ wTargetConfigFile =
+ addTextRow(parent, wSkipExisting,
"KettleImportDialog.Label.TargetConfigFile", false, null);
+ wTargetConfigFile.setEditable(false);
+ }
+
+ private void addMetadataTab(CTabFolder folder) {
+ Composite parent = addTab(folder, "KettleImportDialog.Tab.Metadata");
+ wPipelineRunConfiguration =
+ new MetaSelectionLine<>(
+ variables,
+ dialogMetadataProvider,
+ PipelineRunConfiguration.class,
+ parent,
+ SWT.NONE,
+ BaseMessages.getString(PKG,
"KettleImportDialog.Pipeline.RunConfiguration.Label"),
+ BaseMessages.getString(PKG,
"KettleImportDialog.RunConfiguration.Tooltip"));
+ wPipelineRunConfiguration.addToConnectionLine(parent, null, null, null);
+
+ wWorkflowRunConfiguration =
+ new MetaSelectionLine<>(
+ variables,
+ dialogMetadataProvider,
+ WorkflowRunConfiguration.class,
+ parent,
+ SWT.NONE,
+ BaseMessages.getString(PKG,
"KettleImportDialog.Workflow.RunConfiguration.Label"),
+ BaseMessages.getString(PKG,
"KettleImportDialog.RunConfiguration.Tooltip"));
+ wWorkflowRunConfiguration.addToConnectionLine(parent,
wPipelineRunConfiguration, null, null);
+
+ wNamingScheme =
+ MetaSelectionLine.forMetadataKey(
+ variables,
+ dialogMetadataProvider,
+ parent,
+ SWT.NONE,
+ NAMING_SCHEME_METADATA_KEY,
+ BaseMessages.getString(PKG,
"KettleImportDialog.NamingScheme.Label"),
+ BaseMessages.getString(PKG,
"KettleImportDialog.NamingScheme.Tooltip"));
+ if (wNamingScheme != null) {
+ wNamingScheme.addToConnectionLine(parent, wWorkflowRunConfiguration,
null, null);
+ }
+ }
+
+ private Composite addTab(CTabFolder folder, String i18nKey) {
+ CTabItem item = new CTabItem(folder, SWT.NONE);
+ item.setFont(GuiResource.getInstance().getFontDefault());
+ item.setText(BaseMessages.getString(PKG, i18nKey));
+ Composite composite = new Composite(folder, SWT.NONE);
+ PropsUi.setLook(composite);
+ FormLayout layout = new FormLayout();
+ layout.marginWidth = PropsUi.getFormMargin();
+ layout.marginHeight = PropsUi.getFormMargin();
+ composite.setLayout(layout);
+ item.setControl(composite);
+ return composite;
+ }
+
+ private Label addLabel(Composite parent, Control previous, String labelKey) {
+ Label label = new Label(parent, SWT.RIGHT);
+ PropsUi.setLook(label);
+ label.setText(BaseMessages.getString(PKG, labelKey));
+ FormData fd = new FormData();
+ fd.left = new FormAttachment(0, 0);
+ fd.right = new FormAttachment(middle, 0);
+ if (previous != null) {
+ fd.top = new FormAttachment(previous, margin);
+ } else {
+ fd.top = new FormAttachment(0, margin);
+ }
+ label.setLayoutData(fd);
+ return label;
+ }
+
+ private Button addBrowseButton(Composite parent, Control alignTo, Listener
browseListener) {
+ Button browse = new Button(parent, SWT.PUSH);
+ PropsUi.setLook(browse);
+ browse.setText(BaseMessages.getString(PKG,
CONST_KETTLE_IMPORT_DIALOG_BUTTON_BROWSE));
+ FormData fd = new FormData();
+ fd.right = new FormAttachment(100, 0);
+ fd.top = new FormAttachment(alignTo, 0, SWT.CENTER);
+ browse.setLayoutData(fd);
+ browse.addListener(SWT.Selection, browseListener);
+ return browse;
+ }
+
+ private TextVar addTextVar(Composite parent, Control alignTo, Control
rightOf) {
+ TextVar text = new TextVar(variables, parent, SWT.SINGLE | SWT.BORDER |
SWT.LEFT);
+ PropsUi.setLook(text);
+ FormData fd = new FormData();
+ fd.left = new FormAttachment(middle, margin);
+ fd.right = rightOf != null ? new FormAttachment(rightOf, -margin) : new
FormAttachment(100, 0);
+ fd.top = new FormAttachment(alignTo, 0, SWT.CENTER);
+ text.setLayoutData(fd);
+ return text;
+ }
+
+ private TextVar addTextRow(
+ Composite parent,
+ Control previous,
+ String labelKey,
+ boolean withBrowse,
+ Listener browseListener) {
+ Label label = addLabel(parent, previous, labelKey);
+ Button browse = withBrowse ? addBrowseButton(parent, label,
browseListener) : null;
+ return addTextVar(parent, label, browse);
+ }
+
+ private Button addCheckboxRow(
+ Composite parent, Control previous, String labelKey, boolean selected) {
+ Label label = addLabel(parent, previous, labelKey);
+ Button checkbox = new Button(parent, SWT.CHECK);
+ PropsUi.setLook(checkbox);
+ FormData fd = new FormData();
+ fd.left = new FormAttachment(middle, margin);
+ fd.right = new FormAttachment(100, 0);
+ fd.top = new FormAttachment(label, 0, SWT.CENTER);
+ checkbox.setLayoutData(fd);
+ checkbox.setSelection(selected);
+ return checkbox;
+ }
+
private void doImport() {
try {
@@ -721,6 +595,7 @@ public class KettleImportDialog extends Dialog {
kettleImport.setValidateInputFolder(sourceFolder);
kettleImport.setValidateOutputFolder(targetFolder);
+ persistDialogMetadataToTarget(kettleImport);
kettleImport.setSharedXmlFilename(variables.resolve(wShared.getText()));
kettleImport.setKettlePropertiesFilename(variables.resolve(wKettleProps.getText()));
kettleImport.setJdbcPropertiesFilename(variables.resolve(wJdbcProps.getText()));
@@ -732,6 +607,10 @@ public class KettleImportDialog extends Dialog {
kettleImport.setDefaultPipelineRunConfiguration(defaultPRC);
String defaultWRC = Const.NVL(wWorkflowRunConfiguration.getText(), "");
kettleImport.setDefaultWorkflowRunConfiguration(defaultWRC);
+ kettleImport.setApplyNamingSchemes(true);
+ if (wNamingScheme != null) {
+ kettleImport.setNamingSchemeName(Const.NVL(wNamingScheme.getText(),
""));
+ }
boolean goForImport = true;
if ((Utils.isEmpty(defaultPRC) && Utils.isEmpty(defaultWRC))
@@ -763,7 +642,7 @@ public class KettleImportDialog extends Dialog {
true,
monitor -> {
try {
- monitor.beginTask("Importing files", 4);
+ monitor.beginTask("Importing files", 5);
kettleImport.runImport(monitor);
monitor.done();
} catch (Throwable e) {
@@ -801,5 +680,138 @@ public class KettleImportDialog extends Dialog {
wImportPath.setEditable(true);
wbImportPath.setEnabled(true);
}
+ bindTargetMetadataProvider();
+ }
+
+ /**
+ * Rebuild the metadata lines so they can list objects from the target
folder and the current
+ * project. New and Edit write only to {@link #scratchMetadata}; nothing is
saved to disk until
+ * Import.
+ */
+ void bindTargetMetadataProvider() {
+ if (shell == null || shell.isDisposed()) {
+ return;
+ }
+ String metadataFolder = metadataFolderFor(peekTargetFolder());
+ if (Objects.equals(metadataFolder, boundMetadataFolder) &&
dialogMetadataProvider != null) {
+ return;
+ }
+ List<IHopMetadataProvider> providers = new ArrayList<>();
+ if (metadataFolder != null) {
+ providers.add(new JsonMetadataProvider(Encr.getEncoder(),
metadataFolder, variables));
+ }
+ HopGui hopGui = HopGui.getInstance();
+ if (hopGui != null && hopGui.getMetadataProvider() != null) {
+ providers.add(hopGui.getMetadataProvider());
+ }
+ providers.add(scratchMetadata);
+ applyMetadataProvider(
+ new MultiMetadataProvider(Encr.getEncoder(), providers, variables),
metadataFolder);
+ }
+
+ private String peekTargetFolder() {
+ if (wImportInExisting != null && wImportInExisting.getSelection()) {
+ String projectName = wImportProject != null ? wImportProject.getText() :
"";
+ if (Utils.isEmpty(projectName)) {
+ return null;
+ }
+ Object[] objects = new Object[] {projectName, ""};
+ try {
+ ExtensionPointHandler.callExtensionPoint(
+ HopGui.getInstance().getLog(), variables, "ProjectHome", objects);
+ return (String) objects[1];
+ } catch (Exception e) {
+ return null;
+ }
+ }
+ if (wImportPath == null) {
+ return null;
+ }
+ return variables.resolve(wImportPath.getText());
+ }
+
+ private void applyMetadataProvider(IHopMetadataProvider provider, String
metadataFolder) {
+ this.dialogMetadataProvider = provider;
+ this.boundMetadataFolder = metadataFolder;
+ if (wPipelineRunConfiguration != null) {
+ wPipelineRunConfiguration.setMetadataProvider(provider);
+ }
+ if (wWorkflowRunConfiguration != null) {
+ wWorkflowRunConfiguration.setMetadataProvider(provider);
+ }
+ if (wNamingScheme != null) {
+ wNamingScheme.setMetadataProvider(provider);
+ }
+ }
+
+ IHopMetadataProvider getDialogMetadataProvider() {
+ return dialogMetadataProvider;
+ }
+
+ static String metadataFolderFor(String targetFolder) {
+ return HopImportBase.metadataFolderFor(targetFolder);
+ }
+
+ /**
+ * Write only the selected metadata objects into the target folder, then
hand that provider to the
+ * importer. Objects created in the dialog live in {@link #scratchMetadata}
until this point.
+ */
+ private void persistDialogMetadataToTarget(KettleImport kettleImport) throws
HopException {
+ String metadataFolder =
metadataFolderFor(kettleImport.getOutputFolder().getName().getURI());
+ JsonMetadataProvider json =
+ new JsonMetadataProvider(Encr.getEncoder(), metadataFolder, variables);
+ copyNamed(
+ dialogMetadataProvider,
+ json,
+ PipelineRunConfiguration.class,
+ wPipelineRunConfiguration.getText());
+ copyNamed(
+ dialogMetadataProvider,
+ json,
+ WorkflowRunConfiguration.class,
+ wWorkflowRunConfiguration.getText());
+ if (wNamingScheme != null) {
+ copyNamedByKey(
+ dialogMetadataProvider, json, NAMING_SCHEME_METADATA_KEY,
wNamingScheme.getText());
+ }
+ applyMetadataProvider(json, metadataFolder);
+ kettleImport.setMetadataTargetFolder(metadataFolder);
+ kettleImport.setMetadataProvider(
+ new MultiMetadataProvider(Encr.getEncoder(), List.of(json),
variables));
+ }
+
+ static void copyNamed(
+ IHopMetadataProvider from,
+ IHopMetadataProvider to,
+ Class<? extends IHopMetadata> type,
+ String name)
+ throws HopException {
+ if (from == null || to == null || type == null ||
StringUtils.isBlank(name)) {
+ return;
+ }
+ IHopMetadataSerializer<IHopMetadata> dest = serializer(to, type);
+ if (dest.exists(name)) {
+ return;
+ }
+ IHopMetadata object = serializer(from, type).load(name);
+ if (object == null) {
+ return;
+ }
+ dest.save(object);
+ }
+
+ static void copyNamedByKey(
+ IHopMetadataProvider from, IHopMetadataProvider to, String metadataKey,
String name)
+ throws HopException {
+ if (from == null || StringUtils.isBlank(metadataKey) ||
StringUtils.isBlank(name)) {
+ return;
+ }
+ copyNamed(from, to, from.getMetadataClassForKey(metadataKey), name);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static IHopMetadataSerializer<IHopMetadata> serializer(
+ IHopMetadataProvider provider, Class<? extends IHopMetadata> type)
throws HopException {
+ return (IHopMetadataSerializer<IHopMetadata>) provider.getSerializer(type);
}
}
diff --git
a/plugins/misc/import/src/main/resources/org/apache/hop/imports/kettle/messages/messages_en_US.properties
b/plugins/misc/import/src/main/resources/org/apache/hop/imports/kettle/messages/messages_en_US.properties
index de7f71ad2d..8118c266df 100644
---
a/plugins/misc/import/src/main/resources/org/apache/hop/imports/kettle/messages/messages_en_US.properties
+++
b/plugins/misc/import/src/main/resources/org/apache/hop/imports/kettle/messages/messages_en_US.properties
@@ -16,6 +16,9 @@
#
KettleImportDialog.Button.Browse=Browse...
+KettleImportDialog.Tab.Source=Source
+KettleImportDialog.Tab.Target=Target
+KettleImportDialog.Tab.Metadata=Metadata
KettleImportDialog.ImportSummary.Imported.Label=Imported :
KettleImportDialog.ImportSummary.ImportedJobs.Label=jobs
KettleImportDialog.ImportSummary.ImportedOther.Label=other files
@@ -37,7 +40,9 @@ KettleImportDialog.NoDefaultRCAll.Message=No default run
configuration has been
KettleImportDialog.NoDefaultRCPrc.Message=No default run configuration has
been specified for pipelines. This can cause errors\nduring the execution of
your process in any case where, in the original PDI process, a run
configuration\n was needed but was not specified.
KettleImportDialog.NoDefaultRCWrc.Message=No default run configuration has
been specified for workflows. This can cause errors\nduring the execution of
your process in any case where, in the original PDI process, a run
configuration\n was needed but was not specified.
KettleImportDialog.Pipeline.RunConfiguration.Label=Pipeline default run
configuration
-KettleImportDialog.RunConfiguration.Tooltip=Sets the default run configuration
in case it was not specified
+KettleImportDialog.RunConfiguration.Tooltip=Sets the default run configuration
in case it was not specified. New and Edit stay in memory until Import, then
the selected object is written to the target project''s metadata folder.
+KettleImportDialog.NamingScheme.Label=Naming scheme for connections
+KettleImportDialog.NamingScheme.Tooltip=Applied to imported relational
connection names. New and Edit stay in memory until Import, then the selected
scheme is written to the target project''s metadata. Leave empty to use the
unique Hop-metadata (or General) scheme.
KettleImportDialog.Shell.Name=Import code to Hop
KettleImportDialog.Workflow.RunConfiguration.Label=Workflow default run
configuration
KettleImportDialog.Error.Title=Kettle/PDI import
diff --git
a/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportDialogTest.java
b/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportDialogTest.java
index e989c84117..8948a85610 100644
---
a/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportDialogTest.java
+++
b/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportDialogTest.java
@@ -36,11 +36,14 @@ import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.database.DatabaseMeta;
import org.apache.hop.core.logging.HopLogStore;
import org.apache.hop.core.logging.ILogChannel;
import org.apache.hop.core.security.Permission;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.imports.gui.HopImportGuiPlugin;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
import org.apache.hop.ui.core.dialog.ErrorDialog;
import org.apache.hop.ui.core.security.HopSecurityUi;
import org.apache.hop.ui.hopgui.HopGui;
@@ -56,8 +59,42 @@ import org.mockito.MockedStatic;
class KettleImportDialogTest {
@BeforeAll
- static void initializeLogging() {
+ static void initializeLogging() throws Exception {
HopLogStore.init();
+ HopClientEnvironment.init();
+ }
+
+ @Test
+ void metadataFolderIsUnderTheTargetProject() {
+ assertEquals(
+ "/projects/sales/metadata",
KettleImportDialog.metadataFolderFor("/projects/sales"));
+ assertEquals(
+ "/projects/sales/metadata",
KettleImportDialog.metadataFolderFor("/projects/sales/"));
+ assertEquals("C:/data/hop/metadata",
KettleImportDialog.metadataFolderFor("C:\\data\\hop"));
+ assertNull(KettleImportDialog.metadataFolderFor(""));
+ assertNull(KettleImportDialog.metadataFolderFor(null));
+ assertEquals(
+ "file:///tmp/project/metadata",
+ KettleImportDialog.metadataFolderFor("file:///tmp/project/"));
+ }
+
+ @Test
+ void copyNamedWritesOnlyMissingObjects() throws Exception {
+ MemoryMetadataProvider from = new MemoryMetadataProvider();
+ MemoryMetadataProvider to = new MemoryMetadataProvider();
+ DatabaseMeta databaseMeta = new DatabaseMeta();
+ databaseMeta.setName("local");
+ from.getSerializer(DatabaseMeta.class).save(databaseMeta);
+
+ KettleImportDialog.copyNamed(from, to, DatabaseMeta.class, "local");
+ DatabaseMeta stored = to.getSerializer(DatabaseMeta.class).load("local");
+ assertEquals("local", stored.getName());
+
+ DatabaseMeta replacement = new DatabaseMeta();
+ replacement.setName("local");
+ from.getSerializer(DatabaseMeta.class).save(replacement);
+ KettleImportDialog.copyNamed(from, to, DatabaseMeta.class, "local");
+ assertSame(stored, to.getSerializer(DatabaseMeta.class).load("local"));
}
@Test
diff --git
a/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportTest.java
b/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportTest.java
index 6cdf534503..fec37a624e 100644
---
a/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportTest.java
+++
b/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportTest.java
@@ -111,6 +111,31 @@ class KettleImportTest {
assertNull(databaseMeta.getIDatabase().getManualUrl());
}
+ @Test
+ void caseVariantConnectionNamesAreDeduped() throws Exception {
+ String xml =
+ "<transformation>"
+ +
"<connection><name>Database</name><type>GENERIC</type><access>Native</access></connection>"
+ + "</transformation>";
+ KettleImport kettleImport = new KettleImport();
+ invokeImportDbConnections(kettleImport, parse(xml));
+
+ DatabaseMeta otherCase = new DatabaseMeta();
+ otherCase.setName("DATABASE");
+ kettleImport.addDatabaseMeta("other.ktr", otherCase);
+
+ assertEquals(1, kettleImport.getConnectionsList().size());
+ assertEquals("Database",
kettleImport.getConnectionsList().get(0).getName());
+ }
+
+ @Test
+ void csvFieldQuotesCommasAndDoublesQuotes() {
+ assertEquals("plain", KettleImport.csvField("plain"));
+ assertEquals("\"a,b\"", KettleImport.csvField("a,b"));
+ assertEquals("\"say \"\"hi\"\"\"", KettleImport.csvField("say \"hi\""));
+ assertEquals("", KettleImport.csvField(null));
+ }
+
private static Document parse(String xml) throws Exception {
try (ByteArrayInputStream in = new
ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))) {
return XmlParserFactoryProducer.createSecureDocBuilderFactory()
diff --git
a/plugins/misc/naming/src/main/java/org/apache/hop/naming/gui/NamingSchemeImportExtension.java
b/plugins/misc/naming/src/main/java/org/apache/hop/naming/gui/NamingSchemeImportExtension.java
new file mode 100644
index 0000000000..e758d88921
--- /dev/null
+++
b/plugins/misc/naming/src/main/java/org/apache/hop/naming/gui/NamingSchemeImportExtension.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.naming.gui;
+
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.extension.ExtensionPoint;
+import org.apache.hop.core.extension.IExtensionPoint;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.imp.HopImportBase;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.naming.engine.NamingEngine;
+import org.apache.hop.naming.metadata.NamingScheme;
+import org.apache.hop.naming.metadata.NamingSchemeSelector;
+import org.apache.hop.naming.metadata.NamingSchemeType;
+
+/**
+ * When a Kettle import is about to normalize relational connection names,
apply the unique
+ * hop-metadata (else general) naming scheme from the target metadata
provider, or an explicitly
+ * selected scheme name.
+ */
+@ExtensionPoint(
+ id = "NamingSchemeHopImportRewriteMetadata",
+ description = "Apply a naming scheme to imported relational connection
names",
+ extensionPointId = "HopImportRewriteMetadata")
+public class NamingSchemeImportExtension implements
IExtensionPoint<HopImportBase> {
+
+ @Override
+ public void callExtensionPoint(ILogChannel log, IVariables variables,
HopImportBase hopImport)
+ throws HopException {
+ if (hopImport == null || !hopImport.isApplyNamingSchemes()) {
+ return;
+ }
+ IHopMetadataProvider provider = hopImport.getMetadataProvider();
+ if (provider == null) {
+ return;
+ }
+ List<NamingScheme> schemes;
+ try {
+ schemes = provider.getSerializer(NamingScheme.class).loadAll();
+ } catch (Exception e) {
+ if (log != null) {
+ log.logError("Unable to load naming schemes from the import target
metadata", e);
+ }
+ return;
+ }
+ NamingScheme scheme =
+ NamingSchemeSelector.resolve(
+ schemes, NamingSchemeType.HOP_METADATA.getCode(),
hopImport.getNamingSchemeName());
+ if (scheme == null &&
StringUtils.isEmpty(hopImport.getNamingSchemeName())) {
+ scheme = NamingSchemeSelector.resolve(schemes,
NamingSchemeType.GENERAL.getCode(), null);
+ }
+ if (scheme == null) {
+ if (log != null &&
StringUtils.isNotEmpty(hopImport.getNamingSchemeName())) {
+ log.logError(
+ "Naming scheme '"
+ + hopImport.getNamingSchemeName()
+ + "' was not found in the target metadata");
+ }
+ return;
+ }
+ NamingScheme chosen = scheme;
+ hopImport.setAppliedNamingSchemeName(chosen.getName());
+ hopImport.setConnectionNameMapper(
+ name -> NamingEngine.apply(chosen, name,
NamingSchemeType.HOP_METADATA.getCode()));
+ if (log != null) {
+ log.logBasic(
+ "Applying naming scheme '"
+ + chosen.getName()
+ + "' to imported relational connection names");
+ }
+ }
+}
diff --git
a/plugins/misc/naming/src/test/java/org/apache/hop/naming/gui/NamingSchemeImportExtensionTest.java
b/plugins/misc/naming/src/test/java/org/apache/hop/naming/gui/NamingSchemeImportExtensionTest.java
new file mode 100644
index 0000000000..36c0673650
--- /dev/null
+++
b/plugins/misc/naming/src/test/java/org/apache/hop/naming/gui/NamingSchemeImportExtensionTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.naming.gui;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+
+import java.util.function.UnaryOperator;
+import org.apache.hop.core.HopClientEnvironment;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.imp.HopImportBase;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
+import org.apache.hop.naming.engine.NamingEngine;
+import org.apache.hop.naming.metadata.NamingCaseStyle;
+import org.apache.hop.naming.metadata.NamingScheme;
+import org.apache.hop.naming.metadata.NamingSchemeType;
+import org.apache.hop.naming.metadata.NamingWordSeparator;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+class NamingSchemeImportExtensionTest {
+
+ @BeforeAll
+ static void init() throws Exception {
+ HopClientEnvironment.init();
+ }
+
+ @Test
+ void uniqueHopMetadataSchemeBecomesTheMapper() throws Exception {
+ NamingScheme scheme = lowerUnderscore("conn-names");
+ scheme.setType(NamingSchemeType.HOP_METADATA.getCode());
+
+ MemoryMetadataProvider memory = new MemoryMetadataProvider();
+ memory.getSerializer(NamingScheme.class).save(scheme);
+
+ HopImportBase hopImport = new TestImport();
+ hopImport.setMetadataProvider(
+ new MultiMetadataProvider(null, java.util.List.of(memory),
hopImport.getVariables()));
+
+ new NamingSchemeImportExtension()
+ .callExtensionPoint(LogChannel.GENERAL, hopImport.getVariables(),
hopImport);
+
+ assertEquals("conn-names", hopImport.getAppliedNamingSchemeName());
+ UnaryOperator<String> mapper = hopImport.getConnectionNameMapper();
+ assertEquals(NamingEngine.apply(scheme, "My DB", "hop-metadata"),
mapper.apply("My DB"));
+ }
+
+ @Test
+ void explicitNameWinsOverType() throws Exception {
+ NamingScheme general = lowerUnderscore("general-one");
+ general.setType(NamingSchemeType.GENERAL.getCode());
+ NamingScheme metadata = lowerUnderscore("meta-one");
+ metadata.setType(NamingSchemeType.HOP_METADATA.getCode());
+ metadata.setCaseStyle(NamingCaseStyle.UPPER.getCode());
+
+ MemoryMetadataProvider memory = new MemoryMetadataProvider();
+ memory.getSerializer(NamingScheme.class).save(general);
+ memory.getSerializer(NamingScheme.class).save(metadata);
+
+ HopImportBase hopImport = new TestImport();
+ hopImport.setMetadataProvider(
+ new MultiMetadataProvider(null, java.util.List.of(memory),
hopImport.getVariables()));
+ hopImport.setNamingSchemeName("general-one");
+
+ new NamingSchemeImportExtension()
+ .callExtensionPoint(LogChannel.GENERAL, hopImport.getVariables(),
hopImport);
+
+ assertEquals("general-one", hopImport.getAppliedNamingSchemeName());
+ assertEquals("my_db", hopImport.getConnectionNameMapper().apply("My DB"));
+ }
+
+ @Test
+ void skippedWhenApplyDisabled() throws Exception {
+ NamingScheme scheme = lowerUnderscore("conn-names");
+ scheme.setType(NamingSchemeType.HOP_METADATA.getCode());
+ MemoryMetadataProvider memory = new MemoryMetadataProvider();
+ memory.getSerializer(NamingScheme.class).save(scheme);
+
+ HopImportBase hopImport = new TestImport();
+ hopImport.setMetadataProvider(
+ new MultiMetadataProvider(null, java.util.List.of(memory),
hopImport.getVariables()));
+ hopImport.setApplyNamingSchemes(false);
+
+ new NamingSchemeImportExtension()
+ .callExtensionPoint(LogChannel.GENERAL, hopImport.getVariables(),
hopImport);
+
+ assertEquals(null, hopImport.getAppliedNamingSchemeName());
+ assertEquals("My DB", hopImport.getConnectionNameMapper().apply("My DB"));
+ }
+
+ @Test
+ void noSchemeLeavesIdentityMapper() throws Exception {
+ HopImportBase hopImport = new TestImport();
+ hopImport.setMetadataProvider(
+ new MultiMetadataProvider(
+ null, java.util.List.of(new MemoryMetadataProvider()),
hopImport.getVariables()));
+
+ new NamingSchemeImportExtension()
+ .callExtensionPoint(LogChannel.GENERAL, hopImport.getVariables(),
hopImport);
+
+ assertEquals("My DB", hopImport.getConnectionNameMapper().apply("My DB"));
+ assertNotEquals("x", hopImport.getConnectionNameMapper().apply("My DB"));
+ }
+
+ private static NamingScheme lowerUnderscore(String name) {
+ NamingScheme scheme = new NamingScheme(name);
+ scheme.setCaseStyle(NamingCaseStyle.LOWER.getCode());
+ scheme.setWordSeparator(NamingWordSeparator.UNDERSCORE.getCode());
+ scheme.setRemoveSpecialCharacters(true);
+ scheme.setCollapseRepeatedSeparators(true);
+ scheme.setTrimEdgeSeparators(true);
+ return scheme;
+ }
+
+ private static final class TestImport extends HopImportBase {
+ @Override
+ public void importFiles() {}
+
+ @Override
+ public void findFilesToImport() {}
+
+ @Override
+ public void importConnections() {}
+
+ @Override
+ public void importVariables() {}
+
+ @Override
+ public String getImportReport() {
+ return "";
+ }
+ }
+}
diff --git
a/ui/src/main/java/org/apache/hop/ui/core/widget/MetaSelectionLine.java
b/ui/src/main/java/org/apache/hop/ui/core/widget/MetaSelectionLine.java
index d5882f679d..1114355043 100644
--- a/ui/src/main/java/org/apache/hop/ui/core/widget/MetaSelectionLine.java
+++ b/ui/src/main/java/org/apache/hop/ui/core/widget/MetaSelectionLine.java
@@ -552,6 +552,23 @@ public class MetaSelectionLine<T extends IHopMetadata>
extends Composite {
return metadataProvider;
}
+ /**
+ * Point this line at another metadata provider (for example the import
target folder) and refresh
+ * the combo items. Keeps the current text when that name still exists.
+ */
+ public void setMetadataProvider(IHopMetadataProvider metadataProvider) {
+ this.metadataProvider = metadataProvider;
+ if (manager != null) {
+ manager.setMetadataProvider(metadataProvider);
+ }
+ try {
+ fillItems();
+ } catch (HopException e) {
+ LogChannel.UI.logError(
+ "Error refreshing list of " + getMetadataDescription() + " metadata
elements", e);
+ }
+ }
+
/**
* Gets variables
*
