This is an automated email from the ASF dual-hosted git repository.
mattcasters pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 63b8f15f74 Improve metadata item naming consistency, fixes #5597
(#8554)
63b8f15f74 is described below
commit 63b8f15f74628d3307f24903720a0f18709df527
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Sep 24 12:51:18 2026 +0200
Improve metadata item naming consistency, fixes #5597 (#8554)
* Improve metadata item naming consistency, fixes #5597
* fix remarks
---
AGENTS.md | 8 +
.../org/apache/hop/metadata/api/HopMetadata.java | 11 +
.../hop/metadata/api/MetadataRefactorUtil.java | 4 +-
.../metadata/serializer/BaseMetadataProvider.java | 16 +
.../serializer/json/JsonMetadataProvider.java | 24 +-
.../serializer/json/JsonMetadataSerializer.java | 180 ++++--
.../apache/hop/metadata/util/HopMetadataUtil.java | 77 +++
.../hop/metadata/serializer/MetadataGuiFlows.java | 91 +++
.../json/JsonMetadataLegacyKeysTest.java | 631 +++++++++++++++++++++
.../serializer/json/renamed/CaseRenamedType.java | 36 +-
.../serializer/json/renamed/RenamedType.java | 36 +-
.../serializer/json/renamed/TwiceRenamedType.java | 36 +-
.../multi/MultiMetadataLegacyKeysTest.java | 283 +++++++++
.../modules/ROOT/pages/metadata-plugins.adoc | 44 +-
.../ROOT/pages/metadata-types/rest-connection.adoc | 2 +-
.../ROOT/pages/pipeline/transforms/rest.adoc | 2 +-
.../metadata/refactor/MetadataReferenceFinder.java | 2 +-
.../SerializableMetadataLegacyKeysTest.java | 192 +++++++
.../apache/hop/spark/metadata/SparkCatalog.java | 3 +-
.../org/apache/hop/git/provider/GitConnection.java | 3 +-
.../{gitconnection => git-connection}/github.json | 0
.../org/apache/hop/lint/LintMetadataSelection.java | 9 +-
.../lint/HopMetadataFileLoaderLegacyKeysTest.java | 111 ++++
.../hop/mail/metadata/MailServerConnection.java | 3 +-
.../apache/hop/metadata/rest/RestConnection.java | 3 +-
.../org/apache/hop/vfs/s3/metadata/S3Meta.java | 3 +-
.../vfs/azure/metadatatype/AzureMetadataType.java | 3 +-
.../databricks/metadata/DatabricksConnection.java | 3 +-
.../metadata/DatabricksVfsConnection.java | 3 +-
.../gs/metadatatype/GoogleStorageMetadataType.java | 3 +-
.../org/apache/hop/vfs/hdfs/metadata/HdfsMeta.java | 3 +-
.../apache/hop/vfs/minio/metadata/MinioMeta.java | 3 +-
.../metadata/salesforce/SalesforceConnection.java | 3 +-
.../hop/vfs/webdav/metadata/WebDavConnection.java | 3 +-
.../transforms/metainput/MetadataInput.java | 3 +-
.../github-releases.json | 0
.../slack-conversations.json | 0
.../perspective/metadata/MetadataPerspective.java | 76 ++-
.../ui/hopgui/search/HopGuiMetadataSearchable.java | 12 +
.../search/HopGuiMetadataSearchableTest.java | 99 ++++
40 files changed, 1868 insertions(+), 156 deletions(-)
diff --git a/AGENTS.md b/AGENTS.md
index ea81fcfd0d..2fa4db5adc 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -123,3 +123,11 @@ Values defined in resource bundles (resource files in
`messages/messages\*.prope
Apache Hop and related projects should use Lombok for all classes to avoid
cluttering classes with boilerplate getter/setter methods.
+# Metadata type keys
+
+The `key` of `@HopMetadata` is the metadata plugin ID and the name of the
folder its objects are stored in (`metadata/<key>/<name>.json`). It is also
written into serialized metadata exports and matched against
`disabledGuiElements`. Treat it as a public, persisted identifier.
+
+- **New metadata types:** the key is lower-case and dash-separated
(kebab-case) and names what the type is, for example `mail-server-connection`
or `pipeline-run-configuration`. Never use PascalCase or the Java class name.
Check this on every new or changed `@HopMetadata` annotation, including in
reviews.
+- **Renaming a key:** never change an existing key without moving the old one
to `legacyKeys`, for example `key = "mail-server-connection", legacyKeys =
{"MailServerConnection"}`. Objects in the old folder keep loading, and each one
moves to the new folder the next time it is saved. Without `legacyKeys`,
existing projects silently lose those objects.
+- **Code that works with type keys or metadata files** must go through
`HopMetadataUtil.getAllKeys()` / `matchesKey()` or
`JsonMetadataSerializer.findFilename()`, not `annotation.key()` alone.
Otherwise objects still in a legacy folder are missed.
+- `rdbms` predates this convention and stays as it is.
diff --git a/core/src/main/java/org/apache/hop/metadata/api/HopMetadata.java
b/core/src/main/java/org/apache/hop/metadata/api/HopMetadata.java
index d8bf843869..85ed151216 100644
--- a/core/src/main/java/org/apache/hop/metadata/api/HopMetadata.java
+++ b/core/src/main/java/org/apache/hop/metadata/api/HopMetadata.java
@@ -31,6 +31,17 @@ public @interface HopMetadata {
*/
String key();
+ /**
+ * Keys this metadata type was known under before it was renamed, for
example to bring it in line
+ * with the lower-case, dash-separated naming convention. Objects stored
under a legacy key (a
+ * JSON sub-folder named after it, or a serialized metadata export) are
still found and loaded.
+ * Saving an object always writes it under {@link #key()} and removes the
legacy copy, so projects
+ * migrate one object at a time as they are edited.
+ *
+ * @return the previous keys of this metadata type, most recent first
+ */
+ String[] legacyKeys() default {};
+
/**
* The key for this hop metadata object. It will be translated into a
sub-folder for the JSON
* serializer
diff --git
a/core/src/main/java/org/apache/hop/metadata/api/MetadataRefactorUtil.java
b/core/src/main/java/org/apache/hop/metadata/api/MetadataRefactorUtil.java
index 67a6863de6..443599d24d 100644
--- a/core/src/main/java/org/apache/hop/metadata/api/MetadataRefactorUtil.java
+++ b/core/src/main/java/org/apache/hop/metadata/api/MetadataRefactorUtil.java
@@ -33,7 +33,7 @@ public final class MetadataRefactorUtil {
* reference used in pipeline/workflow XML (e.g. RDBMS_CONNECTION for
"rdbms").
*
* @param provider metadata provider (used to resolve the class for the key)
- * @param metadataKey the metadata type key (e.g. "rdbms", "restconnection")
+ * @param metadataKey the metadata type key (e.g. "rdbms", "rest-connection")
* @return the property type, or {@link HopMetadataPropertyType#NONE} if not
declared or unknown
*/
public static HopMetadataPropertyType getPropertyTypeForMetadataKey(
@@ -56,7 +56,7 @@ public final class MetadataRefactorUtil {
* pipelines and workflows when renaming). Uses {@link
HopMetadata#supportsGlobalReplace()}.
*
* @param provider metadata provider (used to resolve the class for the key)
- * @param metadataKey the metadata type key (e.g. "rdbms", "restconnection")
+ * @param metadataKey the metadata type key (e.g. "rdbms", "rest-connection")
* @return true if global replace is supported, false otherwise
*/
public static boolean supportsGlobalReplace(IHopMetadataProvider provider,
String metadataKey) {
diff --git
a/core/src/main/java/org/apache/hop/metadata/serializer/BaseMetadataProvider.java
b/core/src/main/java/org/apache/hop/metadata/serializer/BaseMetadataProvider.java
index 6c1f2b43c9..553ebac0f6 100644
---
a/core/src/main/java/org/apache/hop/metadata/serializer/BaseMetadataProvider.java
+++
b/core/src/main/java/org/apache/hop/metadata/serializer/BaseMetadataProvider.java
@@ -25,8 +25,10 @@ import org.apache.hop.core.gui.plugin.GuiRegistry;
import org.apache.hop.core.plugins.IPlugin;
import org.apache.hop.core.plugins.PluginRegistry;
import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.HopMetadata;
import org.apache.hop.metadata.api.IHopMetadata;
import org.apache.hop.metadata.plugin.MetadataPluginType;
+import org.apache.hop.metadata.util.HopMetadataUtil;
public class BaseMetadataProvider {
@@ -51,6 +53,12 @@ public class BaseMetadataProvider {
}
String className = plugin.getClassMap().get(plugin.getMainType());
Class<?> pluginClass =
registry.getClassLoader(plugin).loadClass(className);
+ // A type can still be disabled with the key it had before it was
renamed.
+ HopMetadata annotation = pluginClass.getAnnotation(HopMetadata.class);
+ if (annotation != null
+ &&
HopMetadataUtil.getAllKeys(annotation).stream().anyMatch(disabledIds::contains))
{
+ continue;
+ }
classes.add((Class<T>) pluginClass);
}
return classes;
@@ -64,6 +72,14 @@ public class BaseMetadataProvider {
PluginRegistry registry = PluginRegistry.getInstance();
IPlugin plugin = registry.findPluginWithId(MetadataPluginType.class,
key);
if (plugin == null) {
+ // The key can be one a metadata type was known under before it was
renamed, for example
+ // in a serialized metadata export written by an older version of Hop.
+ //
+ for (Class<T> metadataClass : this.<T>getMetadataClasses()) {
+ if
(HopMetadataUtil.matchesKey(metadataClass.getAnnotation(HopMetadata.class),
key)) {
+ return metadataClass;
+ }
+ }
throw new HopException(
"The metadata plugin for key " + key + " could not be found in the
plugin registry");
}
diff --git
a/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataProvider.java
b/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataProvider.java
index b5c6895969..4a36ae57a1 100644
---
a/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataProvider.java
+++
b/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataProvider.java
@@ -17,6 +17,8 @@
package org.apache.hop.metadata.serializer.json;
+import java.util.ArrayList;
+import java.util.List;
import lombok.Getter;
import lombok.Setter;
import org.apache.hop.core.Const;
@@ -32,6 +34,7 @@ import org.apache.hop.metadata.api.IHopMetadata;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.api.IHopMetadataSerializer;
import org.apache.hop.metadata.serializer.BaseMetadataProvider;
+import org.apache.hop.metadata.util.HopMetadataUtil;
@Getter
@Setter
@@ -83,16 +86,25 @@ public class JsonMetadataProvider extends
BaseMetadataProvider implements IHopMe
+ " it needs to have annotation "
+ HopMetadata.class.getName());
}
- String classFolder = Const.NVL(hopMetadata.key(), hopMetadata.name());
- String serializerBaseFolderName =
- baseFolder
- + (baseFolder.endsWith(Const.FILE_SEPARATOR) ? "" :
Const.FILE_SEPARATOR)
- + classFolder;
+ // The current key names the folder to save in, legacy keys name folders
we still read from.
+ //
+ List<String> keys = HopMetadataUtil.getAllKeys(hopMetadata);
+ String serializerBaseFolderName = calculateTypeFolder(keys.get(0));
+ List<String> legacyFolders = new ArrayList<>();
+ for (String legacyKey : keys.subList(1, keys.size())) {
+ legacyFolders.add(calculateTypeFolder(legacyKey));
+ }
String description = TranslateUtil.translate(hopMetadata.name(),
managedClass);
return new JsonMetadataSerializer<>(
- this, serializerBaseFolderName, managedClass, variables, description);
+ this, serializerBaseFolderName, legacyFolders, managedClass,
variables, description);
+ }
+
+ private String calculateTypeFolder(String key) {
+ return baseFolder
+ + (baseFolder.endsWith(Const.FILE_SEPARATOR) ? "" :
Const.FILE_SEPARATOR)
+ + key;
}
/**
diff --git
a/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java
b/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java
index fde873cf1f..acb478c559 100644
---
a/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java
+++
b/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java
@@ -28,7 +28,9 @@ import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
@@ -55,6 +57,13 @@ public class JsonMetadataSerializer<T extends IHopMetadata>
implements IHopMetad
@Setter @Getter protected Class<T> managedClass;
@Setter @Getter protected String description;
+ /**
+ * Folders named after the {@link
org.apache.hop.metadata.api.HopMetadata#legacyKeys()} of the
+ * managed class. Objects are still read from these folders, but they are
always saved in the base
+ * folder.
+ */
+ @Setter @Getter protected List<String> legacyFolders;
+
protected JsonMetadataParser<T> parser;
protected IVariables variables;
@@ -67,7 +76,24 @@ public class JsonMetadataSerializer<T extends IHopMetadata>
implements IHopMetad
Class<T> managedClass,
IVariables variables,
String description) {
+ this(
+ metadataProvider,
+ baseFolder,
+ Collections.emptyList(),
+ managedClass,
+ variables,
+ description);
+ }
+
+ public JsonMetadataSerializer(
+ IHopMetadataProvider metadataProvider,
+ String baseFolder,
+ List<String> legacyFolders,
+ Class<T> managedClass,
+ IVariables variables,
+ String description) {
this.metadataProvider = metadataProvider;
+ this.legacyFolders = legacyFolders == null ? Collections.emptyList() :
legacyFolders;
this.baseFolder = baseFolder;
this.managedClass = managedClass;
this.parser = new JsonMetadataParser<>(managedClass, metadataProvider);
@@ -78,10 +104,6 @@ public class JsonMetadataSerializer<T extends IHopMetadata>
implements IHopMetad
@Override
public List<T> loadAll() throws HopException {
List<T> list = new ArrayList<>();
- validateBaseFolder(false);
- if (!baseFolderExists) {
- return list;
- }
List<String> names = listObjectNames();
Collections.sort(names);
for (String name : names) {
@@ -120,12 +142,11 @@ public class JsonMetadataSerializer<T extends
IHopMetadata> implements IHopMetad
if (name == null) {
throw new HopException("Error: you need to specify the name of the
metadata object to load");
}
- if (!exists(name)) {
+ String filename = findFilename(name);
+ if (filename == null) {
return null;
}
- String filename = calculateFilename(name);
-
try {
// Load the JSON in a streaming fashion so we can parse the properties
one by one...
//
@@ -235,18 +256,86 @@ public class JsonMetadataSerializer<T extends
IHopMetadata> implements IHopMetad
throw new HopException(
"Unable to save object '" + t.getName() + "' to JSON file '" +
filename + "'", e);
}
+
+ // The object now lives in the base folder: a copy in a legacy folder is
outdated.
+ //
+ removeLegacyCopies(t.getName());
}
+ /** Removes the files of an object from the legacy folders, moving it to the
base folder. */
+ private void removeLegacyCopies(String name) throws HopException {
+ for (String legacyFolder : legacyFolders) {
+ // On a case-insensitive file system a key which only changed case is
the same folder: we
+ // would delete the file we just saved.
+ if (legacyFolder.equalsIgnoreCase(baseFolder)) {
+ continue;
+ }
+ String legacyFilename = calculateFilename(legacyFolder, name);
+ try {
+ if (HopVfs.fileExists(legacyFilename)) {
+ HopVfs.getFileObject(legacyFilename).delete();
+ if (HopLogStore.isInitialized()) {
+ LogChannel.GENERAL.logBasic(
+ "Metadata object '"
+ + name
+ + "' was moved from legacy folder '"
+ + legacyFolder
+ + "' to folder '"
+ + baseFolder
+ + "'");
+ }
+ }
+ } catch (Exception e) {
+ throw new HopException(
+ "Error removing the legacy copy of metadata object '"
+ + name
+ + "' in file '"
+ + legacyFilename
+ + "'",
+ e);
+ }
+ }
+ }
+
+ /**
+ * @param name the name of the metadata object
+ * @return the file in the base folder where the object is saved
+ */
public String calculateFilename(String name) {
- return baseFolder + "/" + name + ".json";
+ return calculateFilename(baseFolder, name);
+ }
+
+ private static String calculateFilename(String folder, String name) {
+ return folder + "/" + name + ".json";
+ }
+
+ /**
+ * Finds the file which holds an existing object: in the base folder or,
failing that, in one of
+ * the legacy folders.
+ *
+ * @param name the name of the metadata object
+ * @return the file of the object or null if it doesn't exist
+ * @throws HopException in case of a file system error
+ */
+ public String findFilename(String name) throws HopException {
+ validateBaseFolder(false);
+ if (baseFolderExists) {
+ String filename = calculateFilename(name);
+ if (HopVfs.fileExists(filename)) {
+ return filename;
+ }
+ }
+ for (String legacyFolder : legacyFolders) {
+ String filename = calculateFilename(legacyFolder, name);
+ if (HopVfs.fileExists(filename)) {
+ return filename;
+ }
+ }
+ return null;
}
@Override
public T delete(String name) throws HopException {
- // Make sure the base folder exists
- //
- validateBaseFolder(true);
-
if (name == null) {
throw new HopException(
"Error: you need to specify the name of the metadata object to
delete");
@@ -255,53 +344,69 @@ public class JsonMetadataSerializer<T extends
IHopMetadata> implements IHopMetad
throw new HopException("Error: Object '" + name + "' doesn't exist");
}
T t = load(name);
- String filename = calculateFilename(name);
- try {
- boolean deleted = HopVfs.getFileObject(filename).delete();
- if (!deleted) {
+
+ // The object can have been saved in the base folder while an older copy
is still sitting in a
+ // legacy folder: remove them all or the old copy would come back. The
legacy copies go first
+ // and the base folder file last: if a delete fails halfway, the file that
is left is the one
+ // load() reads, never a stale legacy copy in its place.
+ //
+ List<String> folders = new ArrayList<>(legacyFolders);
+ folders.add(baseFolder);
+ for (String folder : folders) {
+ String filename = calculateFilename(folder, name);
+ if (!HopVfs.fileExists(filename)) {
+ continue;
+ }
+ try (FileObject file = HopVfs.getFileObject(filename)) {
+ if (!file.delete()) {
+ throw new HopException(
+ "Error: Object '" + name + "' could not be deleted, filename : "
+ filename);
+ }
+ } catch (FileSystemException e) {
throw new HopException(
- "Error: Object '" + name + "' could not be deleted, filename : " +
filename);
+ "Error deleting Object '" + name + "' with filename : " +
filename, e);
}
- } catch (FileSystemException e) {
- throw new HopException("Error deleting Object '" + name + "' with
filename : " + filename);
}
return t;
}
@Override
public List<String> listObjectNames() throws HopException {
- List<String> names = new ArrayList<>();
+ // An object in the base folder and a legacy folder is listed once.
+ //
+ Set<String> names = new LinkedHashSet<>();
- // Read-only access doesn't require a folder
+ // Read-only access doesn't require a folder. If there is none, we simply
don't have objects
+ // of the given type: this is not an error.
+ //
validateBaseFolder(false);
- if (!baseFolderExists) {
- // This is not an error. We simply don't have objects of the given type.
- //
- return names;
+ if (baseFolderExists) {
+ addObjectNames(baseFolder, names);
}
+ for (String legacyFolder : legacyFolders) {
+ if (HopVfs.fileExists(legacyFolder)) {
+ addObjectNames(legacyFolder, names);
+ }
+ }
+ return new ArrayList<>(names);
+ }
- FileObject folder = HopVfs.getFileObject(baseFolder);
-
+ private static void addObjectNames(String folderName, Set<String> names)
throws HopException {
try {
+ FileObject folder = HopVfs.getFileObject(folderName);
List<FileObject> jsonFiles = HopVfs.findFiles(folder, "json", false);
for (FileObject jsonFile : jsonFiles) {
String baseName = jsonFile.getName().getBaseName();
names.add(baseName.replaceAll("\\.json$", ""));
}
- return names;
} catch (Exception e) {
- throw new HopException("Error searching for JSON files", e);
+ throw new HopException("Error searching for JSON files in folder '" +
folderName + "'", e);
}
}
@Override
public boolean exists(String name) throws HopException {
- // Read-only access doesn't require a folder
- validateBaseFolder(false);
- if (!baseFolderExists) {
- return false;
- }
- return HopVfs.fileExists(calculateFilename(name));
+ return findFilename(name) != null;
}
/**
@@ -314,12 +419,11 @@ public class JsonMetadataSerializer<T extends
IHopMetadata> implements IHopMetad
if (name == null) {
throw new HopException("Error: you need to specify the name of the
metadata object to load");
}
- validateBaseFolder(false);
- if (!baseFolderExists || !exists(name)) {
+ String filename = findFilename(name);
+ if (filename == null) {
throw new HopException("Object '" + name + "' does not exist");
}
- String filename = calculateFilename(name);
try (InputStream fileInputStream = HopVfs.getInputStream(filename)) {
JsonFactory jsonFactory = new JsonFactory();
try (com.fasterxml.jackson.core.JsonParser jsonParser =
diff --git
a/core/src/main/java/org/apache/hop/metadata/util/HopMetadataUtil.java
b/core/src/main/java/org/apache/hop/metadata/util/HopMetadataUtil.java
index f8b2a4144d..92bfc3ad86 100644
--- a/core/src/main/java/org/apache/hop/metadata/util/HopMetadataUtil.java
+++ b/core/src/main/java/org/apache/hop/metadata/util/HopMetadataUtil.java
@@ -23,11 +23,14 @@ import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.hop.core.Const;
import org.apache.hop.core.encryption.Encr;
+import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.metadata.api.HopMetadata;
import org.apache.hop.metadata.api.IHopMetadata;
import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
import org.apache.hop.metadata.serializer.json.JsonMetadataProvider;
+import org.apache.hop.metadata.serializer.json.JsonMetadataSerializer;
import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
public class HopMetadataUtil {
@@ -62,11 +65,85 @@ public class HopMetadataUtil {
return new MultiMetadataProvider(Encr.getEncoder(), providers, variables);
}
+ /**
+ * The file holding a metadata object, found the way {@link
+ *
org.apache.hop.metadata.serializer.multi.MultiMetadataSerializer#load(String)}
finds the object
+ * itself: the last provider which has it wins, which is the child project
when a project inherits
+ * from another. Within a file based provider the folder of the current key
comes before the
+ * folders of the {@link HopMetadata#legacyKeys()}.
+ *
+ * @param provider the provider, can be a multi-provider
+ * @param managedClass the metadata type
+ * @param name the name of the metadata object
+ * @return the file, or null when no provider has the object or the one
which has it is not file
+ * based
+ * @throws HopException in case of a file system error
+ */
+ public static <T extends IHopMetadata> String findFilename(
+ IHopMetadataProvider provider, Class<T> managedClass, String name)
throws HopException {
+ List<IHopMetadataProvider> providers = getProviders(provider);
+ for (int i = providers.size() - 1; i >= 0; i--) {
+ IHopMetadataSerializer<T> serializer =
providers.get(i).getSerializer(managedClass);
+ if (serializer instanceof JsonMetadataSerializer jsonSerializer) {
+ String filename = jsonSerializer.findFilename(name);
+ if (filename != null) {
+ return filename;
+ }
+ } else if (serializer.exists(name)) {
+ return null;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * The providers behind a provider, a multi-provider unwrapped, in the order
it lists them: the
+ * parent project first, the child project last.
+ */
+ public static List<IHopMetadataProvider> getProviders(IHopMetadataProvider
provider) {
+ List<IHopMetadataProvider> providers = new ArrayList<>();
+ if (provider instanceof MultiMetadataProvider multiProvider) {
+ for (IHopMetadataProvider childProvider : multiProvider.getProviders()) {
+ providers.addAll(getProviders(childProvider));
+ }
+ } else if (provider != null) {
+ providers.add(provider);
+ }
+ return providers;
+ }
+
public static <T extends IHopMetadata> HopMetadata getHopMetadataAnnotation(
Class<T> managedClass) {
return managedClass.getAnnotation(HopMetadata.class);
}
+ /**
+ * The current key of a metadata type followed by its {@link
HopMetadata#legacyKeys()}. This is
+ * the order in which to look for stored objects: the current key always
wins.
+ *
+ * @param annotation the metadata type annotation
+ * @return the current key and the legacy keys, without empty or duplicate
values
+ */
+ public static List<String> getAllKeys(HopMetadata annotation) {
+ List<String> keys = new ArrayList<>();
+ keys.add(Const.NVL(annotation.key(), annotation.name()));
+ for (String legacyKey : annotation.legacyKeys()) {
+ if (StringUtils.isNotEmpty(legacyKey) && !keys.contains(legacyKey)) {
+ keys.add(legacyKey);
+ }
+ }
+ return keys;
+ }
+
+ /**
+ * @param annotation the metadata type annotation
+ * @param key a metadata type key, for example from a filter or a serialized
metadata export
+ * @return true if the key is the current key or one of the legacy keys of
the metadata type
+ */
+ public static boolean matchesKey(HopMetadata annotation, String key) {
+ return annotation != null && key != null &&
getAllKeys(annotation).contains(key);
+ }
+
public static String[] getHopMetadataKeys(IHopMetadataProvider provider) {
List<String> keys = new ArrayList<>();
for (Class<IHopMetadata> metadataClass : provider.getMetadataClasses()) {
diff --git
a/core/src/test/java/org/apache/hop/metadata/serializer/MetadataGuiFlows.java
b/core/src/test/java/org/apache/hop/metadata/serializer/MetadataGuiFlows.java
new file mode 100644
index 0000000000..ebe12ee59a
--- /dev/null
+++
b/core/src/test/java/org/apache/hop/metadata/serializer/MetadataGuiFlows.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.metadata.serializer;
+
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+
+/**
+ * The serializer calls the Hop GUI makes to rename and duplicate metadata
objects, without the GUI
+ * around them. Keep these in sync with the classes they mirror so the tests
using them exercise
+ * what users actually do.
+ */
+public final class MetadataGuiFlows {
+
+ private MetadataGuiFlows() {}
+
+ /**
+ * Renaming an element in the metadata perspective tree: {@code
MetadataManager.rename()}.
+ *
+ * @return false if the new name is already taken, like the GUI which
refuses the rename
+ */
+ public static <T extends IHopMetadata> boolean renameInTree(
+ IHopMetadataSerializer<T> serializer, String oldName, String newName)
throws HopException {
+ if (serializer.exists(newName)) {
+ return false;
+ }
+ T metadata = serializer.load(oldName);
+ metadata.setName(newName);
+ serializer.save(metadata);
+ serializer.delete(oldName);
+ return true;
+ }
+
+ /**
+ * Changing the name of an open element in its editor and saving it: {@code
+ * MetadataEditor.save()}.
+ *
+ * @throws HopException if the new name is already taken, like the GUI
+ */
+ public static <T extends IHopMetadata> void renameInEditor(
+ IHopMetadataSerializer<T> serializer, T metadata, String originalName)
throws HopException {
+ boolean isRename = false;
+ if (!originalName.equals(metadata.getName())) {
+ if (serializer.exists(metadata.getName())) {
+ throw new HopException("Name '" + metadata.getName() + "' already
exists");
+ }
+ isRename = true;
+ }
+ serializer.save(metadata);
+ if (isRename && serializer.exists(originalName)) {
+ serializer.delete(originalName);
+ }
+ }
+
+ /**
+ * Duplicating an element in the metadata perspective: {@code
+ * MetadataPerspective.duplicateMetadata()}.
+ *
+ * @return the name of the copy
+ */
+ public static <T extends IHopMetadata> String duplicate(
+ IHopMetadataSerializer<T> serializer, String name) throws HopException {
+ T metadata = serializer.load(name);
+ int copyNr = 2;
+ while (true) {
+ String newName = name + " " + copyNr;
+ if (!serializer.exists(newName)) {
+ metadata.setName(newName);
+ serializer.save(metadata);
+ return newName;
+ }
+ copyNr++;
+ }
+ }
+}
diff --git
a/core/src/test/java/org/apache/hop/metadata/serializer/json/JsonMetadataLegacyKeysTest.java
b/core/src/test/java/org/apache/hop/metadata/serializer/json/JsonMetadataLegacyKeysTest.java
new file mode 100644
index 0000000000..c92e5787f2
--- /dev/null
+++
b/core/src/test/java/org/apache/hop/metadata/serializer/json/JsonMetadataLegacyKeysTest.java
@@ -0,0 +1,631 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.metadata.serializer.json;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.encryption.HopTwoWayPasswordEncoder;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.metadata.serializer.BaseMetadataProvider;
+import org.apache.hop.metadata.serializer.MetadataGuiFlows;
+import org.apache.hop.metadata.serializer.json.occupation.Occupation;
+import org.apache.hop.metadata.serializer.json.renamed.CaseRenamedType;
+import org.apache.hop.metadata.serializer.json.renamed.RenamedType;
+import org.apache.hop.metadata.serializer.json.renamed.TwiceRenamedType;
+import org.apache.hop.metadata.util.HopMetadataUtil;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * A metadata type which is renamed keeps its old key in {@link
HopMetadata#legacyKeys()}: objects
+ * in the old folder are still loaded, and saving them moves them to the new
folder (issue #5597).
+ *
+ * <p>Most tests use a "mixed" project, which is what users have while a
project migrates: some
+ * objects only in the legacy folder, some only in the current folder and some
in both.
+ */
+class JsonMetadataLegacyKeysTest {
+
+ @TempDir Path folder;
+
+ private JsonMetadataProvider provider;
+ private JsonMetadataSerializer<RenamedType> serializer;
+ private Path legacyFolder;
+ private Path currentFolder;
+
+ @BeforeEach
+ void setUp() throws Exception {
+ provider =
+ new JsonMetadataProvider(
+ new HopTwoWayPasswordEncoder(),
+ folder.toString(),
+ Variables.getADefaultVariableSpace());
+ serializer = (JsonMetadataSerializer<RenamedType>)
provider.getSerializer(RenamedType.class);
+ legacyFolder = folder.resolve("RenamedType");
+ currentFolder = folder.resolve("renamed-type");
+ }
+
+ private static void writeObject(Path typeFolder, String name, String
description)
+ throws Exception {
+ Files.createDirectories(typeFolder);
+ Files.write(
+ typeFolder.resolve(name + ".json"),
+ ("{\"name\":\"" + name + "\",\"description\":\"" + description +
"\"}").getBytes(UTF_8));
+ }
+
+ /**
+ * Writes an object the way an older version of Hop did: in the folder named
after the old key.
+ */
+ private void writeLegacyObject(String name, String description) throws
Exception {
+ writeObject(legacyFolder, name, description);
+ }
+
+ private void writeCurrentObject(String name, String description) throws
Exception {
+ writeObject(currentFolder, name, description);
+ }
+
+ /** legacy-only: only in the old folder, current-only: only in the new one,
both: in both. */
+ private void writeMixedProject() throws Exception {
+ writeLegacyObject("legacy-only", "legacy");
+ writeCurrentObject("current-only", "current");
+ writeLegacyObject("both", "outdated legacy copy");
+ writeCurrentObject("both", "current copy");
+ }
+
+ private boolean inLegacy(String name) {
+ return Files.exists(legacyFolder.resolve(name + ".json"));
+ }
+
+ private boolean inCurrent(String name) {
+ return Files.exists(currentFolder.resolve(name + ".json"));
+ }
+
+ private List<String> sortedNames() throws HopException {
+ List<String> names = new ArrayList<>(serializer.listObjectNames());
+ names.sort(String::compareTo);
+ return names;
+ }
+
+ //
---------------------------------------------------------------------------------------------
+ // Reading
+ //
---------------------------------------------------------------------------------------------
+
+ @Test
+ void testLoadFromLegacyFolder() throws Exception {
+ writeLegacyObject("old", "Saved by an older version");
+
+ assertEquals(List.of("old"), serializer.listObjectNames());
+ assertTrue(serializer.exists("old"));
+ RenamedType loaded = serializer.load("old");
+ assertNotNull(loaded);
+ assertEquals("Saved by an older version", loaded.getDescription());
+ assertEquals(1, serializer.loadAll().size());
+ assertEquals(
+ legacyFolder.resolve("old.json").toString(),
+ Path.of(serializer.findFilename("old")).toString());
+
+ // Only reading doesn't move anything.
+ assertFalse(Files.exists(currentFolder));
+ }
+
+ @Test
+ void testMixedProjectIsReadAsOne() throws Exception {
+ writeMixedProject();
+
+ assertEquals(List.of("both", "current-only", "legacy-only"),
sortedNames());
+ assertTrue(serializer.exists("legacy-only"));
+ assertTrue(serializer.exists("current-only"));
+ assertTrue(serializer.exists("both"));
+ assertFalse(serializer.exists("nowhere"));
+ assertNull(serializer.load("nowhere"));
+ assertNull(serializer.findFilename("nowhere"));
+
+ assertEquals("legacy", serializer.load("legacy-only").getDescription());
+ assertEquals("current", serializer.load("current-only").getDescription());
+ // The current folder wins over the legacy folder.
+ assertEquals("current copy", serializer.load("both").getDescription());
+ assertEquals(
+ currentFolder.resolve("both.json").toString(),
+ Path.of(serializer.findFilename("both")).toString());
+
+ List<RenamedType> all = serializer.loadAll();
+ assertEquals(3, all.size());
+ assertTrue(all.stream().anyMatch(t -> "current
copy".equals(t.getDescription())));
+ assertFalse(all.stream().anyMatch(t -> "outdated legacy
copy".equals(t.getDescription())));
+ }
+
+ /** The provider name is how a multi-provider knows where to save an object
back to. */
+ @Test
+ void testObjectsFromLegacyFolderKnowTheirProvider() throws Exception {
+ writeLegacyObject("old", "legacy");
+
+ assertEquals(provider.getDescription(),
serializer.load("old").getMetadataProviderName());
+ }
+
+ @Test
+ void testReadVirtualPath() throws Exception {
+ Files.createDirectories(legacyFolder);
+ Files.write(
+ legacyFolder.resolve("filed.json"),
+
"{\"name\":\"filed\",\"virtualPath\":\"/legacy/path\"}".getBytes(UTF_8));
+ Files.write(
+ legacyFolder.resolve("both.json"),
+
"{\"name\":\"both\",\"virtualPath\":\"/legacy/path\"}".getBytes(UTF_8));
+ Files.createDirectories(currentFolder);
+ Files.write(
+ currentFolder.resolve("both.json"),
+
"{\"name\":\"both\",\"virtualPath\":\"/current/path\"}".getBytes(UTF_8));
+
+ assertEquals("/legacy/path", serializer.readVirtualPath("filed"));
+ assertEquals("/current/path", serializer.readVirtualPath("both"));
+ assertThrows(HopException.class, () ->
serializer.readVirtualPath("nowhere"));
+ }
+
+ @Test
+ void testNoFoldersAtAll() throws Exception {
+ assertTrue(serializer.listObjectNames().isEmpty());
+ assertTrue(serializer.loadAll().isEmpty());
+ assertFalse(serializer.exists("anything"));
+
+ // Reading never creates folders.
+ assertFalse(Files.exists(currentFolder));
+ assertFalse(Files.exists(legacyFolder));
+ }
+
+ @Test
+ void testOnlyJsonFilesDirectlyInTheLegacyFolderAreObjects() throws Exception
{
+ writeLegacyObject("real", "legacy");
+ Files.write(legacyFolder.resolve("notes.txt"), "not
metadata".getBytes(UTF_8));
+ Files.createDirectories(legacyFolder.resolve("sub"));
+ writeObject(legacyFolder.resolve("sub"), "nested", "not listed");
+
+ assertEquals(List.of("real"), serializer.listObjectNames());
+ }
+
+ @Test
+ void testNamesWithSpacesAndDotsInLegacyFolder() throws Exception {
+ writeLegacyObject("My connection v1.2", "legacy");
+
+ assertEquals(List.of("My connection v1.2"), serializer.listObjectNames());
+ assertEquals("legacy", serializer.load("My connection
v1.2").getDescription());
+
+ RenamedType loaded = serializer.load("My connection v1.2");
+ serializer.save(loaded);
+ assertTrue(inCurrent("My connection v1.2"));
+ assertFalse(inLegacy("My connection v1.2"));
+ }
+
+ /** A broken legacy object shouldn't hide the others, while loading it by
name still reports. */
+ @Test
+ void testCorruptLegacyObjectIsSkippedByLoadAll() throws Exception {
+ writeMixedProject();
+ Files.write(legacyFolder.resolve("corrupt.json"), "{ \"name\":
".getBytes(UTF_8));
+
+ assertEquals(4, serializer.listObjectNames().size());
+ assertEquals(3, serializer.loadAll().size());
+ assertThrows(HopException.class, () -> serializer.load("corrupt"));
+ }
+
+ //
---------------------------------------------------------------------------------------------
+ // Saving
+ //
---------------------------------------------------------------------------------------------
+
+ @Test
+ void testSaveMovesObjectToCurrentFolder() throws Exception {
+ writeLegacyObject("old", "Saved by an older version");
+
+ RenamedType loaded = serializer.load("old");
+ loaded.setDescription("Changed");
+ serializer.save(loaded);
+
+ assertTrue(inCurrent("old"));
+ assertFalse(inLegacy("old"));
+ assertEquals(List.of("old"), serializer.listObjectNames());
+ assertEquals("Changed", serializer.load("old").getDescription());
+ }
+
+ @Test
+ void testSavingOneObjectOnlyMovesThatObject() throws Exception {
+ writeMixedProject();
+ writeLegacyObject("other-legacy", "legacy");
+
+ serializer.save(serializer.load("legacy-only"));
+
+ assertTrue(inCurrent("legacy-only"));
+ assertFalse(inLegacy("legacy-only"));
+ // Untouched objects stay where they are.
+ assertTrue(inLegacy("other-legacy"));
+ assertFalse(inCurrent("other-legacy"));
+ assertTrue(inLegacy("both"));
+ assertEquals(List.of("both", "current-only", "legacy-only",
"other-legacy"), sortedNames());
+ }
+
+ /** Saving an object in both folders removes the outdated legacy copy. */
+ @Test
+ void testSavingAnObjectInBothFoldersRemovesTheLegacyCopy() throws Exception {
+ writeMixedProject();
+
+ RenamedType both = serializer.load("both");
+ both.setDescription("saved again");
+ serializer.save(both);
+
+ assertTrue(inCurrent("both"));
+ assertFalse(inLegacy("both"));
+ assertEquals("saved again", serializer.load("both").getDescription());
+ }
+
+ @Test
+ void testNewObjectsGoToCurrentFolder() throws Exception {
+ serializer.save(new RenamedType("new", "Brand new"));
+
+ assertTrue(inCurrent("new"));
+ assertFalse(Files.exists(legacyFolder));
+ }
+
+ @Test
+ void testSavingACurrentObjectDoesNotTouchTheLegacyFolder() throws Exception {
+ writeMixedProject();
+
+ serializer.save(serializer.load("current-only"));
+
+ assertTrue(inCurrent("current-only"));
+ assertFalse(inLegacy("current-only"));
+ assertTrue(inLegacy("legacy-only"));
+ }
+
+ @Test
+ void testMigrateAWholeProjectBySavingEverything() throws Exception {
+ writeMixedProject();
+ writeLegacyObject("another", "legacy");
+
+ for (RenamedType t : serializer.loadAll()) {
+ serializer.save(t);
+ }
+
+ try (var files = Files.list(legacyFolder)) {
+ assertEquals(0, files.count(), "the legacy folder is empty after saving
everything");
+ }
+ assertEquals(List.of("another", "both", "current-only", "legacy-only"),
sortedNames());
+ assertEquals("current copy", serializer.load("both").getDescription());
+ }
+
+ //
---------------------------------------------------------------------------------------------
+ // Deleting
+ //
---------------------------------------------------------------------------------------------
+
+ @Test
+ void testDeleteRemovesAllCopies() throws Exception {
+ writeMixedProject();
+
+ RenamedType deleted = serializer.delete("both");
+
+ // The object returned is the one the user saw: the current copy.
+ assertEquals("current copy", deleted.getDescription());
+ assertFalse(serializer.exists("both"));
+ assertFalse(inCurrent("both"));
+ assertFalse(inLegacy("both"));
+ assertEquals(List.of("current-only", "legacy-only"), sortedNames());
+ }
+
+ @Test
+ void testDeleteLegacyOnlyObject() throws Exception {
+ writeLegacyObject("old", "legacy");
+
+ serializer.delete("old");
+
+ assertFalse(serializer.exists("old"));
+ assertFalse(inLegacy("old"));
+ // Deleting doesn't create the current folder as a side effect.
+ assertFalse(Files.exists(currentFolder));
+ }
+
+ /**
+ * A delete that fails on a legacy copy must leave the current file in
place: the legacy copies go
+ * first, so the stale one can never end up being the object.
+ */
+ @Test
+ void testFailedDeleteOfALegacyCopyKeepsTheCurrentFile() throws Exception {
+ writeCurrentObject("stuck", "current copy");
+ // A folder with something in it, where the legacy copy would be: it can't
be deleted.
+ Path undeletable = legacyFolder.resolve("stuck.json");
+ Files.createDirectories(undeletable);
+ Files.writeString(undeletable.resolve("keep"), "x");
+
+ assertThrows(HopException.class, () -> serializer.delete("stuck"));
+
+ assertTrue(inCurrent("stuck"));
+ assertEquals("current copy", serializer.load("stuck").getDescription());
+ }
+
+ @Test
+ void testDeleteUnknownObjectFails() {
+ assertThrows(HopException.class, () -> serializer.delete("nowhere"));
+ }
+
+ //
---------------------------------------------------------------------------------------------
+ // Renaming, as the GUI does it
+ //
---------------------------------------------------------------------------------------------
+
+ @Test
+ void testRenameLegacyObjectInTree() throws Exception {
+ writeMixedProject();
+
+ assertTrue(MetadataGuiFlows.renameInTree(serializer, "legacy-only",
"renamed"));
+
+ assertTrue(inCurrent("renamed"));
+ assertFalse(inLegacy("legacy-only"));
+ assertFalse(inCurrent("legacy-only"));
+ assertEquals("legacy", serializer.load("renamed").getDescription());
+ assertEquals(List.of("both", "current-only", "renamed"), sortedNames());
+ }
+
+ @Test
+ void testRenameObjectInBothFoldersInTree() throws Exception {
+ writeMixedProject();
+
+ assertTrue(MetadataGuiFlows.renameInTree(serializer, "both", "renamed"));
+
+ // The renamed object is the current copy, and no stale copy of the old
name is left behind to
+ // reappear under the old name.
+ assertEquals("current copy", serializer.load("renamed").getDescription());
+ assertFalse(serializer.exists("both"));
+ assertFalse(inLegacy("both"));
+ assertFalse(inCurrent("both"));
+ }
+
+ @Test
+ void testRenameLegacyObjectInEditor() throws Exception {
+ writeMixedProject();
+
+ RenamedType metadata = serializer.load("legacy-only");
+ metadata.setName("renamed");
+ metadata.setDescription("edited");
+ MetadataGuiFlows.renameInEditor(serializer, metadata, "legacy-only");
+
+ assertTrue(inCurrent("renamed"));
+ assertFalse(serializer.exists("legacy-only"));
+ assertEquals("edited", serializer.load("renamed").getDescription());
+ }
+
+ /** Saving an open editor without changing the name is a plain save: it
migrates the object. */
+ @Test
+ void testSaveInEditorWithoutRename() throws Exception {
+ writeLegacyObject("old", "legacy");
+
+ RenamedType metadata = serializer.load("old");
+ metadata.setDescription("edited");
+ MetadataGuiFlows.renameInEditor(serializer, metadata, "old");
+
+ assertTrue(inCurrent("old"));
+ assertFalse(inLegacy("old"));
+ assertEquals("edited", serializer.load("old").getDescription());
+ }
+
+ /** A name only used in the legacy folder is still taken: renaming onto it
would lose data. */
+ @Test
+ void testRenameOntoNameInLegacyFolderIsRefused() throws Exception {
+ writeMixedProject();
+
+ assertFalse(MetadataGuiFlows.renameInTree(serializer, "current-only",
"legacy-only"));
+ RenamedType metadata = serializer.load("current-only");
+ metadata.setName("legacy-only");
+ assertThrows(
+ HopException.class,
+ () -> MetadataGuiFlows.renameInEditor(serializer, metadata,
"current-only"));
+
+ // Nothing changed.
+ assertEquals("legacy", serializer.load("legacy-only").getDescription());
+ assertEquals("current", serializer.load("current-only").getDescription());
+ assertEquals(List.of("both", "current-only", "legacy-only"),
sortedNames());
+ }
+
+ @Test
+ void testRenameOntoNameInCurrentFolderIsRefused() throws Exception {
+ writeMixedProject();
+
+ assertFalse(MetadataGuiFlows.renameInTree(serializer, "legacy-only",
"current-only"));
+ assertTrue(inLegacy("legacy-only"));
+ assertEquals("current", serializer.load("current-only").getDescription());
+ }
+
+ @Test
+ void testRenameBackAndForth() throws Exception {
+ writeLegacyObject("a", "legacy");
+
+ assertTrue(MetadataGuiFlows.renameInTree(serializer, "a", "b"));
+ assertTrue(MetadataGuiFlows.renameInTree(serializer, "b", "a"));
+
+ assertEquals(List.of("a"), serializer.listObjectNames());
+ assertTrue(inCurrent("a"));
+ assertFalse(inLegacy("a"));
+ assertEquals("legacy", serializer.load("a").getDescription());
+ }
+
+ //
---------------------------------------------------------------------------------------------
+ // Duplicating, as the GUI does it
+ //
---------------------------------------------------------------------------------------------
+
+ @Test
+ void testDuplicateLegacyObject() throws Exception {
+ writeLegacyObject("conn", "legacy");
+
+ String copy = MetadataGuiFlows.duplicate(serializer, "conn");
+
+ assertEquals("conn 2", copy);
+ assertTrue(inCurrent("conn 2"));
+ assertEquals("legacy", serializer.load("conn 2").getDescription());
+ // The original isn't touched: duplicating isn't editing it.
+ assertTrue(inLegacy("conn"));
+ assertFalse(inCurrent("conn"));
+ assertEquals(List.of("conn", "conn 2"), sortedNames());
+ }
+
+ /** A copy name used in the legacy folder is taken, so the next number is
used. */
+ @Test
+ void testDuplicateSkipsNamesTakenInEitherFolder() throws Exception {
+ writeCurrentObject("conn", "current");
+ writeLegacyObject("conn 2", "legacy copy");
+ writeCurrentObject("conn 3", "current copy");
+
+ String copy = MetadataGuiFlows.duplicate(serializer, "conn");
+
+ assertEquals("conn 4", copy);
+ assertEquals("current", serializer.load("conn 4").getDescription());
+ assertEquals("legacy copy", serializer.load("conn 2").getDescription());
+ assertTrue(inLegacy("conn 2"));
+ assertEquals(List.of("conn", "conn 2", "conn 3", "conn 4"), sortedNames());
+ }
+
+ @Test
+ void testDuplicateObjectInBothFoldersCopiesTheCurrentCopy() throws Exception
{
+ writeMixedProject();
+
+ String copy = MetadataGuiFlows.duplicate(serializer, "both");
+
+ assertEquals("current copy", serializer.load(copy).getDescription());
+ }
+
+ //
---------------------------------------------------------------------------------------------
+ // Types renamed more than once, or only in case
+ //
---------------------------------------------------------------------------------------------
+
+ @Test
+ void testTypeRenamedTwice() throws Exception {
+ JsonMetadataSerializer<TwiceRenamedType> twice =
+ (JsonMetadataSerializer<TwiceRenamedType>)
provider.getSerializer(TwiceRenamedType.class);
+ Path current = folder.resolve("twice-renamed");
+ Path previous = folder.resolve("TwiceRenamed");
+ Path original = folder.resolve("OriginalName");
+
+ writeObject(original, "only-original", "original");
+ writeObject(previous, "only-previous", "previous");
+ writeObject(original, "previous-and-original", "original");
+ writeObject(previous, "previous-and-original", "previous");
+ writeObject(original, "everywhere", "original");
+ writeObject(previous, "everywhere", "previous");
+ writeObject(current, "everywhere", "current");
+
+ List<String> names = new ArrayList<>(twice.listObjectNames());
+ names.sort(String::compareTo);
+ assertEquals(
+ List.of("everywhere", "only-original", "only-previous",
"previous-and-original"), names);
+
+ // The current key wins, then the most recent legacy key.
+ assertEquals("current", twice.load("everywhere").getDescription());
+ assertEquals("previous",
twice.load("previous-and-original").getDescription());
+ assertEquals("original", twice.load("only-original").getDescription());
+
+ // Saving removes the object from every legacy folder.
+ twice.save(twice.load("everywhere"));
+ twice.save(twice.load("only-original"));
+ assertFalse(Files.exists(original.resolve("everywhere.json")));
+ assertFalse(Files.exists(previous.resolve("everywhere.json")));
+ assertFalse(Files.exists(original.resolve("only-original.json")));
+ assertTrue(Files.exists(current.resolve("only-original.json")));
+
+ // Deleting removes every copy.
+ twice.delete("previous-and-original");
+ assertFalse(twice.exists("previous-and-original"));
+ assertFalse(Files.exists(original.resolve("previous-and-original.json")));
+ assertFalse(Files.exists(previous.resolve("previous-and-original.json")));
+ }
+
+ /**
+ * A key which only changed case is the same folder on a case-insensitive
file system (macOS,
+ * Windows). Saving must then never delete the file it just wrote. On a
case-sensitive file system
+ * the old copy stays behind but the current one wins, so either way the
saved object is loaded.
+ */
+ @Test
+ void testCaseOnlyRenameNeverLosesTheSavedObject() throws Exception {
+ JsonMetadataSerializer<CaseRenamedType> caseSerializer =
+ (JsonMetadataSerializer<CaseRenamedType>)
provider.getSerializer(CaseRenamedType.class);
+ writeObject(folder.resolve("Case-Renamed"), "obj", "old");
+
+ CaseRenamedType loaded = caseSerializer.load("obj");
+ assertEquals("old", loaded.getDescription());
+ loaded.setDescription("saved");
+ caseSerializer.save(loaded);
+
+
assertTrue(Files.exists(folder.resolve("case-renamed").resolve("obj.json")));
+ assertEquals("saved", caseSerializer.load("obj").getDescription());
+ assertEquals(List.of("obj"), caseSerializer.listObjectNames());
+
+ caseSerializer.delete("obj");
+ assertFalse(caseSerializer.exists("obj"));
+ }
+
+ //
---------------------------------------------------------------------------------------------
+ // Keys
+ //
---------------------------------------------------------------------------------------------
+
+ @Test
+ void testAllKeysAndMatching() {
+ HopMetadata annotation =
RenamedType.class.getAnnotation(HopMetadata.class);
+
+ assertEquals(List.of("renamed-type", "RenamedType"),
HopMetadataUtil.getAllKeys(annotation));
+ assertTrue(HopMetadataUtil.matchesKey(annotation, "renamed-type"));
+ assertTrue(HopMetadataUtil.matchesKey(annotation, "RenamedType"));
+ assertFalse(HopMetadataUtil.matchesKey(annotation, "renamedtype"));
+ assertFalse(HopMetadataUtil.matchesKey(annotation, null));
+ assertFalse(HopMetadataUtil.matchesKey(null, "renamed-type"));
+
+ assertEquals(
+ List.of("twice-renamed", "TwiceRenamed", "OriginalName"),
+
HopMetadataUtil.getAllKeys(TwiceRenamedType.class.getAnnotation(HopMetadata.class)));
+ }
+
+ /** A type without legacy keys only has its own folder. */
+ @Test
+ void testTypeWithoutLegacyKeys() throws Exception {
+ JsonMetadataSerializer<?> plain =
+ (JsonMetadataSerializer<?>) provider.getSerializer(Occupation.class);
+ assertTrue(plain.getLegacyFolders().isEmpty());
+ }
+
+ /** A serialized metadata export from an older version of Hop uses the
legacy key. */
+ @Test
+ void testMetadataClassForLegacyKey() throws Exception {
+ BaseMetadataProvider base =
+ new BaseMetadataProvider(Variables.getADefaultVariableSpace(), "test")
{
+ @Override
+ public <T extends IHopMetadata> List<Class<T>> getMetadataClasses() {
+ return List.of(
+ (Class<T>) (Class<?>) RenamedType.class,
+ (Class<T>) (Class<?>) TwiceRenamedType.class);
+ }
+ };
+
+ assertEquals(RenamedType.class,
base.getMetadataClassForKey("RenamedType"));
+ assertEquals(TwiceRenamedType.class,
base.getMetadataClassForKey("TwiceRenamed"));
+ assertEquals(TwiceRenamedType.class,
base.getMetadataClassForKey("OriginalName"));
+ assertThrows(HopException.class, () ->
base.getMetadataClassForKey("NoSuchType"));
+ }
+}
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
b/core/src/test/java/org/apache/hop/metadata/serializer/json/renamed/CaseRenamedType.java
similarity index 50%
copy from
plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
copy to
core/src/test/java/org/apache/hop/metadata/serializer/json/renamed/CaseRenamedType.java
index 71586f010b..88f03a2baf 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
+++
b/core/src/test/java/org/apache/hop/metadata/serializer/json/renamed/CaseRenamedType.java
@@ -14,43 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hop.vfs.gs.metadatatype;
-import java.io.Serializable;
+package org.apache.hop.metadata.serializer.json.renamed;
+
import lombok.Getter;
import lombok.Setter;
-import org.apache.hop.core.vfs.IVfsBrowseLocation;
import org.apache.hop.metadata.api.HopMetadata;
import org.apache.hop.metadata.api.HopMetadataBase;
-import org.apache.hop.metadata.api.HopMetadataCategory;
import org.apache.hop.metadata.api.HopMetadataProperty;
-import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadata;
-@HopMetadata(
- key = "GoogleStorageConnectionDefinition",
- name = "i18n::GoogleStorageMetadataType.Name",
- description = "i18n::GoogleStorageMetadataType.Description",
- image = "ui/images/authentication.svg",
- category = HopMetadataCategory.FILE_STORAGE,
- documentationUrl = "/metadata-types/google-storage-authentication.html",
- hopMetadataPropertyType = HopMetadataPropertyType.VFS_GCP_CONNECTION,
- classLoaderGroup = "vfs-google")
+/** A metadata type whose key only changed case: from "Case-Renamed" to
"case-renamed". */
@Getter
@Setter
-public class GoogleStorageMetadataType extends HopMetadataBase
- implements Serializable, IHopMetadata, IVfsBrowseLocation {
+@HopMetadata(
+ name = "CaseRenamedType",
+ key = "case-renamed",
+ legacyKeys = {"Case-Renamed"})
+public class CaseRenamedType extends HopMetadataBase implements IHopMetadata {
- private static final Class<?> PKG = GoogleStorageMetadataType.class;
@HopMetadataProperty private String description;
- @HopMetadataProperty(password = true)
- private GoogleStorageCredentialsType storageCredentialsType;
-
- @HopMetadataProperty(password = true)
- private String storageAccountKey;
+ public CaseRenamedType() {}
- public GoogleStorageMetadataType() {
- // Do nothing
+ public CaseRenamedType(String name, String description) {
+ this.name = name;
+ this.description = description;
}
}
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
b/core/src/test/java/org/apache/hop/metadata/serializer/json/renamed/RenamedType.java
similarity index 50%
copy from
plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
copy to
core/src/test/java/org/apache/hop/metadata/serializer/json/renamed/RenamedType.java
index 71586f010b..b12d6f9098 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
+++
b/core/src/test/java/org/apache/hop/metadata/serializer/json/renamed/RenamedType.java
@@ -14,43 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hop.vfs.gs.metadatatype;
-import java.io.Serializable;
+package org.apache.hop.metadata.serializer.json.renamed;
+
import lombok.Getter;
import lombok.Setter;
-import org.apache.hop.core.vfs.IVfsBrowseLocation;
import org.apache.hop.metadata.api.HopMetadata;
import org.apache.hop.metadata.api.HopMetadataBase;
-import org.apache.hop.metadata.api.HopMetadataCategory;
import org.apache.hop.metadata.api.HopMetadataProperty;
-import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadata;
-@HopMetadata(
- key = "GoogleStorageConnectionDefinition",
- name = "i18n::GoogleStorageMetadataType.Name",
- description = "i18n::GoogleStorageMetadataType.Description",
- image = "ui/images/authentication.svg",
- category = HopMetadataCategory.FILE_STORAGE,
- documentationUrl = "/metadata-types/google-storage-authentication.html",
- hopMetadataPropertyType = HopMetadataPropertyType.VFS_GCP_CONNECTION,
- classLoaderGroup = "vfs-google")
+/** A metadata type which was renamed from key "RenamedType" to
"renamed-type". */
@Getter
@Setter
-public class GoogleStorageMetadataType extends HopMetadataBase
- implements Serializable, IHopMetadata, IVfsBrowseLocation {
+@HopMetadata(
+ name = "Renamed type",
+ key = "renamed-type",
+ legacyKeys = {"RenamedType"})
+public class RenamedType extends HopMetadataBase implements IHopMetadata {
- private static final Class<?> PKG = GoogleStorageMetadataType.class;
@HopMetadataProperty private String description;
- @HopMetadataProperty(password = true)
- private GoogleStorageCredentialsType storageCredentialsType;
-
- @HopMetadataProperty(password = true)
- private String storageAccountKey;
+ public RenamedType() {}
- public GoogleStorageMetadataType() {
- // Do nothing
+ public RenamedType(String name, String description) {
+ this.name = name;
+ this.description = description;
}
}
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
b/core/src/test/java/org/apache/hop/metadata/serializer/json/renamed/TwiceRenamedType.java
similarity index 50%
copy from
plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
copy to
core/src/test/java/org/apache/hop/metadata/serializer/json/renamed/TwiceRenamedType.java
index 71586f010b..02b0ec11f6 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
+++
b/core/src/test/java/org/apache/hop/metadata/serializer/json/renamed/TwiceRenamedType.java
@@ -14,43 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hop.vfs.gs.metadatatype;
-import java.io.Serializable;
+package org.apache.hop.metadata.serializer.json.renamed;
+
import lombok.Getter;
import lombok.Setter;
-import org.apache.hop.core.vfs.IVfsBrowseLocation;
import org.apache.hop.metadata.api.HopMetadata;
import org.apache.hop.metadata.api.HopMetadataBase;
-import org.apache.hop.metadata.api.HopMetadataCategory;
import org.apache.hop.metadata.api.HopMetadataProperty;
-import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadata;
-@HopMetadata(
- key = "GoogleStorageConnectionDefinition",
- name = "i18n::GoogleStorageMetadataType.Name",
- description = "i18n::GoogleStorageMetadataType.Description",
- image = "ui/images/authentication.svg",
- category = HopMetadataCategory.FILE_STORAGE,
- documentationUrl = "/metadata-types/google-storage-authentication.html",
- hopMetadataPropertyType = HopMetadataPropertyType.VFS_GCP_CONNECTION,
- classLoaderGroup = "vfs-google")
+/** A metadata type renamed twice: from "OriginalName" to "TwiceRenamed" to
"twice-renamed". */
@Getter
@Setter
-public class GoogleStorageMetadataType extends HopMetadataBase
- implements Serializable, IHopMetadata, IVfsBrowseLocation {
+@HopMetadata(
+ name = "TwiceRenamedType",
+ key = "twice-renamed",
+ legacyKeys = {"TwiceRenamed", "OriginalName"})
+public class TwiceRenamedType extends HopMetadataBase implements IHopMetadata {
- private static final Class<?> PKG = GoogleStorageMetadataType.class;
@HopMetadataProperty private String description;
- @HopMetadataProperty(password = true)
- private GoogleStorageCredentialsType storageCredentialsType;
-
- @HopMetadataProperty(password = true)
- private String storageAccountKey;
+ public TwiceRenamedType() {}
- public GoogleStorageMetadataType() {
- // Do nothing
+ public TwiceRenamedType(String name, String description) {
+ this.name = name;
+ this.description = description;
}
}
diff --git
a/core/src/test/java/org/apache/hop/metadata/serializer/multi/MultiMetadataLegacyKeysTest.java
b/core/src/test/java/org/apache/hop/metadata/serializer/multi/MultiMetadataLegacyKeysTest.java
new file mode 100644
index 0000000000..2007b0f8d5
--- /dev/null
+++
b/core/src/test/java/org/apache/hop/metadata/serializer/multi/MultiMetadataLegacyKeysTest.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.metadata.serializer.multi;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.encryption.HopTwoWayPasswordEncoder;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.serializer.MetadataGuiFlows;
+import org.apache.hop.metadata.serializer.json.JsonMetadataProvider;
+import org.apache.hop.metadata.serializer.json.renamed.RenamedType;
+import org.apache.hop.metadata.util.HopMetadataUtil;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Renamed metadata types with several metadata folders: a parent project and
a child project, or
+ * the folders listed in {@code HOP_METADATA_FOLDER}. Each folder can be
anywhere in its migration,
+ * independently of the others (issue #5597).
+ */
+class MultiMetadataLegacyKeysTest {
+
+ @TempDir Path parentFolder;
+ @TempDir Path childFolder;
+
+ private MultiMetadataProvider multi;
+ private IHopMetadataSerializer<RenamedType> serializer;
+
+ @BeforeEach
+ void setUp() throws Exception {
+ HopTwoWayPasswordEncoder encoder = new HopTwoWayPasswordEncoder();
+ JsonMetadataProvider parent =
+ new JsonMetadataProvider(
+ encoder, parentFolder.toString(),
Variables.getADefaultVariableSpace());
+ JsonMetadataProvider child =
+ new JsonMetadataProvider(
+ encoder, childFolder.toString(),
Variables.getADefaultVariableSpace());
+ // Like a project with a parent project: the last provider is the child,
which wins.
+ multi =
+ new MultiMetadataProvider(
+ encoder,
+ new ArrayList<IHopMetadataProvider>(List.of(parent, child)),
+ Variables.getADefaultVariableSpace());
+ serializer = multi.getSerializer(RenamedType.class);
+ }
+
+ private static void write(Path metadataFolder, String key, String name,
String description)
+ throws Exception {
+ Path typeFolder = metadataFolder.resolve(key);
+ Files.createDirectories(typeFolder);
+ Files.write(
+ typeFolder.resolve(name + ".json"),
+ ("{\"name\":\"" + name + "\",\"description\":\"" + description +
"\"}").getBytes(UTF_8));
+ }
+
+ private static boolean has(Path metadataFolder, String key, String name) {
+ return Files.exists(metadataFolder.resolve(key).resolve(name + ".json"));
+ }
+
+ private static final String LEGACY = "RenamedType";
+ private static final String CURRENT = "renamed-type";
+
+ private List<String> sortedNames() throws HopException {
+ List<String> names = new ArrayList<>(serializer.listObjectNames());
+ names.sort(String::compareTo);
+ return names;
+ }
+
+ @Test
+ void testChildOverridesParentWhateverTheFolders() throws Exception {
+ // Parent migrated, child didn't.
+ write(parentFolder, CURRENT, "a", "parent");
+ write(childFolder, LEGACY, "a", "child");
+ // Parent didn't migrate, child did.
+ write(parentFolder, LEGACY, "b", "parent");
+ write(childFolder, CURRENT, "b", "child");
+
+ assertEquals("child", serializer.load("a").getDescription());
+ assertEquals("child", serializer.load("b").getDescription());
+ assertEquals(List.of("a", "b"), sortedNames());
+ assertEquals(2, serializer.loadAll().size());
+ assertTrue(
+ serializer.loadAll().stream().allMatch(t ->
"child".equals(t.getDescription())),
+ "loadAll gives the child's objects, like load");
+ }
+
+ /**
+ * The file of an object is the one load() reads: the child project before
the parent, the current
+ * folder before the legacy one. That is what the metadata perspective opens
and what a search
+ * result points at.
+ */
+ @Test
+ void testFilenameIsTheFileLoadReads() throws Exception {
+ // Parent migrated, child didn't.
+ write(parentFolder, CURRENT, "a", "parent");
+ write(childFolder, LEGACY, "a", "child");
+ // Parent didn't migrate, child did.
+ write(parentFolder, LEGACY, "b", "parent");
+ write(childFolder, CURRENT, "b", "child");
+ // Only the parent has it, not migrated.
+ write(parentFolder, LEGACY, "c", "parent");
+ // The child has an outdated legacy copy next to the current one.
+ write(childFolder, LEGACY, "d", "outdated");
+ write(childFolder, CURRENT, "d", "child");
+
+ assertFilename(childFolder, LEGACY, "a");
+ assertFilename(childFolder, CURRENT, "b");
+ assertFilename(parentFolder, LEGACY, "c");
+ assertFilename(childFolder, CURRENT, "d");
+ assertEquals("child", serializer.load("d").getDescription());
+ assertEquals(null, HopMetadataUtil.findFilename(multi, RenamedType.class,
"nowhere"));
+ }
+
+ private void assertFilename(Path metadataFolder, String key, String name)
throws Exception {
+ String filename = HopMetadataUtil.findFilename(multi, RenamedType.class,
name);
+ assertEquals(
+ metadataFolder.resolve(key).resolve(name + ".json").toRealPath(),
+ Path.of(HopVfs.getFileObject(filename).getURL().toURI()).toRealPath(),
+ name);
+ }
+
+ @Test
+ void testMixedFoldersInEveryProvider() throws Exception {
+ write(parentFolder, LEGACY, "parent-legacy", "p");
+ write(parentFolder, CURRENT, "parent-current", "p");
+ write(childFolder, LEGACY, "child-legacy", "c");
+ write(childFolder, CURRENT, "child-current", "c");
+
+ assertEquals(
+ List.of("child-current", "child-legacy", "parent-current",
"parent-legacy"), sortedNames());
+ for (String name : sortedNames()) {
+ assertTrue(serializer.exists(name), name);
+ }
+ assertEquals(4, serializer.loadAll().size());
+ }
+
+ /** An object is saved back to the project it came from, and moves there. */
+ @Test
+ void testSaveStaysInTheOwningProvider() throws Exception {
+ write(parentFolder, LEGACY, "shared", "parent");
+
+ RenamedType shared = serializer.load("shared");
+ shared.setDescription("edited");
+ serializer.save(shared);
+
+ assertTrue(has(parentFolder, CURRENT, "shared"));
+ assertFalse(has(parentFolder, LEGACY, "shared"));
+ assertFalse(has(childFolder, CURRENT, "shared"));
+ assertFalse(has(childFolder, LEGACY, "shared"));
+ assertEquals("edited", serializer.load("shared").getDescription());
+ }
+
+ @Test
+ void testNewObjectGoesToTheChildsCurrentFolder() throws Exception {
+ write(parentFolder, LEGACY, "existing", "parent");
+
+ serializer.save(new RenamedType("new", "new"));
+
+ assertTrue(has(childFolder, CURRENT, "new"));
+ assertFalse(Files.exists(childFolder.resolve(LEGACY)));
+ assertFalse(has(parentFolder, CURRENT, "new"));
+ }
+
+ @Test
+ void testRenameInTreeInParentProject() throws Exception {
+ write(parentFolder, LEGACY, "old", "parent");
+ write(childFolder, CURRENT, "unrelated", "child");
+
+ assertTrue(MetadataGuiFlows.renameInTree(serializer, "old", "new"));
+
+ assertTrue(has(parentFolder, CURRENT, "new"));
+ assertFalse(has(parentFolder, LEGACY, "old"));
+ assertFalse(serializer.exists("old"));
+ assertEquals("parent", serializer.load("new").getDescription());
+ assertEquals(List.of("new", "unrelated"), sortedNames());
+ }
+
+ @Test
+ void testRenameInEditorInChildProject() throws Exception {
+ write(childFolder, LEGACY, "old", "child");
+
+ RenamedType metadata = serializer.load("old");
+ metadata.setName("new");
+ MetadataGuiFlows.renameInEditor(serializer, metadata, "old");
+
+ assertTrue(has(childFolder, CURRENT, "new"));
+ assertFalse(has(childFolder, LEGACY, "old"));
+ assertEquals(List.of("new"), sortedNames());
+ }
+
+ /** Names are unique over all the projects and all the folders. */
+ @Test
+ void testRenameOntoNameInOtherProjectsLegacyFolderIsRefused() throws
Exception {
+ write(parentFolder, LEGACY, "taken", "parent");
+ write(childFolder, CURRENT, "mine", "child");
+
+ assertFalse(MetadataGuiFlows.renameInTree(serializer, "mine", "taken"));
+ assertEquals("parent", serializer.load("taken").getDescription());
+ assertEquals("child", serializer.load("mine").getDescription());
+ }
+
+ /** The copy goes where the original lives, in the current folder. */
+ @Test
+ void testDuplicateInParentProject() throws Exception {
+ write(parentFolder, LEGACY, "conn", "parent");
+ write(childFolder, LEGACY, "conn 2", "child");
+
+ String copy = MetadataGuiFlows.duplicate(serializer, "conn");
+
+ assertEquals("conn 3", copy);
+ assertTrue(has(parentFolder, CURRENT, "conn 3"));
+ assertTrue(has(parentFolder, LEGACY, "conn"), "the original isn't moved");
+ assertTrue(has(childFolder, LEGACY, "conn 2"));
+ assertEquals(List.of("conn", "conn 2", "conn 3"), sortedNames());
+ }
+
+ /**
+ * Deleting an object the child overrides removes the child's copies only,
in all its folders. The
+ * parent's object then shows through again, as it did before legacy keys
existed.
+ */
+ @Test
+ void testDeleteOverriddenObject() throws Exception {
+ write(parentFolder, LEGACY, "x", "parent");
+ write(childFolder, LEGACY, "x", "child outdated");
+ write(childFolder, CURRENT, "x", "child");
+
+ RenamedType deleted = serializer.delete("x");
+
+ assertEquals("child", deleted.getDescription());
+ assertFalse(has(childFolder, CURRENT, "x"));
+ assertFalse(has(childFolder, LEGACY, "x"));
+ assertTrue(serializer.exists("x"));
+ assertEquals("parent", serializer.load("x").getDescription());
+
+ serializer.delete("x");
+ assertFalse(serializer.exists("x"));
+ assertThrows(HopException.class, () -> serializer.delete("x"));
+ }
+
+ @Test
+ void testReadVirtualPathFollowsTheSameOrder() throws Exception {
+ Path parentCurrent = parentFolder.resolve(CURRENT);
+ Files.createDirectories(parentCurrent);
+ Files.write(
+ parentCurrent.resolve("v.json"),
+ "{\"name\":\"v\",\"virtualPath\":\"/parent\"}".getBytes(UTF_8));
+ Path childLegacy = childFolder.resolve(LEGACY);
+ Files.createDirectories(childLegacy);
+ Files.write(
+ childLegacy.resolve("v.json"),
+ "{\"name\":\"v\",\"virtualPath\":\"/child\"}".getBytes(UTF_8));
+
+ assertEquals("/child", serializer.readVirtualPath("v"));
+ }
+}
diff --git a/docs/hop-dev-manual/modules/ROOT/pages/metadata-plugins.adoc
b/docs/hop-dev-manual/modules/ROOT/pages/metadata-plugins.adoc
index e8b14f8bac..18fd64bfdb 100644
--- a/docs/hop-dev-manual/modules/ROOT/pages/metadata-plugins.adoc
+++ b/docs/hop-dev-manual/modules/ROOT/pages/metadata-plugins.adoc
@@ -28,7 +28,9 @@ The class which carries this annotation will contain the
metadata.
Here are the attributes of the @HopMetadata annotation:
-* key : this uniquely identifies the plugin and will be the name of the folder
in which the metadata resides when serialising to JSON (see below)
+* key : this uniquely identifies the plugin and will be the name of the folder
in which the metadata resides when serialising to JSON (see below).
+Use lower-case words separated by dashes, see <<metadata-key-naming>>.
+* legacyKeys : the keys this metadata type was known under before its key was
renamed (optional), see <<metadata-key-renaming>>
* name : a human-readable name
* description : an extended description
* image : the path to an image which helps identify the metadata in the Hop GUI
@@ -37,6 +39,46 @@ The class with this annotation will be found either because
it lives in the ```p
Example:
https://github.com/apache/hop/blob/f8f2ad2d0bc0cfd143ae90cc32e73b6c9e1cda78/engine/src/main/java/org/apache/hop/partition/PartitionSchema.java#L47[PartitionSchema.java]
+[[metadata-key-naming]]
+=== Choosing a key
+
+The key is more than an internal ID.
+It is the name of the folder in a project which holds the metadata objects
(`metadata/<key>/<name>.json`), it is written into the metadata exported for
remote execution, and it is the ID used to disable the type in
`disabledGuiElements`.
+Users see it and check it into version control, so it is effectively public.
+
+Use lower-case words separated by dashes (kebab-case) which describe the type,
like the existing `pipeline-run-configuration`, `sftp-connection` or
`mail-server-connection`.
+Don't use the Java class name or PascalCase (`MailServerConnection`): the
folders in a project then no longer follow one convention.
+
+[[metadata-key-renaming]]
+=== Renaming a key
+
+Changing the key of an existing metadata type would make all the objects users
already have invisible, because they live in a folder named after the old key.
+If you have to rename a key, move the old one to `legacyKeys`:
+
+[source,java]
+----
+@HopMetadata(
+ key = "mail-server-connection",
+ legacyKeys = {"MailServerConnection"},
+ name = "i18n::MailServerConnection.name",
+ ...)
+public class MailServerConnection extends HopMetadataBase implements
IHopMetadata {
+----
+
+With a legacy key in place:
+
+* objects in the folder named after the legacy key are still listed and loaded.
+If an object exists in both folders, the one in the folder named after the
current key wins.
+* saving an object always writes it to the folder named after the current key
and removes the copy in the legacy folder.
+Projects move to the new folder one object at a time, as users edit them.
+* deleting an object removes it from all the folders.
+* a metadata export which uses the legacy key, for example one sent by an
older Hop client to a Hop server, is still understood.
+
+Code which looks for metadata files or compares type keys should use
`HopMetadataUtil.getAllKeys()`, `HopMetadataUtil.matchesKey()` or
`JsonMetadataSerializer.findFilename()` rather than the `key` alone, so it also
finds objects which haven't moved yet.
+
+Note that older Hop versions only know the old key: once an object has been
saved by a newer version, older versions no longer see it.
+Mention the rename in the release notes.
+
== Metadata Properties
All properties you want to have as part of the shared Hop Metadata should get
the ```*@HopMetadataProperty*``` annotation.
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rest-connection.adoc
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rest-connection.adoc
index 69c9caf47c..f5a6570cb0 100644
---
a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rest-connection.adoc
+++
b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/rest-connection.adoc
@@ -189,4 +189,4 @@ TIP: For `LINK_HEADER` and `BODY_NEXT_URL`, the path on the
REST client transfor
== Samples
-link:https://github.com/apache/hop/blob/main/plugins/transforms/rest/src/main/samples/metadata/restconnection/github-releases.json[github-releases.json]
— public GitHub API, `LINK_HEADER` pagination (used by
link:https://github.com/apache/hop/blob/main/plugins/transforms/rest/src/main/samples/transforms/rest-client-github-releases-loop.hpl[rest-client-github-releases-loop.hpl]
in the REST transform samples project).
\ No newline at end of file
+link:https://github.com/apache/hop/blob/main/plugins/transforms/rest/src/main/samples/metadata/rest-connection/github-releases.json[github-releases.json]
— public GitHub API, `LINK_HEADER` pagination (used by
link:https://github.com/apache/hop/blob/main/plugins/transforms/rest/src/main/samples/transforms/rest-client-github-releases-loop.hpl[rest-client-github-releases-loop.hpl]
in the REST transform samples project).
\ No newline at end of file
diff --git
a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rest.adoc
b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rest.adoc
index 36429c53b0..a65f09e14c 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rest.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/rest.adoc
@@ -237,4 +237,4 @@ The block is built from the request that is actually sent,
so it also shows what
== Samples
-link:https://github.com/apache/hop/blob/main/plugins/transforms/rest/src/main/samples/transforms/rest-client-github-releases-loop.hpl[rest-client-github-releases-loop.hpl]
— lists `apache/hop` GitHub releases via `LINK_HEADER` pagination
(`per_page=5`), splits `+$[*]+`, and parses fields with a JSON Input transform.
Connection metadata:
link:https://github.com/apache/hop/blob/main/plugins/transforms/rest/src/main/samples/metadata/restconnection/github-releases.json[github-releases.json]
[...]
+link:https://github.com/apache/hop/blob/main/plugins/transforms/rest/src/main/samples/transforms/rest-client-github-releases-loop.hpl[rest-client-github-releases-loop.hpl]
— lists `apache/hop` GitHub releases via `LINK_HEADER` pagination
(`per_page=5`), splits `+$[*]+`, and parses fields with a JSON Input transform.
Connection metadata:
link:https://github.com/apache/hop/blob/main/plugins/transforms/rest/src/main/samples/metadata/rest-connection/github-releases.json[github-releases.json]
[...]
diff --git
a/engine/src/main/java/org/apache/hop/metadata/refactor/MetadataReferenceFinder.java
b/engine/src/main/java/org/apache/hop/metadata/refactor/MetadataReferenceFinder.java
index 4c86b3629b..d7ac20d8d3 100644
---
a/engine/src/main/java/org/apache/hop/metadata/refactor/MetadataReferenceFinder.java
+++
b/engine/src/main/java/org/apache/hop/metadata/refactor/MetadataReferenceFinder.java
@@ -586,7 +586,7 @@ public class MetadataReferenceFinder {
* Finds all pipeline and workflow files under the given root paths that
contain a reference to
* the given metadata element name (for the given metadata type key).
*
- * @param metadataKey metadata type key (e.g. "rdbms", "restconnection")
+ * @param metadataKey metadata type key (e.g. "rdbms", "rest-connection")
* @param elementName the metadata element name to search for (e.g.
connection name)
* @param searchRootPaths root paths to search recursively for .hpl and .hwf
files
* @return list of files with at least one reference, and the count per file
diff --git
a/engine/src/test/java/org/apache/hop/core/metadata/SerializableMetadataLegacyKeysTest.java
b/engine/src/test/java/org/apache/hop/core/metadata/SerializableMetadataLegacyKeysTest.java
new file mode 100644
index 0000000000..e45f80dd73
--- /dev/null
+++
b/engine/src/test/java/org/apache/hop/core/metadata/SerializableMetadataLegacyKeysTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.metadata;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.encryption.HopTwoWayPasswordEncoder;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.HopLogStore;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.HopMetadataBase;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.serializer.json.JsonMetadataProvider;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Metadata is serialized to JSON, keyed by metadata type, to ship it to a Hop
server or another
+ * engine. Client and server can be different versions of Hop, so both keys of
a renamed type have
+ * to be understood (issue #5597).
+ */
+class SerializableMetadataLegacyKeysTest {
+
+ /** A metadata type which was renamed from key "RenamedType" to
"renamed-type". */
+ @Getter
+ @Setter
+ @HopMetadata(
+ name = "Renamed type",
+ key = "renamed-type",
+ legacyKeys = {"RenamedType"})
+ public static class RenamedType extends HopMetadataBase implements
IHopMetadata {
+ @HopMetadataProperty private String description;
+
+ public RenamedType() {}
+ }
+
+ /** An unknown type in an export is logged and skipped. */
+ @BeforeAll
+ static void initLogging() {
+ HopLogStore.init();
+ }
+
+ /** The metadata types known in these tests, instead of the plugin registry.
*/
+ private static <T extends IHopMetadata> List<Class<T>> testTypes() {
+ return List.of((Class<T>) (Class<?>) RenamedType.class);
+ }
+
+ private static SerializableMetadataProvider fromJson(String json) throws
HopException {
+ return new SerializableMetadataProvider(json) {
+ @Override
+ public <T extends IHopMetadata> List<Class<T>> getMetadataClasses() {
+ return testTypes();
+ }
+ };
+ }
+
+ private static List<String> sortedNames(IHopMetadataSerializer<?> serializer)
+ throws HopException {
+ List<String> names = new ArrayList<>(serializer.listObjectNames());
+ names.sort(String::compareTo);
+ return names;
+ }
+
+ /** What an older Hop client sends: the objects under the legacy key. */
+ @Test
+ void testExportWithLegacyKeyIsLoaded() throws Exception {
+ String json = "{\"RenamedType\":[{\"name\":\"old\",\"description\":\"from
an older client\"}]}";
+
+ SerializableMetadataProvider provider = fromJson(json);
+
+ IHopMetadataSerializer<RenamedType> serializer =
provider.getSerializer(RenamedType.class);
+ assertEquals(List.of("old"), serializer.listObjectNames());
+ assertEquals("from an older client",
serializer.load("old").getDescription());
+ }
+
+ @Test
+ void testExportWithCurrentKeyIsLoaded() throws Exception {
+ String json =
"{\"renamed-type\":[{\"name\":\"new\",\"description\":\"current\"}]}";
+
+ SerializableMetadataProvider provider = fromJson(json);
+
+ assertEquals("current",
provider.getSerializer(RenamedType.class).load("new").getDescription());
+ }
+
+ /** Should both keys ever end up in one export, the objects of both are
loaded. */
+ @Test
+ void testExportWithBothKeysIsMerged() throws Exception {
+ String json =
+ "{\"RenamedType\":[{\"name\":\"a\",\"description\":\"legacy\"}],"
+ +
"\"renamed-type\":[{\"name\":\"b\",\"description\":\"current\"}]}";
+
+ SerializableMetadataProvider provider = fromJson(json);
+
+ assertEquals(List.of("a", "b"),
sortedNames(provider.getSerializer(RenamedType.class)));
+ }
+
+ @Test
+ void testExportWithUnknownKeyIsStillSkipped() throws Exception {
+ String json =
+ "{\"NoSuchType\":[{\"name\":\"x\"}],"
+ + "\"RenamedType\":[{\"name\":\"a\",\"description\":\"legacy\"}]}";
+
+ SerializableMetadataProvider provider = fromJson(json);
+
+ assertEquals(List.of("a"),
provider.getSerializer(RenamedType.class).listObjectNames());
+ }
+
+ /**
+ * Exporting a project which is halfway its migration ships every object
once, the current copy of
+ * an object in both folders, under the current key.
+ */
+ @Test
+ void testExportOfMixedProject(@TempDir Path folder) throws Exception {
+ write(folder.resolve("RenamedType"), "legacy-only", "legacy");
+ write(folder.resolve("RenamedType"), "both", "outdated legacy copy");
+ write(folder.resolve("renamed-type"), "both", "current copy");
+ write(folder.resolve("renamed-type"), "current-only", "current");
+
+ JsonMetadataProvider project =
+ new JsonMetadataProvider(
+ new HopTwoWayPasswordEncoder(),
+ folder.toString(),
+ Variables.getADefaultVariableSpace()) {
+ @Override
+ public <T extends IHopMetadata> List<Class<T>> getMetadataClasses() {
+ return testTypes();
+ }
+ };
+
+ SerializableMetadataProvider exported =
+ new SerializableMetadataProvider(project) {
+ @Override
+ public <T extends IHopMetadata> List<Class<T>> getMetadataClasses() {
+ return testTypes();
+ }
+ };
+ IHopMetadataSerializer<RenamedType> exportedSerializer =
+ exported.getSerializer(RenamedType.class);
+ assertEquals(List.of("both", "current-only", "legacy-only"),
sortedNames(exportedSerializer));
+ assertEquals("current copy",
exportedSerializer.load("both").getDescription());
+
+ String json = exported.toJson();
+ assertTrue(json.contains("\"renamed-type\""), json);
+ assertFalse(json.contains("\"RenamedType\""), json);
+ assertFalse(json.contains("outdated legacy copy"), json);
+
+ // What the server makes of it.
+ IHopMetadataSerializer<RenamedType> received =
fromJson(json).getSerializer(RenamedType.class);
+ assertEquals(List.of("both", "current-only", "legacy-only"),
sortedNames(received));
+ assertEquals("current copy", received.load("both").getDescription());
+ assertEquals("legacy", received.load("legacy-only").getDescription());
+
+ // Exporting reads, it never moves files around in the project.
+
assertTrue(Files.exists(folder.resolve("RenamedType").resolve("legacy-only.json")));
+
assertTrue(Files.exists(folder.resolve("RenamedType").resolve("both.json")));
+ }
+
+ private static void write(Path typeFolder, String name, String description)
throws Exception {
+ Files.createDirectories(typeFolder);
+ Files.write(
+ typeFolder.resolve(name + ".json"),
+ ("{\"name\":\"" + name + "\",\"description\":\"" + description +
"\"}").getBytes(UTF_8));
+ }
+}
diff --git
a/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/SparkCatalog.java
b/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/SparkCatalog.java
index 13c22a88b1..7ca4f067a7 100644
---
a/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/SparkCatalog.java
+++
b/plugins/engines/spark/src/main/java/org/apache/hop/spark/metadata/SparkCatalog.java
@@ -45,7 +45,8 @@ import org.eclipse.swt.widgets.Shell;
@Setter
@GuiPlugin
@HopMetadata(
- key = "SparkCatalog",
+ key = "spark-catalog",
+ legacyKeys = {"SparkCatalog"},
name = "i18n::SparkCatalog.Name",
description = "i18n::SparkCatalog.Description",
image = "spark-catalog.svg",
diff --git
a/plugins/misc/git/src/main/java/org/apache/hop/git/provider/GitConnection.java
b/plugins/misc/git/src/main/java/org/apache/hop/git/provider/GitConnection.java
index 04e828a3bb..1d2f7d86d9 100644
---
a/plugins/misc/git/src/main/java/org/apache/hop/git/provider/GitConnection.java
+++
b/plugins/misc/git/src/main/java/org/apache/hop/git/provider/GitConnection.java
@@ -33,7 +33,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
@Getter
@Setter
@HopMetadata(
- key = "gitconnection",
+ key = "git-connection",
+ legacyKeys = {"gitconnection"},
name = "i18n::GitConnection.name",
description = "i18n::GitConnection.description",
image = "git.svg",
diff --git
a/plugins/misc/git/src/main/samples/metadata/gitconnection/github.json
b/plugins/misc/git/src/main/samples/metadata/git-connection/github.json
similarity index 100%
rename from plugins/misc/git/src/main/samples/metadata/gitconnection/github.json
rename to plugins/misc/git/src/main/samples/metadata/git-connection/github.json
diff --git
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintMetadataSelection.java
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintMetadataSelection.java
index bbd6735503..c8acd3f853 100644
---
a/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintMetadataSelection.java
+++
b/plugins/misc/lint/src/main/java/org/apache/hop/lint/LintMetadataSelection.java
@@ -76,7 +76,14 @@ final class LintMetadataSelection {
if (annotation == null || Utils.isEmpty(annotation.key())) {
return null;
}
- return resolveMetadataPath(annotation.key(), metadata.getName());
+ // An object which wasn't saved since its type was renamed still lives in
a legacy key folder.
+ for (String key : HopMetadataUtil.getAllKeys(annotation)) {
+ String path = resolveMetadataPath(key, metadata.getName());
+ if (path != null) {
+ return path;
+ }
+ }
+ return null;
}
private static String resolveMetadataPath(String key, String name) {
diff --git
a/plugins/misc/lint/src/test/java/org/apache/hop/lint/HopMetadataFileLoaderLegacyKeysTest.java
b/plugins/misc/lint/src/test/java/org/apache/hop/lint/HopMetadataFileLoaderLegacyKeysTest.java
new file mode 100644
index 0000000000..da57f15355
--- /dev/null
+++
b/plugins/misc/lint/src/test/java/org/apache/hop/lint/HopMetadataFileLoaderLegacyKeysTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.lint;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.hop.core.encryption.HopTwoWayPasswordEncoder;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.HopMetadataBase;
+import org.apache.hop.metadata.api.HopMetadataProperty;
+import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.serializer.json.JsonMetadataProvider;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * A metadata file in a folder named after a legacy key of a renamed type
(issue #5597) is still
+ * linted: the folder name resolves to the renamed type.
+ */
+class HopMetadataFileLoaderLegacyKeysTest {
+
+ /** A metadata type which was renamed from key "RenamedType" to
"renamed-type". */
+ @Getter
+ @Setter
+ @HopMetadata(
+ name = "Renamed type",
+ key = "renamed-type",
+ legacyKeys = {"RenamedType"})
+ public static class RenamedType extends HopMetadataBase implements
IHopMetadata {
+ @HopMetadataProperty private String description;
+
+ public RenamedType() {}
+ }
+
+ private static IHopMetadataProvider provider(Path metadataFolder) {
+ return new JsonMetadataProvider(
+ new HopTwoWayPasswordEncoder(),
+ metadataFolder.toString(),
+ Variables.getADefaultVariableSpace()) {
+ @Override
+ public <T extends IHopMetadata> List<Class<T>> getMetadataClasses() {
+ return List.of((Class<T>) (Class<?>) RenamedType.class);
+ }
+ };
+ }
+
+ private static File write(Path metadataFolder, String key, String name,
String description)
+ throws Exception {
+ Path typeFolder = metadataFolder.resolve(key);
+ Files.createDirectories(typeFolder);
+ Path file = typeFolder.resolve(name + ".json");
+ Files.writeString(
+ file,
+ "{\"name\":\"" + name + "\",\"description\":\"" + description + "\"}",
+ StandardCharsets.UTF_8);
+ return file.toFile();
+ }
+
+ @Test
+ void fileInLegacyFolderIsLoaded(@TempDir Path dir) throws Exception {
+ Path metadataFolder = dir.resolve("metadata");
+ File file = write(metadataFolder, "RenamedType", "old", "legacy");
+
+ HopMetadataFileLoader.MetadataLoad load =
+ HopMetadataFileLoader.read(file, provider(metadataFolder));
+
+ assertFalse(load.isFailure(), load.error());
+ RenamedType loaded = assertInstanceOf(RenamedType.class, load.object());
+ assertEquals("old", loaded.getName());
+ assertEquals("legacy", loaded.getDescription());
+ }
+
+ @Test
+ void fileInCurrentFolderIsLoaded(@TempDir Path dir) throws Exception {
+ Path metadataFolder = dir.resolve("metadata");
+ write(metadataFolder, "RenamedType", "legacy-only", "legacy");
+ File file = write(metadataFolder, "renamed-type", "new", "current");
+
+ HopMetadataFileLoader.MetadataLoad load =
+ HopMetadataFileLoader.read(file, provider(metadataFolder));
+
+ RenamedType loaded = assertInstanceOf(RenamedType.class, load.object());
+ assertEquals("current", loaded.getDescription());
+ }
+}
diff --git
a/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnection.java
b/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnection.java
index a698a245e9..8af71a3aa6 100644
---
a/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnection.java
+++
b/plugins/misc/mail/src/main/java/org/apache/hop/mail/metadata/MailServerConnection.java
@@ -76,7 +76,8 @@ import org.eclipse.angus.mail.pop3.POP3SSLStore;
@Getter
@Setter
@HopMetadata(
- key = "MailServerConnection",
+ key = "mail-server-connection",
+ legacyKeys = {"MailServerConnection"},
name = "i18n::MailServerConnection.name",
description = "i18n::MailServerConnection.description",
image = "mail.svg",
diff --git
a/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnection.java
b/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnection.java
index 4e3f9e71b4..fccbdcdf41 100644
---
a/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnection.java
+++
b/plugins/misc/rest/src/main/java/org/apache/hop/metadata/rest/RestConnection.java
@@ -60,7 +60,8 @@ import org.apache.hop.metadata.rest.client.RestOAuth2Grant;
@Getter
@Setter
@HopMetadata(
- key = "restconnection",
+ key = "rest-connection",
+ legacyKeys = {"restconnection"},
name = "i18n::RestConnection.name",
description = "i18n::RestConnection.description",
image = "rest.svg",
diff --git
a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3Meta.java
b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3Meta.java
index 6eb9724ec4..1ad1509fd5 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3Meta.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3Meta.java
@@ -32,7 +32,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
@Setter
@GuiPlugin
@HopMetadata(
- key = "S3ConnectionDefinition",
+ key = "s3-connection",
+ legacyKeys = {"S3ConnectionDefinition"},
name = "i18n::S3Meta.name",
description = "i18n::S3Meta.description",
image = "s3.svg",
diff --git
a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataType.java
b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataType.java
index fb1dfe5be2..7c93f2c3dd 100644
---
a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataType.java
+++
b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataType.java
@@ -28,7 +28,8 @@ import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadata;
@HopMetadata(
- key = "AzureConnectionDefinition",
+ key = "azure-connection",
+ legacyKeys = {"AzureConnectionDefinition"},
name = "i18n::AzureMetadataType.Name",
description = "i18n::AzureMetadataType.Description",
image = "ui/images/authentication.svg",
diff --git
a/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java
b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java
index 0836f89991..879cad9040 100644
---
a/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java
+++
b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java
@@ -39,7 +39,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
@Setter
@GuiPlugin
@HopMetadata(
- key = "DatabricksConnection",
+ key = "databricks-connection",
+ legacyKeys = {"DatabricksConnection"},
name = "i18n::DatabricksConnection.Name",
description = "i18n::DatabricksConnection.Description",
image = "databricks-connection.svg",
diff --git
a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnection.java
b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnection.java
index 5c2d6215ee..d87e584bc2 100644
---
a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnection.java
+++
b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnection.java
@@ -41,7 +41,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
@Setter
@GuiPlugin
@HopMetadata(
- key = "DatabricksVfsConnectionDefinition",
+ key = "databricks-vfs-connection",
+ legacyKeys = {"DatabricksVfsConnectionDefinition"},
name = "i18n::DatabricksVfsConnection.Name",
description = "i18n::DatabricksVfsConnection.Description",
image = "databricks-connection.svg",
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
index 71586f010b..00fa23c5e9 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
+++
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
@@ -28,7 +28,8 @@ import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadata;
@HopMetadata(
- key = "GoogleStorageConnectionDefinition",
+ key = "google-storage-connection",
+ legacyKeys = {"GoogleStorageConnectionDefinition"},
name = "i18n::GoogleStorageMetadataType.Name",
description = "i18n::GoogleStorageMetadataType.Description",
image = "ui/images/authentication.svg",
diff --git
a/plugins/tech/hadoop/src/main/java/org/apache/hop/vfs/hdfs/metadata/HdfsMeta.java
b/plugins/tech/hadoop/src/main/java/org/apache/hop/vfs/hdfs/metadata/HdfsMeta.java
index a156399dd1..c6bafce6a0 100644
---
a/plugins/tech/hadoop/src/main/java/org/apache/hop/vfs/hdfs/metadata/HdfsMeta.java
+++
b/plugins/tech/hadoop/src/main/java/org/apache/hop/vfs/hdfs/metadata/HdfsMeta.java
@@ -43,7 +43,8 @@ import org.eclipse.swt.widgets.MessageBox;
@Setter
@GuiPlugin(classLoaderGroup = "vfs-hdfs")
@HopMetadata(
- key = "HdfsConnectionDefinition",
+ key = "hdfs-connection",
+ legacyKeys = {"HdfsConnectionDefinition"},
name = "i18n::HdfsMeta.Name",
description = "i18n::HdfsMeta.Description",
image = "hdfs.svg",
diff --git
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMeta.java
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMeta.java
index 5a0a9d7c15..45d7d15a36 100644
---
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMeta.java
+++
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMeta.java
@@ -34,7 +34,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
@Setter
@GuiPlugin
@HopMetadata(
- key = "MinioConnectionDefinition",
+ key = "minio-connection",
+ legacyKeys = {"MinioConnectionDefinition"},
name = "i18n::MinioMeta.Name",
description = "i18n::MinioMeta.Description",
image = "minio.svg",
diff --git
a/plugins/tech/salesforce/src/main/java/org/apache/hop/metadata/salesforce/SalesforceConnection.java
b/plugins/tech/salesforce/src/main/java/org/apache/hop/metadata/salesforce/SalesforceConnection.java
index c41252bc2a..09828ff750 100644
---
a/plugins/tech/salesforce/src/main/java/org/apache/hop/metadata/salesforce/SalesforceConnection.java
+++
b/plugins/tech/salesforce/src/main/java/org/apache/hop/metadata/salesforce/SalesforceConnection.java
@@ -29,7 +29,8 @@ import org.apache.hop.metadata.api.HopMetadataPropertyType;
import org.apache.hop.metadata.api.IHopMetadata;
@HopMetadata(
- key = "salesforceconnection",
+ key = "salesforce-connection",
+ legacyKeys = {"salesforceconnection"},
name = "i18n::SalesforceConnection.name",
description = "i18n::SalesforceConnection.description",
image = "salesforce.svg",
diff --git
a/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnection.java
b/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnection.java
index dca27634f6..8790c1463d 100644
---
a/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnection.java
+++
b/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnection.java
@@ -36,7 +36,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
@Getter
@Setter
@HopMetadata(
- key = "WebDavConnectionDefinition",
+ key = "webdav-connection",
+ legacyKeys = {"WebDavConnectionDefinition"},
name = "i18n::WebDavConnection.Name",
description = "i18n::WebDavConnection.Description",
image = "ui/images/authentication.svg",
diff --git
a/plugins/transforms/metadata/src/main/java/org/apache/hop/pipeline/transforms/metainput/MetadataInput.java
b/plugins/transforms/metadata/src/main/java/org/apache/hop/pipeline/transforms/metainput/MetadataInput.java
index de68fbe3b6..37824ab278 100644
---
a/plugins/transforms/metadata/src/main/java/org/apache/hop/pipeline/transforms/metainput/MetadataInput.java
+++
b/plugins/transforms/metadata/src/main/java/org/apache/hop/pipeline/transforms/metainput/MetadataInput.java
@@ -70,7 +70,8 @@ public class MetadataInput extends
BaseTransform<MetadataInputMeta, MetadataInpu
//
boolean include = typeKeyFilters.isEmpty();
for (String typeKeyFilter : typeKeyFilters) {
- if (typeKeyFilter.equals(annotation.key())) {
+ // A filter can still use the key the type had before it was renamed.
+ if (HopMetadataUtil.matchesKey(annotation, typeKeyFilter)) {
include = true;
}
}
diff --git
a/plugins/transforms/rest/src/main/samples/metadata/restconnection/github-releases.json
b/plugins/transforms/rest/src/main/samples/metadata/rest-connection/github-releases.json
similarity index 100%
rename from
plugins/transforms/rest/src/main/samples/metadata/restconnection/github-releases.json
rename to
plugins/transforms/rest/src/main/samples/metadata/rest-connection/github-releases.json
diff --git
a/plugins/transforms/rest/src/main/samples/metadata/restconnection/slack-conversations.json
b/plugins/transforms/rest/src/main/samples/metadata/rest-connection/slack-conversations.json
similarity index 100%
rename from
plugins/transforms/rest/src/main/samples/metadata/restconnection/slack-conversations.json
rename to
plugins/transforms/rest/src/main/samples/metadata/rest-connection/slack-conversations.json
diff --git
a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/metadata/MetadataPerspective.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/metadata/MetadataPerspective.java
index d865eb84f2..4fb6192724 100644
---
a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/metadata/MetadataPerspective.java
+++
b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/metadata/MetadataPerspective.java
@@ -63,7 +63,6 @@ import
org.apache.hop.metadata.refactor.MetadataObjectReference;
import org.apache.hop.metadata.refactor.MetadataReferenceFinder;
import org.apache.hop.metadata.refactor.MetadataReferenceResult;
import org.apache.hop.metadata.serializer.json.JsonMetadataProvider;
-import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
import org.apache.hop.metadata.util.HopMetadataUtil;
import org.apache.hop.ui.core.ConstUi;
import org.apache.hop.ui.core.FormDataBuilder;
@@ -2461,7 +2460,10 @@ public class MetadataPerspective implements
IHopPerspective, TabClosable, IMetad
annotation.image(),
metadataClass);
- knownKeys.add(annotation.key());
+ // A folder named after a legacy key holds objects of this type which
weren't saved since
+ // the type was renamed: they are not unknown.
+ //
+ knownKeys.addAll(HopMetadataUtil.getAllKeys(annotation));
IHopMetadataSerializer<IHopMetadata> serializer =
metadataProvider.getSerializer(metadataClass);
@@ -2531,7 +2533,9 @@ public class MetadataPerspective implements
IHopPerspective, TabClosable, IMetad
IHopMetadataProvider metadataProvider,
Set<String> knownKeys,
Map<String, UnknownTypeModel> unknownByKey) {
- for (JsonMetadataProvider jsonProvider :
getJsonProviders(metadataProvider)) {
+ // Child provider first, the way load() looks for an element.
+ //
+ for (JsonMetadataProvider jsonProvider :
getJsonProviders(metadataProvider).reversed()) {
try {
FileObject baseFolder =
HopVfs.getFileObject(jsonProvider.getBaseFolder());
if (!baseFolder.exists()) {
@@ -2552,8 +2556,8 @@ public class MetadataPerspective implements
IHopPerspective, TabClosable, IMetad
unknownByKey.computeIfAbsent(key, k -> new UnknownTypeModel(k,
k));
for (FileObject jsonFile : jsonFiles) {
String name =
jsonFile.getName().getBaseName().replaceAll("\\.json$", "");
- // The same element can live in a parent project as well: like
anywhere else the first
- // provider which has it wins, so we don't list it twice.
+ // The same element can live in a parent project as well: like
anywhere else the child
+ // project's copy wins, so we don't list it twice.
if (unknownType.items.stream().noneMatch(item ->
item.name.equals(name))) {
unknownType.items.add(
new UnknownItemModel(name, HopVfs.getFilename(jsonFile),
reason));
@@ -2567,25 +2571,42 @@ public class MetadataPerspective implements
IHopPerspective, TabClosable, IMetad
}
}
- /** The JSON (file based) providers behind the given provider, which can be
a multi-provider. */
+ /**
+ * The JSON (file based) providers behind the given provider, which can be a
multi-provider: the
+ * parent project first, the child project last.
+ */
private static List<JsonMetadataProvider>
getJsonProviders(IHopMetadataProvider provider) {
List<JsonMetadataProvider> jsonProviders = new ArrayList<>();
- if (provider instanceof MultiMetadataProvider multiProvider) {
- for (IHopMetadataProvider childProvider : multiProvider.getProviders()) {
- jsonProviders.addAll(getJsonProviders(childProvider));
+ for (IHopMetadataProvider leaf : HopMetadataUtil.getProviders(provider)) {
+ if (leaf instanceof JsonMetadataProvider jsonProvider) {
+ jsonProviders.add(jsonProvider);
}
- } else if (provider instanceof JsonMetadataProvider jsonProvider) {
- jsonProviders.add(jsonProvider);
}
return jsonProviders;
}
/**
- * The file behind a metadata element: {@code <base folder>/<type
key>/<name>.json} in the first
- * provider which has it. Returns null if no file was found (or the metadata
isn't file based).
+ * The file behind a metadata element, the one {@code load()} reads: in the
last (child) provider
+ * which has it, in the folder of the current key before a legacy one.
Returns null if no file was
+ * found (or the metadata isn't file based).
*/
private String findMetadataFilename(String typeKey, String name) {
- for (JsonMetadataProvider jsonProvider :
getJsonProviders(hopGui.getMetadataProvider())) {
+ IHopMetadataProvider metadataProvider = hopGui.getMetadataProvider();
+ Class<IHopMetadata> metadataClass = null;
+ try {
+ metadataClass = metadataProvider.getMetadataClassForKey(typeKey);
+ } catch (Exception e) {
+ // An unknown type: only look in the folder named after the key, below.
+ }
+ if (metadataClass != null) {
+ try {
+ return HopMetadataUtil.findFilename(metadataProvider, metadataClass,
name);
+ } catch (Exception e) {
+ LogChannel.UI.logError("Error looking for the file of metadata element
" + name, e);
+ return null;
+ }
+ }
+ for (JsonMetadataProvider jsonProvider :
getJsonProviders(metadataProvider).reversed()) {
String filename = jsonProvider.getBaseFolder() + "/" + typeKey + "/" +
name + ".json";
try {
if (HopVfs.fileExists(filename)) {
@@ -2603,29 +2624,46 @@ public class MetadataPerspective implements
IHopPerspective, TabClosable, IMetad
* its metadata type model, so explicitly-created (and possibly empty)
folders are rendered.
*/
private void loadPersistedFolders() {
+ // Folders can have been stored under a key the metadata type had before
it was renamed.
+ //
Map<String, MetadataTypeModel> byKey = new LinkedHashMap<>();
for (MetadataTypeModel typeModel : typeModels) {
- byKey.put(typeModel.key, typeModel);
+ HopMetadata annotation =
HopMetadataUtil.getHopMetadataAnnotation(typeModel.metadataClass);
+ for (String key : HopMetadataUtil.getAllKeys(annotation)) {
+ byKey.putIfAbsent(key, typeModel);
+ }
}
try {
- AuditList list =
- AuditManager.getActive().retrieveList(getAuditNamespace(),
FOLDER_AUDIT_TYPE);
+ IAuditManager auditManager = AuditManager.getActive();
+ String namespace = getAuditNamespace();
+ AuditList list = auditManager.retrieveList(namespace, FOLDER_AUDIT_TYPE);
if (list == null || list.getNames() == null) {
return;
}
- for (String entry : list.getNames()) {
+ boolean migrated = false;
+ for (int i = 0; i < list.getNames().size(); i++) {
+ String entry = list.getNames().get(i);
int sep = entry.indexOf(FOLDER_AUDIT_SEPARATOR);
if (sep < 0) {
continue;
}
- MetadataTypeModel typeModel = byKey.get(entry.substring(0, sep));
+ String key = entry.substring(0, sep);
+ MetadataTypeModel typeModel = byKey.get(key);
String path = entry.substring(sep + FOLDER_AUDIT_SEPARATOR.length());
+ if (typeModel != null && !typeModel.key.equals(key)) {
+ // Store it under the current key so removing the folder later on
works.
+ list.getNames().set(i, typeModel.key + FOLDER_AUDIT_SEPARATOR +
path);
+ migrated = true;
+ }
if (typeModel != null
&& !Utils.isEmpty(path)
&& !typeModel.folderVirtualPaths.contains(path)) {
typeModel.folderVirtualPaths.add(path);
}
}
+ if (migrated) {
+ auditManager.storeList(namespace, FOLDER_AUDIT_TYPE, list);
+ }
} catch (Exception e) {
LogChannel.UI.logError("Error reading metadata virtual folders from the
audit trail", e);
}
diff --git
a/ui/src/main/java/org/apache/hop/ui/hopgui/search/HopGuiMetadataSearchable.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/search/HopGuiMetadataSearchable.java
index 7c542db465..00650185d8 100644
---
a/ui/src/main/java/org/apache/hop/ui/hopgui/search/HopGuiMetadataSearchable.java
+++
b/ui/src/main/java/org/apache/hop/ui/hopgui/search/HopGuiMetadataSearchable.java
@@ -23,6 +23,7 @@ import org.apache.hop.metadata.api.IHopMetadata;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.api.IHopMetadataSerializer;
import org.apache.hop.metadata.serializer.json.JsonMetadataSerializer;
+import org.apache.hop.metadata.util.HopMetadataUtil;
import org.apache.hop.ui.core.metadata.MetadataManager;
import org.apache.hop.ui.hopgui.HopGui;
@@ -61,6 +62,17 @@ public class HopGuiMetadataSearchable implements
ISearchable<IHopMetadata> {
@Override
public String getFilename() {
+ // The serializer is usually a multi-provider's, which knows no files. Ask
the file based
+ // provider which holds the object: the child project before its parent,
and a legacy folder of
+ // a renamed metadata type if that is where the object still lives.
+ try {
+ String filename = HopMetadataUtil.findFilename(metadataProvider,
managedClass, getName());
+ if (filename != null) {
+ return filename;
+ }
+ } catch (Exception e) {
+ // Fall back to where the object would be saved.
+ }
if (serializer instanceof JsonMetadataSerializer jsonMetadataSerializer) {
return jsonMetadataSerializer.calculateFilename(getName());
}
diff --git
a/ui/src/test/java/org/apache/hop/ui/hopgui/search/HopGuiMetadataSearchableTest.java
b/ui/src/test/java/org/apache/hop/ui/hopgui/search/HopGuiMetadataSearchableTest.java
new file mode 100644
index 0000000000..87eef0f293
--- /dev/null
+++
b/ui/src/test/java/org/apache/hop/ui/hopgui/search/HopGuiMetadataSearchableTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.hopgui.search;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hop.core.encryption.HopTwoWayPasswordEncoder;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.metadata.api.HopMetadata;
+import org.apache.hop.metadata.api.HopMetadataBase;
+import org.apache.hop.metadata.api.IHopMetadata;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.api.IHopMetadataSerializer;
+import org.apache.hop.metadata.serializer.json.JsonMetadataProvider;
+import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * A metadata search result points at the file of the object. Project search
hands it a
+ * multi-provider's serializer, which knows no files: the file has to come
from the provider which
+ * holds the object (issue #5597).
+ */
+class HopGuiMetadataSearchableTest {
+
+ /** A metadata type which was renamed: its objects can still be in the
SearchType folder. */
+ @HopMetadata(name = "Search type", key = "search-type", legacyKeys =
"SearchType")
+ public static class SearchType extends HopMetadataBase implements
IHopMetadata {
+ public SearchType() {
+ // For the serializer.
+ }
+ }
+
+ @TempDir Path parentFolder;
+ @TempDir Path childFolder;
+
+ @Test
+ @SuppressWarnings("unchecked")
+ void theFileOfAnObjectFoundThroughAMultiProvider() throws Exception {
+ // The parent migrated to the current key, the child still has the object
in the legacy folder
+ // and wins, the way load() resolves it.
+ write(parentFolder, "search-type", "api");
+ write(childFolder, "SearchType", "api");
+
+ HopTwoWayPasswordEncoder encoder = new HopTwoWayPasswordEncoder();
+ MultiMetadataProvider multi =
+ new MultiMetadataProvider(
+ encoder,
+ new ArrayList<IHopMetadataProvider>(
+ List.of(
+ new JsonMetadataProvider(
+ encoder, parentFolder.toString(),
Variables.getADefaultVariableSpace()),
+ new JsonMetadataProvider(
+ encoder, childFolder.toString(),
Variables.getADefaultVariableSpace()))),
+ Variables.getADefaultVariableSpace());
+ IHopMetadataSerializer<IHopMetadata> serializer =
+ (IHopMetadataSerializer<IHopMetadata>)
+ (IHopMetadataSerializer<?>) multi.getSerializer(SearchType.class);
+
+ HopGuiMetadataSearchable searchable =
+ new HopGuiMetadataSearchable(
+ multi,
+ serializer,
+ serializer.load("api"),
+ (Class<IHopMetadata>) (Class<?>) SearchType.class);
+
+ assertEquals(
+ childFolder.resolve("SearchType").resolve("api.json").toRealPath(),
+
Path.of(HopVfs.getFileObject(searchable.getFilename()).getURL().toURI()).toRealPath());
+ }
+
+ private static void write(Path metadataFolder, String key, String name)
throws Exception {
+ Path typeFolder = metadataFolder.resolve(key);
+ Files.createDirectories(typeFolder);
+ Files.write(
+ typeFolder.resolve(name + ".json"), ("{\"name\":\"" + name +
"\"}").getBytes(UTF_8));
+ }
+}
