This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 0801a69fd3 Give each execution its own VFS namespace, fixes #8106
(#8119)
0801a69fd3 is described below
commit 0801a69fd34859c66dbbcab6f0a38fbde91410cd
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Thu Aug 27 14:08:43 2026 +0200
Give each execution its own VFS namespace, fixes #8106 (#8119)
---
.../java/org/apache/hop/core/scope/IHopScope.java | 121 +++++++++
.../main/java/org/apache/hop/core/vfs/HopVfs.java | 114 +++++++-
.../org/apache/hop/core/vfs/HopVfsNamespace.java | 161 ++++++++++++
.../org/apache/hop/core/vfs/HopVfsNamespaces.java | 292 +++++++++++++++++++++
.../java/org/apache/hop/core/vfs/plugin/IVfs.java | 19 ++
.../hop/metadata/util/HopMetadataInstance.java | 27 +-
.../variables/VariablesMetadataAnchorTest.java | 60 +++++
.../hop/core/vfs/HopVfsNamespaceScopingTest.java | 238 +++++++++++++++++
.../hop/core/vfs/HopVfsNamespaceThreadTest.java | 187 +++++++++++++
.../integration-tests-hop_server.yaml | 30 ++-
.../java/org/apache/hop/pipeline/Pipeline.java | 51 ++++
.../java/org/apache/hop/workflow/Workflow.java | 28 ++
.../hop/pipeline/PipelineVfsNamespaceTest.java | 78 ++++++
.../hop_server/0014-write-to-export-only-vfs.hpl | 153 +++++++++++
.../hop_server/0014-write-to-export-only-vfs.hwf | 93 +++++++
.../main-0014-test-export-only-vfs-connection.hwf | 159 +++++++++++
.../export-only-storage.json | 12 +
.../apache/hop/projects/gui/ProjectsGuiPlugin.java | 6 +-
.../java/org/apache/hop/vfs/s3/S3VfsPlugin.java | 10 +-
.../org/apache/hop/vfs/s3/metadata/S3Meta.java | 3 +-
.../apache/hop/vfs/s3/metadata/S3MetaEditor.java | 2 +-
.../org/apache/hop/vfs/azure/AzureVfsPlugin.java | 13 +-
.../vfs/azure/metadatatype/AzureMetadataType.java | 3 +-
.../metadatatype/AzureMetadataTypeEditor.java | 2 +-
.../databricks/metadata/DatabricksConnection.java | 3 +-
.../hop/vfs/databricks/DatabricksVfsPlugin.java | 13 +-
.../metadata/DatabricksVfsConnection.java | 3 +-
.../java/org/apache/hop/vfs/ftp/FtpVfsPlugin.java | 10 +-
.../apache/hop/vfs/ftp/metadata/FtpConnection.java | 3 +-
.../hop/vfs/ftp/metadata/FtpConnectionEditor.java | 2 +-
.../apache/hop/vfs/gs/GoogleStorageVfsPlugin.java | 10 +-
.../gs/metadatatype/GoogleStorageMetadataType.java | 3 +-
.../GoogleStorageMetadataTypeEditor.java | 2 +-
.../org/apache/hop/vfs/minio/MinioVfsPlugin.java | 17 +-
.../apache/hop/vfs/minio/metadata/MinioMeta.java | 3 +-
.../hop/vfs/minio/metadata/MinioMetaEditor.java | 2 +-
.../apache/hop/vfs/minio/MinioVfsPluginTest.java | 23 +-
.../org/apache/hop/vfs/sftp/SftpVfsPlugin.java | 10 +-
.../hop/vfs/sftp/metadata/SftpConnection.java | 3 +-
.../vfs/sftp/metadata/SftpConnectionEditor.java | 2 +-
.../org/apache/hop/vfs/webdav/WebDavVfsPlugin.java | 10 +-
.../hop/vfs/webdav/metadata/WebDavConnection.java | 3 +-
.../webdav/metadata/WebDavConnectionEditor.java | 2 +-
.../main/java/org/apache/hop/ui/hopgui/HopWeb.java | 6 +
.../org/apache/hop/ui/hopgui/HopWebEntryPoint.java | 8 +-
.../org/apache/hop/ui/hopgui/RapSessionScope.java | 115 ++++++++
.../apache/hop/ui/hopgui/RapSessionScopeTest.java | 66 +++++
.../main/java/org/apache/hop/ui/hopgui/HopGui.java | 77 +++++-
48 files changed, 2212 insertions(+), 46 deletions(-)
diff --git a/core/src/main/java/org/apache/hop/core/scope/IHopScope.java
b/core/src/main/java/org/apache/hop/core/scope/IHopScope.java
new file mode 100644
index 0000000000..8f24c40161
--- /dev/null
+++ b/core/src/main/java/org/apache/hop/core/scope/IHopScope.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.scope;
+
+/**
+ * Holds one "current" value, for the ambient state Hop keeps outside of any
object it can hand
+ * around: the metadata provider in use, the VFS namespace to resolve files in.
+ *
+ * <p>How far "current" reaches depends on how Hop is running, and that is the
only part that
+ * differs. In a client, or in a server for state set once at startup, it is
the whole process. For
+ * state an execution owns it is the thread and the threads it starts. In Hop
Web one JVM serves
+ * many people at once and neither answer is right: there it has to be the
session, or one user's
+ * project silently becomes another's.
+ *
+ * <p>Each piece of ambient state owns its scope and picks the default that
suits it - see {@link
+ * #process()} and {@link #inheritedByThreads()}. A runtime that needs
different reach implements
+ * this once and hands an instance to each of them.
+ *
+ * @param <T> the type of the value held
+ */
+public interface IHopScope<T> {
+
+ /**
+ * The current value, or null when nothing is set.
+ *
+ * @return the current value, may be null
+ */
+ T get();
+
+ /**
+ * Set the current value.
+ *
+ * @param value the value to set
+ */
+ void set(T value);
+
+ /** Clear the current value. */
+ void remove();
+
+ /**
+ * A scope reaching the whole process: one value, shared by everything. The
right default for
+ * state that is set once at startup.
+ *
+ * @param <T> the type of the value held
+ * @return a new process wide scope
+ */
+ static <T> IHopScope<T> process() {
+ return new ProcessScope<>();
+ }
+
+ /**
+ * A scope reaching the current thread and the threads it starts. The right
default for state an
+ * execution owns: {@code Pipeline} and {@code Workflow} hand their work to
threads they start
+ * themselves, so those inherit it, while a second execution running
alongside keeps its own.
+ *
+ * @param <T> the type of the value held
+ * @return a new thread scope
+ */
+ static <T> IHopScope<T> inheritedByThreads() {
+ return new ThreadScope<>();
+ }
+
+ /**
+ * @see IHopScope#process()
+ */
+ class ProcessScope<T> implements IHopScope<T> {
+ private volatile T value;
+
+ @Override
+ public T get() {
+ return value;
+ }
+
+ @Override
+ public void set(T value) {
+ this.value = value;
+ }
+
+ @Override
+ public void remove() {
+ this.value = null;
+ }
+ }
+
+ /**
+ * @see IHopScope#inheritedByThreads()
+ */
+ class ThreadScope<T> implements IHopScope<T> {
+ private final InheritableThreadLocal<T> value = new
InheritableThreadLocal<>();
+
+ @Override
+ public T get() {
+ return value.get();
+ }
+
+ @Override
+ public void set(T value) {
+ this.value.set(value);
+ }
+
+ @Override
+ public void remove() {
+ value.remove();
+ }
+ }
+}
diff --git a/core/src/main/java/org/apache/hop/core/vfs/HopVfs.java
b/core/src/main/java/org/apache/hop/core/vfs/HopVfs.java
index 6eb3f7d575..2928896d9f 100644
--- a/core/src/main/java/org/apache/hop/core/vfs/HopVfs.java
+++ b/core/src/main/java/org/apache/hop/core/vfs/HopVfs.java
@@ -54,6 +54,8 @@ import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.vfs.plugin.IVfs;
import org.apache.hop.core.vfs.plugin.VfsPluginType;
import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.util.HopMetadataInstance;
public class HopVfs {
private static final Class<?> PKG = HopVfs.class;
@@ -73,6 +75,12 @@ public class HopVfs {
/** Set once the metadata driven providers have been registered on {@link
#fsm}. */
private static boolean namedProvidersRegistered;
+ /**
+ * The metadata the named connections on {@link #fsm} were read from, or
null while nothing has
+ * registered any yet.
+ */
+ private static IHopMetadataProvider defaultNamespaceProvider;
+
/**
* Guards against re-entrant registration: reading the VFS connection
metadata resolves files
* through this very class.
@@ -138,6 +146,11 @@ public class HopVfs {
registeringNamedProviders = true;
try {
registerNamedProviders(fsm, bootstrapVariables);
+ // Remember whose connections these are. Everything running against
this same metadata can
+ // use this manager and needs no namespace of its own; everything else
does. Note it is the
+ // provider the manager was *built from* that matters, not whatever is
current later on:
+ // in Hop Web "current" is per session, and would stop describing this
shared manager.
+ defaultNamespaceProvider = HopMetadataInstance.getMetadataProvider();
} finally {
registeringNamedProviders = false;
namedProvidersRegistered = true;
@@ -154,6 +167,13 @@ public class HopVfs {
* @see #getFileSystemManager()
*/
public static synchronized DefaultFileSystemManager
getFileSystemManager(IVariables variables) {
+ // An execution running against its own metadata - an export on a Hop
Server - has its own
+ // namespace, with its own named VFS connections. Everything else uses the
process wide
+ // manager below, exactly as before. See issue #8106.
+ HopVfsNamespace namespace = HopVfsNamespaces.resolve(variables);
+ if (namespace != null) {
+ return namespace.getFileSystemManager();
+ }
bootstrapWith(variables);
return getFileSystemManager();
}
@@ -177,11 +197,27 @@ public class HopVfs {
*/
private static void registerNamedProviders(
DefaultFileSystemManager manager, IVariables variables) {
+ registerNamedProviders(manager, variables, null);
+ }
+
+ /**
+ * Register a provider for every named VFS connection, reading them from
{@code metadataProvider}
+ * when one is given. A null provider leaves every VFS plugin to find the
metadata itself from the
+ * variables, which is what the process wide manager does.
+ *
+ * @param manager the manager to register the providers on
+ * @param variables the variables the providers resolve their settings with
+ * @param metadataProvider the metadata holding the connections, or null to
derive it
+ */
+ static void registerNamedProviders(
+ DefaultFileSystemManager manager,
+ IVariables variables,
+ IHopMetadataProvider metadataProvider) {
PluginRegistry registry = PluginRegistry.getInstance();
for (IPlugin plugin : registry.getPlugins(VfsPluginType.class)) {
try {
IVfs iVfs = registry.loadClass(plugin, IVfs.class);
- Map<String, FileProvider> fileProviderMap =
iVfs.getProviders(variables);
+ Map<String, FileProvider> fileProviderMap =
iVfs.getProviders(variables, metadataProvider);
if (fileProviderMap == null) {
continue;
}
@@ -193,7 +229,11 @@ public class HopVfs {
+ scheme
+ "' of plugin "
+ plugin.getIds()[0]
- + " is ignored: a provider is already registered for that
scheme.");
+ + " is ignored: a provider is already registered for that
scheme."
+ + " Two named connections can not share a name: rename one
of them,"
+ + " otherwise files resolved through '"
+ + scheme
+ + ":' silently use the other connection.");
continue;
}
manager.addProvider(scheme, entry.getValue());
@@ -216,7 +256,17 @@ public class HopVfs {
* @throws HopException
*/
@SuppressWarnings("java:S2095") // the file system manager is a process-wide
singleton
- private static DefaultFileSystemManager createFileSystemManager() throws
HopException {
+ /**
+ * The metadata whose named VFS connections are registered on the process
wide file system
+ * manager. Anything running against this same metadata is already served by
it.
+ *
+ * @return the metadata behind the process wide manager, or null if nothing
registered yet
+ */
+ static synchronized IHopMetadataProvider getDefaultNamespaceProvider() {
+ return defaultNamespaceProvider;
+ }
+
+ static DefaultFileSystemManager createFileSystemManager() throws
HopException {
try {
DefaultFileSystemManager fsm = new DefaultFileSystemManager();
fsm.addProvider("ram", new
org.apache.commons.vfs2.provider.ram.RamFileProvider());
@@ -292,20 +342,32 @@ public class HopVfs {
}
/**
+ * Resolve a file in the VFS namespace these variables belong to.
+ *
+ * <p>Prefer this over {@link #getFileObject(String)} anywhere inside an
execution: the variables
+ * are what tell us which named VFS connections apply. An execution carrying
its own metadata - an
+ * export running on a Hop Server - has its own connections, and resolving
without variables can
+ * only fall back to the namespace bound to the current thread.
+ *
* @param vfsFilename the name of the file to resolve
- * @param variables the variables to bootstrap the metadata driven providers
with, in case nothing
- * did that yet. They play no role in resolving the file itself.
+ * @param variables the variables of the caller
* @return the file object
* @see #getFileObject(String)
*/
public static FileObject getFileObject(String vfsFilename, IVariables
variables)
throws HopFileException {
- bootstrapWith(variables);
- return getFileObject(vfsFilename);
+ return resolveWith(vfsFilename, getFileSystemManager(variables));
}
public static synchronized FileObject getFileObject(String vfsFilename)
throws HopFileException {
- DefaultFileSystemManager fsManager = getFileSystemManager();
+ // Nothing to go on but the thread: the namespace of the execution running
on it, if any.
+ HopVfsNamespace namespace = HopVfsNamespaces.getCurrent();
+ return resolveWith(
+ vfsFilename, namespace == null ? getFileSystemManager() :
namespace.getFileSystemManager());
+ }
+
+ private static FileObject resolveWith(String vfsFilename,
DefaultFileSystemManager fsManager)
+ throws HopFileException {
try {
// We have one problem with VFS: if the file is in a subdirectory of the
current one:
@@ -785,7 +847,43 @@ public class HopVfs {
* the next time it's used. The bootstrap variables are kept: use {@link
* #setBootstrapVariables(IVariables)} to change those.
*/
+ /**
+ * Read the named VFS connections again for whoever these variables belong
to, after one of them
+ * was added or changed.
+ *
+ * <p>Use this rather than {@link #reset()} from anything that belongs to
one project, one session
+ * or one execution - saving a connection in its editor, switching project.
{@link #reset()}
+ * empties the whole JVM, which in Hop Web means one user's save
invalidating every open file of
+ * everyone else.
+ *
+ * @param variables the variables of whoever changed a connection
+ */
+ public static void refresh(IVariables variables) {
+ try {
+ if (HopVfsNamespaces.refresh(variables)) {
+ return;
+ }
+ } catch (Exception e) {
+ // Deliberately not falling back to reset(): the caller has a namespace
of its own, which
+ // means tenants share this JVM, and emptying it for all of them is the
very thing this
+ // avoids. Their connections stay as they were until the next attempt.
+ LogChannel.GENERAL.logError(
+ "Error re-reading the named VFS connections of this caller. They are
unchanged.", e);
+ return;
+ }
+ // Nothing of their own to refresh: what they resolve files in is the
process wide manager.
+ reset();
+ }
+
+ /**
+ * Throw away the process wide file system manager and everything in it, so
the next use builds it
+ * again. Every file object anywhere in the JVM stops working, so keep this
for process level
+ * events - starting a client, starting a worker. Everything else wants
{@link
+ * #refresh(IVariables)}.
+ */
public static synchronized void reset() {
+ HopVfsNamespaces.reset();
+ defaultNamespaceProvider = null;
if (fsm != null) {
fsm.freeUnusedResources();
fsm.close();
diff --git a/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespace.java
b/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespace.java
new file mode 100644
index 0000000000..eebbaccbaa
--- /dev/null
+++ b/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespace.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.vfs;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+
+/**
+ * A private VFS world: one {@link DefaultFileSystemManager} carrying the
fixed schemes plus the
+ * named VFS connections of exactly one metadata provider.
+ *
+ * <p>Named connections are registered as URI schemes, and the scheme is the
connection's name. On a
+ * shared JVM - a Hop Server running exports for several people, or Hop Web
serving several sessions
+ * - one manager therefore means one global namespace: the first connection
called {@code mydata}
+ * wins and everybody else silently gets it. A namespace per metadata provider
keeps the scheme
+ * table, the provider instances, the files cache and the replicator apart.
+ *
+ * <p>Instances are handed out by {@link HopVfsNamespaces}, never constructed
directly.
+ *
+ * @see HopVfsNamespaces
+ */
+public class HopVfsNamespace implements AutoCloseable {
+
+ private final String description;
+
+ /** Swapped for a fresh one by {@link #rebuild()}; read without locking, so
volatile. */
+ private volatile DefaultFileSystemManager fileSystemManager;
+
+ /** What the named connections were registered from, so this namespace can
rebuild itself. */
+ private IVariables variables;
+
+ private IHopMetadataProvider metadataProvider;
+
+ /** How many executions are using this namespace; it closes when the last
one lets go. */
+ private int useCount;
+
+ HopVfsNamespace(String description) throws HopException {
+ this.description = description;
+ this.fileSystemManager = HopVfs.createFileSystemManager();
+ try {
+ this.fileSystemManager.init();
+ } catch (Exception e) {
+ throw new HopException("Error initializing the file system manager of "
+ description, e);
+ }
+ }
+
+ /**
+ * Register the named VFS connections of this namespace's metadata. Called
once, by {@link
+ * HopVfsNamespaces}, after the namespace is published so re-entrant
resolution finds it.
+ */
+ void registerNamedProviders(IVariables variables, IHopMetadataProvider
metadataProvider) {
+ this.variables = variables;
+ this.metadataProvider = metadataProvider;
+
+ Set<String> before = new
HashSet<>(Arrays.asList(fileSystemManager.getSchemes()));
+ HopVfs.registerNamedProviders(fileSystemManager, variables,
metadataProvider);
+ List<String> registered =
+ Arrays.stream(fileSystemManager.getSchemes())
+ .filter(scheme -> !before.contains(scheme))
+ .sorted()
+ .toList();
+ // Worth saying out loud: an empty list here is why a path like
'mydata://...' quietly turns
+ // into a relative local file instead of reaching the object store.
+ LogChannel.GENERAL.logBasic(
+ "VFS namespace of " + description + " : named connections registered "
+ registered);
+ }
+
+ /**
+ * Read the named connections again into a fresh file system manager, after
one of them changed.
+ *
+ * <p>A provider holds the settings of its connection as they were when it
was registered, and a
+ * manager gives no way to replace one, so the whole manager is built again.
Every {@link
+ * org.apache.commons.vfs2.FileObject} resolved through the old one stops
working - which is what
+ * makes this worth scoping to the namespace whose connections actually
changed, rather than doing
+ * it to every namespace in the JVM.
+ *
+ * <p>The namespace itself survives, so whoever holds it keeps working and
the use count stands.
+ *
+ * @throws HopException if the new manager cannot be created
+ */
+ void rebuild() throws HopException {
+ DefaultFileSystemManager previous = fileSystemManager;
+ DefaultFileSystemManager fresh = HopVfs.createFileSystemManager();
+ try {
+ fresh.init();
+ } catch (Exception e) {
+ throw new HopException("Error initializing the new file system manager
of " + description, e);
+ }
+ fileSystemManager = fresh;
+ if (metadataProvider != null) {
+ registerNamedProviders(variables, metadataProvider);
+ }
+ closeManager(previous);
+ }
+
+ public DefaultFileSystemManager getFileSystemManager() {
+ return fileSystemManager;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ int retain() {
+ return ++useCount;
+ }
+
+ int release() {
+ return --useCount;
+ }
+
+ int getUseCount() {
+ return useCount;
+ }
+
+ /**
+ * Close the manager and everything it holds. Only the owner of the last use
closes a namespace:
+ * every {@link org.apache.commons.vfs2.FileObject} resolved through it dies
with it, and result
+ * files carry those objects out of an execution.
+ */
+ @Override
+ public void close() {
+ closeManager(fileSystemManager);
+ }
+
+ private void closeManager(DefaultFileSystemManager manager) {
+ try {
+ manager.freeUnusedResources();
+ manager.close();
+ } catch (Exception e) {
+ LogChannel.GENERAL.logError("Error closing the file system manager of "
+ description, e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "VFS namespace of " + description;
+ }
+}
diff --git a/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespaces.java
b/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespaces.java
new file mode 100644
index 0000000000..66d9dc1542
--- /dev/null
+++ b/core/src/main/java/org/apache/hop/core/vfs/HopVfsNamespaces.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.vfs;
+
+import java.util.IdentityHashMap;
+import java.util.Map;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.LogChannel;
+import org.apache.hop.core.scope.IHopScope;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.util.HopMetadataInstance;
+
+/**
+ * Hands out {@link HopVfsNamespace}s, one per metadata provider.
+ *
+ * <p>What decides the meaning of a named VFS connection is not the thread,
the session or the
+ * execution: it is the metadata the connection is read from. Runs that share
a metadata provider
+ * genuinely share a definition and share a namespace; anything running
against its own metadata -
+ * an export on a Hop Server carries its own, bundled in the ZIP - gets its
own.
+ *
+ * <p>Everything running against the metadata the process was started with
keeps using the process
+ * wide manager of {@link HopVfs}, so nothing changes for Hop GUI, {@code
hop-run}, or a server
+ * running a pipeline by filename.
+ *
+ * <p>See Apache Hop issue #8106.
+ */
+public class HopVfsNamespaces {
+
+ /**
+ * Keyed on provider identity, not equality: two providers holding the same
connections are still
+ * two namespaces. Deliberate - equality would have to compare whole
metadata trees.
+ */
+ private static final Map<IHopMetadataProvider, HopVfsNamespace> NAMESPACES =
+ new IdentityHashMap<>();
+
+ /**
+ * Where the current namespace lives, for the call sites that resolve a file
without any variables
+ * in hand. Per thread unless something replaces it: {@code Pipeline} and
{@code Workflow} spawn
+ * their threads with a plain {@code new Thread(...)}, so the threads of an
execution inherit the
+ * namespace it bound. Hop Web replaces this with a per session scope - see
{@link
+ * IHopVfsNamespaceScope}.
+ *
+ * <p>This is the fallback. Where an {@link IVariables} is available,
resolving through it is
+ * exact and does not depend on where the work ended up running.
+ */
+ private static IHopScope<HopVfsNamespace> scope =
IHopScope.inheritedByThreads();
+
+ /**
+ * Change how far "current" reaches. Call this once at startup, before
anything resolves a file:
+ * Hop Web installs a per session scope here - the same one it installs on
{@link
+ * org.apache.hop.metadata.util.HopMetadataInstance} - and everything else
leaves the per thread
+ * default in place.
+ *
+ * @param newScope the scope to hold the current namespace in, null restores
the default
+ */
+ public static synchronized void setScope(IHopScope<HopVfsNamespace>
newScope) {
+ scope = newScope == null ? IHopScope.inheritedByThreads() : newScope;
+ // A runtime only installs a scope of its own when several tenants share
the JVM. There, no
+ // tenant may fall back to the process wide manager: they would share it,
and one of them
+ // rebuilding its connections would take the others down with it.
+ isolateEverything = newScope != null;
+ }
+
+ /** Set when the runtime serves several tenants at once, so nobody shares
the process manager. */
+ private static boolean isolateEverything;
+
+ private HopVfsNamespaces() {
+ // Utility class
+ }
+
+ /**
+ * The namespace to resolve a file with, or null when the process wide
manager of {@link HopVfs}
+ * applies.
+ *
+ * @param variables the variables of the caller, may be null
+ * @return the namespace to use, or null for the process wide manager
+ */
+ public static HopVfsNamespace resolve(IVariables variables) {
+ IHopMetadataProvider provider =
+ variables == null ? null : variables.findExecutionMetadataProvider();
+
+ // Running against the metadata the process was set up with? Then the
process wide manager
+ // already holds exactly these connections and there is nothing to isolate.
+ if (provider != null && !isProcessMetadata(provider)) {
+ HopVfsNamespace namespace = existing(provider);
+ if (namespace != null) {
+ return namespace;
+ }
+ // Metadata of its own, but nobody took a namespace for it. Resolution
falls back below, so
+ // a named connection carried by this metadata is about to be missed.
Nothing is wired to
+ // acquire() on this path: worth knowing which one it is.
+ if (LogChannel.GENERAL.isDebug()) {
+ LogChannel.GENERAL.logDebug(
+ "Resolving a file against metadata with no VFS namespace of its
own: "
+ + provider.getDescription()
+ + ". Its named VFS connections are not registered.");
+ }
+ }
+ return scope.get();
+ }
+
+ /** The namespace bound to this thread, or null. */
+ public static HopVfsNamespace getCurrent() {
+ return scope.get();
+ }
+
+ /**
+ * Take the namespace of a context running against its own metadata,
creating it if this is the
+ * first user, and return it. Every caller that gets a namespace back must
{@link #release} it
+ * when it is done - an execution when it finishes, a Hop Web session when
it is disposed.
+ *
+ * <p>Returns null - and takes nothing - when the context runs against the
metadata of the process
+ * itself, which is the ordinary case for Hop GUI, {@code hop-run} and a
server running a pipeline
+ * by filename. Those keep using the process wide manager of {@link HopVfs}.
+ *
+ * @param variables the variables the connections resolve their settings with
+ * @param metadataProvider the metadata this context runs against
+ * @param description what this namespace belongs to, for logging
+ * @return the namespace, or null when the process wide manager applies
+ */
+ public static synchronized HopVfsNamespace acquire(
+ IVariables variables, IHopMetadataProvider metadataProvider, String
description) {
+ if (metadataProvider == null || isProcessMetadata(metadataProvider)) {
+ LogChannel.GENERAL.logDebug(
+ "No separate VFS namespace for "
+ + description
+ + ": it runs against the metadata of the process itself");
+ return null;
+ }
+
+ HopVfsNamespace namespace = NAMESPACES.get(metadataProvider);
+ if (namespace == null) {
+ try {
+ namespace = new HopVfsNamespace(description);
+ } catch (HopException e) {
+ LogChannel.GENERAL.logError(
+ "Unable to create a VFS namespace for "
+ + description
+ + ", falling back to the process wide file system manager",
+ e);
+ return null;
+ }
+ // Publish before registering: registering the named connections reads
metadata, which
+ // resolves files, which lands right back in resolve().
+ NAMESPACES.put(metadataProvider, namespace);
+ namespace.registerNamedProviders(variables, metadataProvider);
+ LogChannel.GENERAL.logBasic(
+ "Created a VFS namespace for "
+ + description
+ + ", holding the named VFS connections of "
+ + metadataProvider.getDescription());
+ }
+ namespace.retain();
+ return namespace;
+ }
+
+ /**
+ * Let go of a namespace taken with {@link #acquire}. It is closed once
nothing is using it any
+ * more: every file object resolved through it dies with it, so release only
after the results of
+ * the execution have been dealt with.
+ *
+ * @param metadataProvider the metadata the namespace was taken for
+ */
+ public static synchronized void release(IHopMetadataProvider
metadataProvider) {
+ if (metadataProvider == null) {
+ return;
+ }
+ HopVfsNamespace namespace = NAMESPACES.get(metadataProvider);
+ if (namespace == null) {
+ return;
+ }
+ if (namespace.release() <= 0) {
+ NAMESPACES.remove(metadataProvider);
+ namespace.close();
+ }
+ }
+
+ /**
+ * Make a namespace the one for this thread and the threads it starts, so
the call sites that
+ * resolve a file without variables land in it too.
+ *
+ * <p>Returns whatever was bound before, which the caller must hand back to
{@link #restoreThread}
+ * in a finally. Executions nest - a pipeline inside a workflow, a resolver
pipeline inside a
+ * transform - and they run on the same thread, so a nested execution that
simply cleared the
+ * binding on its way out would leave the rest of its parent resolving files
in the wrong
+ * namespace.
+ *
+ * <p>A null namespace leaves the binding untouched: an execution running
against the metadata of
+ * the process has nothing of its own to bind, and must not drop what its
parent bound. Threads
+ * already started keep the namespace they inherited either way.
+ *
+ * @param namespace the namespace to bind, or null to leave the binding as
it is
+ * @return the namespace that was bound before, to pass to {@link
#restoreThread}
+ */
+ public static HopVfsNamespace bindThread(HopVfsNamespace namespace) {
+ HopVfsNamespace previous = scope.get();
+ if (namespace != null) {
+ scope.set(namespace);
+ }
+ return previous;
+ }
+
+ /**
+ * Put back what {@link #bindThread} returned.
+ *
+ * @param previous the namespace that was bound before, may be null
+ */
+ public static void restoreThread(HopVfsNamespace previous) {
+ if (previous == null) {
+ scope.remove();
+ } else {
+ scope.set(previous);
+ }
+ }
+
+ /**
+ * Is this the metadata the process wide file system manager already holds
the connections of?
+ *
+ * <p>Compared against the provider that manager was built from, not against
whatever is current
+ * now: {@link HopMetadataInstance} reaches only the session in Hop Web, so
the current provider
+ * there says nothing about a manager shared by every session. Before
anything has registered, the
+ * current provider is the best answer available and matches how the manager
is about to be built.
+ */
+ private static boolean isProcessMetadata(IHopMetadataProvider provider) {
+ if (isolateEverything) {
+ return false;
+ }
+ IHopMetadataProvider behindTheManager =
HopVfs.getDefaultNamespaceProvider();
+ if (behindTheManager != null) {
+ return provider == behindTheManager;
+ }
+ return provider == HopMetadataInstance.getMetadataProvider();
+ }
+
+ /**
+ * Read the named connections again for the namespace these variables
resolve files in, after one
+ * of them changed.
+ *
+ * <p>Scoped on purpose: rebuilding invalidates every file object of that
namespace, and in a JVM
+ * shared by several tenants - Hop Web - the others must not notice. Returns
false when the caller
+ * has no namespace of its own and the process wide manager is what needs
rebuilding instead.
+ *
+ * @param variables the variables of whoever changed a connection
+ * @return true when a namespace was rebuilt
+ * @throws HopException if the namespace cannot be rebuilt
+ */
+ public static synchronized boolean refresh(IVariables variables) throws
HopException {
+ HopVfsNamespace namespace = resolve(variables);
+ if (namespace == null) {
+ return false;
+ }
+ namespace.rebuild();
+ return true;
+ }
+
+ private static synchronized HopVfsNamespace existing(IHopMetadataProvider
provider) {
+ return NAMESPACES.get(provider);
+ }
+
+ /**
+ * Close every namespace and unbind the current one. For tests and for a
clean shutdown.
+ *
+ * <p>Leaves the installed scope in place: which runtime this is does not
change because the
+ * namespaces were closed. A test that installed one puts it back with
{@code setScope(null)}.
+ */
+ public static synchronized void reset() {
+ NAMESPACES.values().forEach(HopVfsNamespace::close);
+ NAMESPACES.clear();
+ scope.remove();
+ }
+
+ /** How many namespaces are open right now. For tests and diagnostics. */
+ public static synchronized int size() {
+ return NAMESPACES.size();
+ }
+}
diff --git a/core/src/main/java/org/apache/hop/core/vfs/plugin/IVfs.java
b/core/src/main/java/org/apache/hop/core/vfs/plugin/IVfs.java
index a6ccf71680..b44fdc1bc7 100644
--- a/core/src/main/java/org/apache/hop/core/vfs/plugin/IVfs.java
+++ b/core/src/main/java/org/apache/hop/core/vfs/plugin/IVfs.java
@@ -20,6 +20,7 @@ package org.apache.hop.core.vfs.plugin;
import java.util.Map;
import org.apache.commons.vfs2.provider.FileProvider;
import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
public interface IVfs {
/**
@@ -47,4 +48,22 @@ public interface IVfs {
* @return a provider per named connection, empty or null if there are none
*/
Map<String, FileProvider> getProviders(IVariables variables);
+
+ /**
+ * The providers of the named connections in {@code metadataProvider}, keyed
by connection name.
+ *
+ * <p>An execution does not always run against the metadata the variables
point at: a pipeline
+ * exported to a Hop Server carries its own, bundled in the export.
Implement this method to read
+ * the connections from the metadata handed in; the default keeps the old
behaviour of deriving
+ * the metadata from the variables, so a plugin that does not implement it
still works.
+ *
+ * @param variables the variables the connections resolve their settings with
+ * @param metadataProvider the metadata holding the connections, or null to
derive it from the
+ * variables
+ * @return a provider per named connection, empty or null if there are none
+ */
+ default Map<String, FileProvider> getProviders(
+ IVariables variables, IHopMetadataProvider metadataProvider) {
+ return getProviders(variables);
+ }
}
diff --git
a/core/src/main/java/org/apache/hop/metadata/util/HopMetadataInstance.java
b/core/src/main/java/org/apache/hop/metadata/util/HopMetadataInstance.java
index 4a24331048..9c28df442d 100644
--- a/core/src/main/java/org/apache/hop/metadata/util/HopMetadataInstance.java
+++ b/core/src/main/java/org/apache/hop/metadata/util/HopMetadataInstance.java
@@ -18,12 +18,22 @@
package org.apache.hop.metadata.util;
+import org.apache.hop.core.scope.IHopScope;
import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
+/**
+ * The metadata Hop is currently working with, for the code that has nothing
else to go on.
+ *
+ * <p>How far "currently" reaches is a property of the runtime, not of this
class. In a client or a
+ * server it is the whole process: one project, or one server configuration,
set once at startup. In
+ * Hop Web it is the session - every user has their own project open in the
same JVM, and a process
+ * wide answer would hand one user's metadata to another. A runtime that needs
a different reach
+ * installs its own scope with {@link #setScope(IHopScope)} before anything
reads this.
+ */
public class HopMetadataInstance {
private static HopMetadataInstance instance;
- private MultiMetadataProvider metadataProvider;
+ private IHopScope<MultiMetadataProvider> scope = IHopScope.process();
private HopMetadataInstance() {
// Nothing to do here.
@@ -36,11 +46,22 @@ public class HopMetadataInstance {
return instance;
}
+ /**
+ * Change how far "current" reaches. Call this once at startup, before
anything reads the metadata
+ * provider: Hop Web installs a per session scope here, everything else
leaves the process wide
+ * default in place.
+ *
+ * @param scope the scope to hold the metadata provider in, null restores
the process wide default
+ */
+ public static void setScope(IHopScope<MultiMetadataProvider> scope) {
+ getInstance().scope = scope == null ? IHopScope.process() : scope;
+ }
+
public static void setMetadataProvider(MultiMetadataProvider
metadataProvider) {
- getInstance().metadataProvider = metadataProvider;
+ getInstance().scope.set(metadataProvider);
}
public static MultiMetadataProvider getMetadataProvider() {
- return getInstance().metadataProvider;
+ return getInstance().scope.get();
}
}
diff --git
a/core/src/test/java/org/apache/hop/core/variables/VariablesMetadataAnchorTest.java
b/core/src/test/java/org/apache/hop/core/variables/VariablesMetadataAnchorTest.java
new file mode 100644
index 0000000000..3b0d1fcea0
--- /dev/null
+++
b/core/src/test/java/org/apache/hop/core/variables/VariablesMetadataAnchorTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.variables;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.mockito.Mockito.mock;
+
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Hop GUI keeps its project's metadata reachable from its variables by
putting it behind them,
+ * rather than in them, so that handing the GUI a fresh set of variables -
which loading a project
+ * does - cannot quietly lose it. This is the property that makes that safe.
+ */
+class VariablesMetadataAnchorTest {
+
+ @Test
+ @DisplayName("An anchor behind a variable space is found, and does not touch
variable lookup")
+ void anchorIsReachableAndInert() {
+ IHopMetadataProvider provider = mock(IHopMetadataProvider.class);
+ IVariables anchor =
+ new Variables() {
+ @Override
+ public IHopMetadataProvider getMetadataProvider() {
+ return provider;
+ }
+ };
+
+ IVariables freshSpace = Variables.getADefaultVariableSpace();
+ freshSpace.setParentVariables(anchor);
+ freshSpace.setVariable("SOME_VARIABLE", "its own value");
+
+ assertSame(
+ provider,
+ freshSpace.findExecutionMetadataProvider(),
+ "The metadata behind the space has to be reachable from it");
+ assertEquals(
+ "its own value",
+ freshSpace.getVariable("SOME_VARIABLE"),
+ "The anchor must not interfere with looking a variable up");
+ }
+}
diff --git
a/core/src/test/java/org/apache/hop/core/vfs/HopVfsNamespaceScopingTest.java
b/core/src/test/java/org/apache/hop/core/vfs/HopVfsNamespaceScopingTest.java
new file mode 100644
index 0000000000..7748cd648e
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/core/vfs/HopVfsNamespaceScopingTest.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.vfs;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.hop.core.logging.HopLogStore;
+import org.apache.hop.core.scope.IHopScope;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
+import org.apache.hop.metadata.util.HopMetadataInstance;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Which context gets a VFS namespace of its own, and which is already served
by the process wide
+ * file system manager. The answer has to hold in a client, on a server
running exports for several
+ * people, and in Hop Web where one JVM serves many sessions at once.
+ */
+class HopVfsNamespaceScopingTest {
+
+ @BeforeAll
+ static void initLogging() {
+ // Creating a namespace logs which connections it registered.
+ HopLogStore.init();
+ }
+
+ @BeforeEach
+ @AfterEach
+ void clearState() {
+ HopVfsNamespaces.reset();
+ HopVfsNamespaces.setScope(null);
+ HopMetadataInstance.setScope(null);
+ HopMetadataInstance.setMetadataProvider(null);
+ HopVfs.setBootstrapVariables(null);
+ HopVfs.reset();
+ }
+
+ /**
+ * What a runtime serving several tenants at once installs at startup: both
the metadata and the
+ * VFS namespace answer per session rather than per process.
+ */
+ private IHopScope<MultiMetadataProvider> installSessionScopes() {
+ IHopScope<MultiMetadataProvider> perSession = IHopScope.process();
+ HopMetadataInstance.setScope(perSession);
+ HopVfsNamespaces.setScope(IHopScope.process());
+ return perSession;
+ }
+
+ /** Build the process wide manager the way loading a project does, from this
metadata. */
+ private void bootstrapProcessManagerFrom(MultiMetadataProvider provider) {
+ HopMetadataInstance.setMetadataProvider(provider);
+ HopVfs.setBootstrapVariables(Variables.getADefaultVariableSpace());
+ HopVfs.getFileSystemManager();
+ }
+
+ @Test
+ @DisplayName("The metadata behind the process manager needs no namespace of
its own")
+ void theProcessMetadataUsesTheProcessManager() {
+ MultiMetadataProvider projectMetadata = mock(MultiMetadataProvider.class);
+ bootstrapProcessManagerFrom(projectMetadata);
+
+ assertNull(
+ HopVfsNamespaces.acquire(new Variables(), projectMetadata, "local
run"),
+ "A run against the metadata the manager was built from should just use
it");
+ }
+
+ @Test
+ @DisplayName("Metadata of its own - an export on a server - gets a namespace
of its own")
+ void otherMetadataGetsItsOwnNamespace() {
+ MultiMetadataProvider serverMetadata = mock(MultiMetadataProvider.class);
+ bootstrapProcessManagerFrom(serverMetadata);
+
+ MultiMetadataProvider exportedMetadata = mock(MultiMetadataProvider.class);
+ HopVfsNamespace namespace =
+ HopVfsNamespaces.acquire(new Variables(), exportedMetadata, "exported
workflow");
+ try {
+ assertNotNull(namespace, "An export carries its own connections and
needs its own namespace");
+ assertNotSame(
+ HopVfs.getFileSystemManager(),
+ namespace.getFileSystemManager(),
+ "The export must not be resolving files on the shared manager");
+ } finally {
+ HopVfsNamespaces.release(exportedMetadata);
+ }
+ }
+
+ @Test
+ @DisplayName("Two Hop Web sessions on their own projects do not share a
namespace")
+ void sessionsDoNotShareANamespace() {
+ // Hop Web gives every session its own metadata, and installs a session
scope so that
+ // HopMetadataInstance answers per session rather than per process.
+ MultiMetadataProvider sessionA = mock(MultiMetadataProvider.class);
+ MultiMetadataProvider sessionB = mock(MultiMetadataProvider.class);
+
+ IHopScope<MultiMetadataProvider> perSession = installSessionScopes();
+
+ // Session A opens first, so the process wide manager ends up holding its
connections.
+ bootstrapProcessManagerFrom(sessionA);
+
+ // Session B opens its own project. Its metadata becomes "current" for
that session, which must
+ // not make it look like the metadata the shared manager was built from.
+ perSession.set(sessionB);
+
+ HopVfsNamespace namespaceB = HopVfsNamespaces.acquire(new Variables(),
sessionB, "session B");
+ HopVfsNamespace namespaceA = HopVfsNamespaces.acquire(new Variables(),
sessionA, "session A");
+ try {
+ assertNotNull(
+ namespaceB, "Session B must not resolve files through the
connections of session A");
+ assertNotNull(
+ namespaceA,
+ "With tenants sharing the JVM nobody falls back to the process
manager: they would share"
+ + " it, and one of them rebuilding its connections would take
the other down");
+ assertNotSame(namespaceA, namespaceB);
+ assertNotSame(HopVfs.getFileSystemManager(),
namespaceB.getFileSystemManager());
+ } finally {
+ HopVfsNamespaces.release(sessionB);
+ HopVfsNamespaces.release(sessionA);
+ }
+ }
+
+ @Test
+ @DisplayName("Contexts sharing metadata share one namespace, and it closes
with the last of them")
+ void sharedMetadataSharesOneNamespace() {
+ bootstrapProcessManagerFrom(mock(MultiMetadataProvider.class));
+ MultiMetadataProvider shared = mock(MultiMetadataProvider.class);
+
+ HopVfsNamespace first = HopVfsNamespaces.acquire(new Variables(), shared,
"workflow");
+ HopVfsNamespace second = HopVfsNamespaces.acquire(new Variables(), shared,
"its pipeline");
+
+ assertSame(first, second, "A pipeline inside a workflow resolves in the
same namespace");
+ assertSame(1, HopVfsNamespaces.size());
+
+ HopVfsNamespaces.release(shared);
+ assertSame(1, HopVfsNamespaces.size(), "The workflow is still using it");
+
+ HopVfsNamespaces.release(shared);
+ assertSame(0, HopVfsNamespaces.size(), "Nothing left using it, so it
closes");
+ }
+
+ @Test
+ @DisplayName("Refreshing one tenant's connections leaves the others alone")
+ void refreshOnlyTouchesTheCallersNamespace() throws Exception {
+ MultiMetadataProvider sessionA = mock(MultiMetadataProvider.class);
+ MultiMetadataProvider sessionB = mock(MultiMetadataProvider.class);
+
+ installSessionScopes();
+ bootstrapProcessManagerFrom(sessionA);
+
+ HopVfsNamespace namespaceA = HopVfsNamespaces.acquire(new Variables(),
sessionA, "session A");
+ HopVfsNamespace namespaceB = HopVfsNamespaces.acquire(new Variables(),
sessionB, "session B");
+ try {
+ DefaultFileSystemManager managerOfA = namespaceA.getFileSystemManager();
+ DefaultFileSystemManager managerOfB = namespaceB.getFileSystemManager();
+
+ // Session B saves a VFS connection. Its own manager is rebuilt to pick
the change up...
+ HopVfsNamespace previous = HopVfsNamespaces.bindThread(namespaceB);
+ try {
+ assertTrue(HopVfsNamespaces.refresh(new Variables()));
+ } finally {
+ HopVfsNamespaces.restoreThread(previous);
+ }
+
+ assertNotSame(managerOfB, namespaceB.getFileSystemManager(), "B should
have read them again");
+ assertSame(
+ managerOfA,
+ namespaceA.getFileSystemManager(),
+ "Session A was in the middle of something and must not have lost its
file system");
+ } finally {
+ HopVfsNamespaces.release(sessionA);
+ HopVfsNamespaces.release(sessionB);
+ }
+ }
+
+ @Test
+ @DisplayName("A scope that answers per session never falls back to what a
pooled thread left")
+ void aSessionNeverInheritsAnotherSessionsNamespace() throws Exception {
+ // Request threads are pooled: the same thread serves one session and then
another. A session
+ // with nothing bound has to get nothing, not whatever the previous one
left on that thread.
+ HopVfsNamespace ofSessionA = new HopVfsNamespace("session A");
+
+ Object[] sessionOnThisThread = {"A"};
+ java.util.Map<Object, HopVfsNamespace> bySession = new
java.util.HashMap<>();
+
+ HopVfsNamespaces.setScope(
+ new IHopScope<HopVfsNamespace>() {
+ @Override
+ public HopVfsNamespace get() {
+ return bySession.get(sessionOnThisThread[0]);
+ }
+
+ @Override
+ public void set(HopVfsNamespace value) {
+ bySession.put(sessionOnThisThread[0], value);
+ }
+
+ @Override
+ public void remove() {
+ bySession.remove(sessionOnThisThread[0]);
+ }
+ });
+ try {
+ HopVfsNamespaces.bindThread(ofSessionA);
+ assertSame(ofSessionA, HopVfsNamespaces.getCurrent());
+
+ // The same thread now serves a session that has opened nothing yet.
+ sessionOnThisThread[0] = "B";
+ assertNull(
+ HopVfsNamespaces.getCurrent(), "Session B was handed the VFS
namespace of session A");
+ } finally {
+ HopVfsNamespaces.setScope(null);
+ }
+ }
+}
diff --git
a/core/src/test/java/org/apache/hop/core/vfs/HopVfsNamespaceThreadTest.java
b/core/src/test/java/org/apache/hop/core/vfs/HopVfsNamespaceThreadTest.java
new file mode 100644
index 0000000000..bb11673bb9
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/core/vfs/HopVfsNamespaceThreadTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.core.vfs;
+
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.mockito.Mockito.mock;
+
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.scope.IHopScope;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Most call sites resolve a file without any variables in hand, so the
namespace of the running
+ * execution has to reach them through the thread. These are the properties
that makes that safe:
+ * executions nest, and they hand work to threads they start themselves.
+ */
+class HopVfsNamespaceThreadTest {
+
+ @AfterEach
+ void tearDown() {
+ HopVfsNamespaces.reset();
+ }
+
+ private HopVfsNamespace namespace(String description) throws HopException {
+ // Bypass the registry: these tests are about the thread binding, not
about the metadata key.
+ return new HopVfsNamespace(description);
+ }
+
+ @Test
+ @DisplayName("A nested execution puts its parent's namespace back on the way
out")
+ void nestedBindingRestoresTheParent() throws Exception {
+ HopVfsNamespace outer = namespace("outer");
+ HopVfsNamespace inner = namespace("inner");
+
+ HopVfsNamespace beforeOuter = HopVfsNamespaces.bindThread(outer);
+ try {
+ assertSame(outer, HopVfsNamespaces.getCurrent());
+
+ HopVfsNamespace beforeInner = HopVfsNamespaces.bindThread(inner);
+ try {
+ assertSame(inner, HopVfsNamespaces.getCurrent());
+ } finally {
+ HopVfsNamespaces.restoreThread(beforeInner);
+ }
+
+ assertSame(
+ outer,
+ HopVfsNamespaces.getCurrent(),
+ "The nested execution left its parent resolving files in the wrong
namespace");
+ } finally {
+ HopVfsNamespaces.restoreThread(beforeOuter);
+ }
+ assertNull(HopVfsNamespaces.getCurrent());
+ }
+
+ @Test
+ @DisplayName("An execution without a namespace of its own does not drop its
parent's")
+ void bindingNothingKeepsTheParentBinding() throws Exception {
+ HopVfsNamespace outer = namespace("outer");
+
+ HopVfsNamespace beforeOuter = HopVfsNamespaces.bindThread(outer);
+ try {
+ // An execution running against the metadata of the process gets null
from acquire().
+ HopVfsNamespace beforeInner = HopVfsNamespaces.bindThread(null);
+ HopVfsNamespaces.restoreThread(beforeInner);
+
+ assertSame(outer, HopVfsNamespaces.getCurrent());
+ } finally {
+ HopVfsNamespaces.restoreThread(beforeOuter);
+ }
+ }
+
+ @Test
+ @DisplayName("Threads an execution starts inherit its namespace")
+ void startedThreadsInheritTheNamespace() throws Exception {
+ HopVfsNamespace ns = namespace("pipeline");
+
+ HopVfsNamespace before = HopVfsNamespaces.bindThread(ns);
+ HopVfsNamespace[] seenByChild = new HopVfsNamespace[1];
+ try {
+ // This is how Pipeline and Workflow hand work to their transforms and
actions.
+ Thread child = new Thread(() -> seenByChild[0] =
HopVfsNamespaces.getCurrent());
+ child.start();
+ child.join();
+ } finally {
+ HopVfsNamespaces.restoreThread(before);
+ }
+
+ assertSame(ns, seenByChild[0], "A transform thread cannot see the
namespace of its pipeline");
+ }
+
+ @Test
+ @DisplayName("Unbinding the starter does not disturb threads already
running")
+ void alreadyStartedThreadsKeepTheirNamespace() throws Exception {
+ HopVfsNamespace ns = namespace("pipeline");
+
+ HopVfsNamespace before = HopVfsNamespaces.bindThread(ns);
+ HopVfsNamespace[] seenByChild = new HopVfsNamespace[1];
+ Thread child = new Thread(() -> seenByChild[0] =
HopVfsNamespaces.getCurrent());
+ child.start();
+ // The server prepares and starts a pipeline on two different request
threads, and each hands
+ // the thread back to its pool afterwards.
+ HopVfsNamespaces.restoreThread(before);
+ child.join();
+
+ assertSame(ns, seenByChild[0]);
+ assertNull(HopVfsNamespaces.getCurrent());
+ }
+
+ @Test
+ @DisplayName("Resolving with variables that know nothing falls back to the
thread")
+ void variablesWithoutMetadataFallBackToTheThread() throws Exception {
+ HopVfsNamespace ns = namespace("pipeline");
+ IHopMetadataProvider unrelated = mock(IHopMetadataProvider.class);
+ assertNull(unrelated.getDescription());
+
+ HopVfsNamespace before = HopVfsNamespaces.bindThread(ns);
+ try {
+ assertSame(ns, HopVfsNamespaces.resolve(new Variables()));
+ assertSame(ns, HopVfsNamespaces.resolve(null));
+ } finally {
+ HopVfsNamespaces.restoreThread(before);
+ }
+ }
+
+ @Test
+ @DisplayName("A replacement scope decides what current means, thread or not")
+ void aReplacementScopeTakesOver() throws Exception {
+ // Hop Web serves many people from one JVM, and the thread handling a
request says nothing
+ // about whose project it belongs to. It binds a per session scope here
instead.
+ HopVfsNamespace sessionNamespace = namespace("session");
+ HopVfsNamespace[] held = new HopVfsNamespace[1];
+ try {
+ HopVfsNamespaces.setScope(
+ new IHopScope<HopVfsNamespace>() {
+ @Override
+ public HopVfsNamespace get() {
+ return held[0];
+ }
+
+ @Override
+ public void set(HopVfsNamespace namespace) {
+ held[0] = namespace;
+ }
+
+ @Override
+ public void remove() {
+ held[0] = null;
+ }
+ });
+
+ HopVfsNamespaces.bindThread(sessionNamespace);
+ assertSame(sessionNamespace, HopVfsNamespaces.getCurrent());
+
+ // Not the thread's: a brand new thread that inherited nothing still
sees it.
+ HopVfsNamespace[] seenElsewhere = new HopVfsNamespace[1];
+ Thread other = new Thread(() -> seenElsewhere[0] =
HopVfsNamespaces.getCurrent());
+ other.start();
+ other.join();
+ assertSame(sessionNamespace, seenElsewhere[0]);
+ } finally {
+ HopVfsNamespaces.setScope(null);
+ }
+
+ assertNull(HopVfsNamespaces.getCurrent(), "The per thread default should
be back");
+ }
+}
diff --git a/docker/integration-tests/integration-tests-hop_server.yaml
b/docker/integration-tests/integration-tests-hop_server.yaml
index a102fe58cb..786326efe6 100644
--- a/docker/integration-tests/integration-tests-hop_server.yaml
+++ b/docker/integration-tests/integration-tests-hop_server.yaml
@@ -23,11 +23,19 @@ services:
depends_on:
hop_server:
condition: service_healthy
+ minio:
+ condition: service_healthy
links:
- hop_server
+ - minio
hop_server:
image: apache/hop:Development
+ depends_on:
+ minio:
+ condition: service_healthy
+ links:
+ - minio
build:
context: ../../.
dockerfile: docker/Dockerfile
@@ -49,4 +57,24 @@ services:
interval: 20s
timeout: 10s
retries: 6
- start_period: 120s
\ No newline at end of file
+ start_period: 120s
+
+ # Storage behind the "export-only-storage" VFS connection of main-0014. That
connection lives in
+ # the client project's metadata-client-only folder, so it only ever reaches
the server inside the
+ # export ZIP - which is what the test is about.
+ minio:
+ image: minio/minio
+ hostname: minio
+ environment:
+ MINIO_REGION: "local"
+ MINIO_REGION_NAME: "local"
+ MINIO_SCHEME: "http"
+ MINIO_ROOT_USER: "ApacheHopAccessKey"
+ MINIO_ROOT_PASSWORD: "ApacheHopSecretKey"
+ command: server /data --console-address ":9001"
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live";]
+ interval: 10s
+ timeout: 10s
+ retries: 12
+ start_period: 20s
diff --git a/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java
b/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java
index c21372d6d1..0306757303 100644
--- a/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java
+++ b/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java
@@ -90,6 +90,8 @@ import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.core.vfs.HopVfsNamespace;
+import org.apache.hop.core.vfs.HopVfsNamespaces;
import org.apache.hop.execution.sampler.IExecutionDataSampler;
import org.apache.hop.execution.sampler.IExecutionDataSamplerStore;
import org.apache.hop.i18n.BaseMessages;
@@ -563,8 +565,44 @@ public abstract class Pipeline
*
* @throws HopException in case the pipeline could not be prepared
(initialized)
*/
+ /** The VFS namespace of this execution, when it runs against metadata of
its own. */
+ private HopVfsNamespace vfsNamespace;
+
+ /** The metadata it was taken for. Kept so it is let go of by the same key
it was taken with. */
+ private IHopMetadataProvider vfsNamespaceProvider;
+
@Override
public void prepareExecution() throws HopException {
+ // A pipeline carrying its own metadata - an export running on a Hop
Server - resolves its
+ // named VFS connections in its own namespace, not in the one the server
was started with.
+ vfsNamespaceProvider = getMetadataProvider();
+ vfsNamespace =
+ HopVfsNamespaces.acquire(this, vfsNamespaceProvider, "pipeline " +
pipelineMeta.getName());
+ HopVfsNamespace previous = HopVfsNamespaces.bindThread(vfsNamespace);
+ boolean prepared = false;
+ try {
+ prepareExecutionInternal();
+ prepared = true;
+ } finally {
+ HopVfsNamespaces.restoreThread(previous);
+ if (!prepared) {
+ // Preparation failed, so nothing will ever finish this pipeline and
let the namespace go.
+ // On a server that would leak a file system manager for every export
that fails to start.
+ releaseVfsNamespace();
+ }
+ }
+ }
+
+ /** Let go of the VFS namespace of this execution, once and by the key it
was taken with. */
+ private void releaseVfsNamespace() {
+ if (vfsNamespace != null) {
+ HopVfsNamespaces.release(vfsNamespaceProvider);
+ vfsNamespace = null;
+ vfsNamespaceProvider = null;
+ }
+ }
+
+ private void prepareExecutionInternal() throws HopException {
setPreparing(true);
executionStartDate = new Date();
setRunning(false);
@@ -1251,6 +1289,15 @@ public abstract class Pipeline
*/
@Override
public void startThreads() throws HopException {
+ HopVfsNamespace previous = HopVfsNamespaces.bindThread(vfsNamespace);
+ try {
+ startThreadsInternal();
+ } finally {
+ HopVfsNamespaces.restoreThread(previous);
+ }
+ }
+
+ private void startThreadsInternal() throws HopException {
// Now prepare to start all the threads...
//
nrOfFinishedTransforms = 0;
@@ -1498,6 +1545,10 @@ public abstract class Pipeline
//
ExtensionPointHandler.callExtensionPoint(
log, this, HopExtensionPoint.PipelineCompleted.id, this);
+
+ // Only now: everything above can still touch files of this namespace, and
closing it
+ // invalidates every file object resolved through it - the result files
carry those.
+ releaseVfsNamespace();
}
public void pipelineCompleted() throws HopException {
diff --git a/engine/src/main/java/org/apache/hop/workflow/Workflow.java
b/engine/src/main/java/org/apache/hop/workflow/Workflow.java
index c19c9e4dd5..07282b78a3 100644
--- a/engine/src/main/java/org/apache/hop/workflow/Workflow.java
+++ b/engine/src/main/java/org/apache/hop/workflow/Workflow.java
@@ -65,6 +65,8 @@ import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.core.vfs.HopVfsNamespace;
+import org.apache.hop.core.vfs.HopVfsNamespaces;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.pipeline.IExecutionFinishedListener;
@@ -287,6 +289,15 @@ public abstract class Workflow extends Variables
return r;
}
+ /** The VFS namespace of this execution, when it runs against metadata of
its own. */
+ private HopVfsNamespace vfsNamespace;
+
+ /** What was bound to the thread before this execution, to put back when it
finishes. */
+ private HopVfsNamespace previousVfsNamespace;
+
+ /** The metadata it was taken for. Kept so it is let go of by the same key
it was taken with. */
+ private IHopMetadataProvider vfsNamespaceProvider;
+
@Override
public Result startExecution() {
@@ -308,6 +319,14 @@ public abstract class Workflow extends Variables
setInternalHopVariables();
+ // A workflow carrying its own metadata - an export running on a Hop
Server - resolves its
+ // named VFS connections in its own namespace, not in the one the server
was started with.
+ vfsNamespaceProvider = getMetadataProvider();
+ vfsNamespace =
+ HopVfsNamespaces.acquire(
+ this, vfsNamespaceProvider, "workflow " +
workflowMeta.getName());
+ previousVfsNamespace = HopVfsNamespaces.bindThread(vfsNamespace);
+
// Run the workflow
//
fireExecutionStartedListeners();
@@ -349,6 +368,15 @@ public abstract class Workflow extends Variables
BaseMessages.getString(PKG, "Workflow.Log.ErrorExecWorkflow",
e.getMessage()), e);
emergencyWriteWorkflowTracker(result);
+ } finally {
+ // Only now: everything above can still touch files of this namespace,
and closing it
+ // invalidates every file object resolved through it.
+ HopVfsNamespaces.restoreThread(previousVfsNamespace);
+ if (vfsNamespace != null) {
+ HopVfsNamespaces.release(vfsNamespaceProvider);
+ vfsNamespace = null;
+ vfsNamespaceProvider = null;
+ }
}
}
diff --git
a/engine/src/test/java/org/apache/hop/pipeline/PipelineVfsNamespaceTest.java
b/engine/src/test/java/org/apache/hop/pipeline/PipelineVfsNamespaceTest.java
new file mode 100644
index 0000000000..e45b31d6e1
--- /dev/null
+++ b/engine/src/test/java/org/apache/hop/pipeline/PipelineVfsNamespaceTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+
+import org.apache.hop.core.HopEnvironment;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.ILogChannel;
+import org.apache.hop.core.vfs.HopVfsNamespaces;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * A pipeline running against metadata of its own - an export on a Hop Server
- takes a VFS
+ * namespace to resolve its named connections in, and has to let go of it
again. Nothing else will:
+ * the namespace holds a file system manager, and one that is never released
is never closed.
+ */
+class PipelineVfsNamespaceTest {
+
+ @BeforeAll
+ static void setUpBeforeClass() throws HopException {
+ HopEnvironment.init();
+ }
+
+ @AfterEach
+ void clearNamespaces() {
+ HopVfsNamespaces.reset();
+ }
+
+ @Test
+ @DisplayName("A pipeline that fails to prepare does not leave its VFS
namespace behind")
+ void aFailedPreparationLetsGoOfTheNamespace() {
+ IHopMetadataProvider ownMetadata = mock(IHopMetadataProvider.class);
+
+ Pipeline pipeline = spy(new LocalPipelineEngine(new PipelineMeta()));
+ pipeline.setLogChannel(mock(ILogChannel.class));
+ pipeline.setMetadataProvider(ownMetadata);
+
+ // Anything at all going wrong while preparing: a transform that will not
initialise, a run
+ // configuration that cannot be loaded, a file that is not there.
+ doThrow(new
HopRuntimeExceptionForTest()).when(pipeline).setPreparing(anyBoolean());
+
+ assertThrows(HopRuntimeExceptionForTest.class, pipeline::prepareExecution);
+
+ assertEquals(
+ 0,
+ HopVfsNamespaces.size(),
+ "The pipeline never runs and never finishes, so nothing else would
ever close this");
+ }
+
+ /** Distinct type so the assertion cannot pass on some other failure. */
+ private static class HopRuntimeExceptionForTest extends RuntimeException {}
+}
diff --git a/integration-tests/hop_server/0014-write-to-export-only-vfs.hpl
b/integration-tests/hop_server/0014-write-to-export-only-vfs.hpl
new file mode 100644
index 0000000000..5e4f307396
--- /dev/null
+++ b/integration-tests/hop_server/0014-write-to-export-only-vfs.hpl
@@ -0,0 +1,153 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<pipeline>
+ <info>
+ <name>0014-write-to-export-only-vfs</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Runs on the remote Hop Server and writes a file through the
"export-only-storage"
+ named VFS connection. That connection only exists in the client project's
+ metadata-client-only folder, so it reaches the server exclusively inside
the export ZIP. If
+ the server does not register the VFS connections it was sent, the scheme
is unknown, the path
+ is treated as a relative local file and nothing lands in the object
store (#8106).</description>
+ <extended_description/>
+ <pipeline_version/>
+ <pipeline_type>Normal</pipeline_type>
+ <parameters>
+ </parameters>
+ <capture_transform_performance>N</capture_transform_performance>
+
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
+
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
+ <created_user>-</created_user>
+ <created_date>2024/01/01 00:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2024/01/01 00:00:00.000</modified_date>
+ <key_for_session_key/>
+ <is_key_private>N</is_key_private>
+ </info>
+ <notepads>
+ </notepads>
+ <order>
+ <hop>
+ <from>Generate rows</from>
+ <to>Write to export-only-storage</to>
+ <enabled>Y</enabled>
+ </hop>
+ </order>
+ <transform>
+ <name>Generate rows</name>
+ <type>RowGenerator</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <fields>
+ <field>
+ <name>marker</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif>written-through-exported-vfs-connection</nullif>
+ <length>-1</length>
+ <precision>-1</precision>
+ <set_empty_string>N</set_empty_string>
+ </field>
+ </fields>
+ <limit>1</limit>
+ <never_ending>N</never_ending>
+ <interval_in_ms>5000</interval_in_ms>
+ <row_time_field>now</row_time_field>
+ <last_time_field>FiveSecondsAgo</last_time_field>
+ <attributes/>
+ <GUI>
+ <xloc>144</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform>
+ <name>Write to export-only-storage</name>
+ <type>TextFileOutput</type>
+ <description/>
+ <distribute>Y</distribute>
+ <custom_distribution/>
+ <copies>1</copies>
+ <partitioning>
+ <method>none</method>
+ <schema_name/>
+ </partitioning>
+ <separator>;</separator>
+ <enclosure>"</enclosure>
+ <enclosure_forced>N</enclosure_forced>
+ <enclosure_fix_disabled>N</enclosure_fix_disabled>
+ <header>N</header>
+ <footer>N</footer>
+ <format>DOS</format>
+ <compression>None</compression>
+ <encoding>UTF-8</encoding>
+ <endedLine/>
+ <fileNameInField>N</fileNameInField>
+ <fileNameField/>
+ <create_parent_folder>Y</create_parent_folder>
+ <file>
+ <name>export-only-storage:///demo/marker-8106</name>
+ <servlet_output>N</servlet_output>
+ <do_not_open_new_file_init>N</do_not_open_new_file_init>
+ <extention>txt</extention>
+ <append>N</append>
+ <split>N</split>
+ <haspartno>N</haspartno>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <SpecifyFormat>N</SpecifyFormat>
+ <date_time_format/>
+ <add_to_result_filenames>N</add_to_result_filenames>
+ <pad>N</pad>
+ <fast_dump>N</fast_dump>
+ <splitevery/>
+ </file>
+ <fields>
+ <field>
+ <name>marker</name>
+ <type>String</type>
+ <format/>
+ <currency/>
+ <decimal/>
+ <group/>
+ <nullif/>
+ <trim_type>none</trim_type>
+ <length>-1</length>
+ <precision>-1</precision>
+ </field>
+ </fields>
+ <attributes/>
+ <GUI>
+ <xloc>384</xloc>
+ <yloc>96</yloc>
+ </GUI>
+ </transform>
+ <transform_error_handling>
+ </transform_error_handling>
+ <attributes/>
+</pipeline>
diff --git a/integration-tests/hop_server/0014-write-to-export-only-vfs.hwf
b/integration-tests/hop_server/0014-write-to-export-only-vfs.hwf
new file mode 100644
index 0000000000..cc11c71780
--- /dev/null
+++ b/integration-tests/hop_server/0014-write-to-export-only-vfs.hwf
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>0014-write-to-export-only-vfs</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Runs entirely on the remote Hop Server, shipped there by the
"remote" run
+ configuration of the parent workflow. All it does is run the pipeline that
writes through the
+ "export-only-storage" named VFS connection; the parent verifies the result
from the client
+ side, where that connection is known to work.</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2024/01/01 00:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2024/01/01 00:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Write through the exported connection</name>
+ <description/>
+ <type>PIPELINE</type>
+ <attributes/>
+ <filename>${PROJECT_HOME}/0014-write-to-export-only-vfs.hpl</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <run_configuration>local</run_configuration>
+ <wait_until_finished>Y</wait_until_finished>
+ <create_parent_folder>N</create_parent_folder>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <set_append_logfile>N</set_append_logfile>
+ <parallel>N</parallel>
+ <xloc>250</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Write through the exported connection</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/hop_server/main-0014-test-export-only-vfs-connection.hwf
b/integration-tests/hop_server/main-0014-test-export-only-vfs-connection.hwf
new file mode 100644
index 0000000000..a270df2624
--- /dev/null
+++ b/integration-tests/hop_server/main-0014-test-export-only-vfs-connection.hwf
@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+-->
+<workflow>
+ <name>main-0014-test-export-only-vfs-connection</name>
+ <name_sync_with_filename>Y</name_sync_with_filename>
+ <description>Regression test for issue #8106: a named VFS connection carried
by an export must be
+ usable by the pipeline running on the remote Hop Server.
+
+ The "export-only-storage" Minio connection lives in the client project's
metadata-client-only
+ folder. The server registers ${PROJECT_HOME}/metadata as its own metadata
folder, so it never
+ sees that connection except inside the export ZIP. The inner workflow runs
remotely and writes
+ a file through it; the check below runs back on the client, where the
connection is known to
+ resolve, and proves the server really wrote to the object store.
+
+ Before the fix the server has no provider for the "export-only-storage"
scheme, so the path is
+ treated as a relative local file, nothing reaches Minio and this test
fails.</description>
+ <extended_description/>
+ <workflow_version/>
+ <created_user>-</created_user>
+ <created_date>2024/01/01 00:00:00.000</created_date>
+ <modified_user>-</modified_user>
+ <modified_date>2024/01/01 00:00:00.000</modified_date>
+ <parameters>
+ </parameters>
+ <actions>
+ <action>
+ <name>Start</name>
+ <description/>
+ <type>SPECIAL</type>
+ <attributes/>
+ <DayOfMonth>1</DayOfMonth>
+ <hour>12</hour>
+ <intervalMinutes>60</intervalMinutes>
+ <intervalSeconds>0</intervalSeconds>
+ <minutes>0</minutes>
+ <repeat>N</repeat>
+ <schedulerType>0</schedulerType>
+ <weekDay>1</weekDay>
+ <parallel>N</parallel>
+ <xloc>50</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Cleanup marker in object store</name>
+ <description>Runs on the client, which can reach the same Minio.
Removing the file first means
+ the check can only succeed on something the server wrote during this
run, and it keeps the
+ export from bundling an existing file in place of the connection
reference.</description>
+ <type>DELETE_FILE</type>
+ <attributes/>
+ <filename>export-only-storage:///demo/marker-8106.txt</filename>
+ <fail_if_file_not_exists>N</fail_if_file_not_exists>
+ <parallel>N</parallel>
+ <xloc>220</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Write on the server through the exported connection</name>
+ <description/>
+ <type>WORKFLOW</type>
+ <attributes/>
+ <run_configuration>remote</run_configuration>
+ <filename>${PROJECT_HOME}/0014-write-to-export-only-vfs.hwf</filename>
+ <params_from_previous>N</params_from_previous>
+ <exec_per_row>N</exec_per_row>
+ <set_logfile>N</set_logfile>
+ <logfile/>
+ <logext/>
+ <add_date>N</add_date>
+ <add_time>N</add_time>
+ <loglevel>Basic</loglevel>
+ <wait_until_finished>Y</wait_until_finished>
+ <create_parent_folder>N</create_parent_folder>
+ <parameters>
+ <pass_all_parameters>Y</pass_all_parameters>
+ </parameters>
+ <set_append_logfile>N</set_append_logfile>
+ <parallel>N</parallel>
+ <xloc>470</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Verify the server wrote to the object store</name>
+ <description/>
+ <type>FILE_EXISTS</type>
+ <attributes/>
+ <filename>export-only-storage:///demo/marker-8106.txt</filename>
+ <parallel>N</parallel>
+ <xloc>730</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ <action>
+ <name>Make sure the bucket exists</name>
+ <description>Runs on the client, through the same connection. The test
is about the server
+ being able to use a connection it only got from the export, not about
who creates the
+ bucket, so the folder is prepared here.</description>
+ <type>CREATE_FOLDER</type>
+ <attributes/>
+ <foldername>export-only-storage:///demo</foldername>
+ <parallel>N</parallel>
+ <xloc>350</xloc>
+ <yloc>50</yloc>
+ <attributes_hac/>
+ </action>
+ </actions>
+ <hops>
+ <hop>
+ <from>Start</from>
+ <to>Cleanup marker in object store</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Cleanup marker in object store</from>
+ <to>Make sure the bucket exists</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Make sure the bucket exists</from>
+ <to>Write on the server through the exported connection</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>Y</unconditional>
+ </hop>
+ <hop>
+ <from>Write on the server through the exported connection</from>
+ <to>Verify the server wrote to the object store</to>
+ <enabled>Y</enabled>
+ <evaluation>Y</evaluation>
+ <unconditional>N</unconditional>
+ </hop>
+ </hops>
+ <notepads>
+ </notepads>
+ <attributes/>
+</workflow>
diff --git
a/integration-tests/hop_server/metadata-client-only/MinioConnectionDefinition/export-only-storage.json
b/integration-tests/hop_server/metadata-client-only/MinioConnectionDefinition/export-only-storage.json
new file mode 100644
index 0000000000..2dc65d223a
--- /dev/null
+++
b/integration-tests/hop_server/metadata-client-only/MinioConnectionDefinition/export-only-storage.json
@@ -0,0 +1,12 @@
+{
+ "endPointHostname": "minio",
+ "virtualPath": "",
+ "secretKey": "ApacheHopSecretKey",
+ "accessKey": "ApacheHopAccessKey",
+ "partSize": "5242880",
+ "name": "export-only-storage",
+ "description": "A named VFS connection that exists ONLY in the client
project's metadata-client-only folder. The Hop Server registers
${PROJECT_HOME}/metadata as its metadata folder, so this connection can only
reach the server inside the export ZIP. Used by main-0014 to prove that VFS
connections carried by an export are usable on the server (issue #8106). Values
are literals on purpose: the test is about the connection reaching the server,
not about variable resolution.",
+ "endPointPort": "9000",
+ "endPointSecure": false,
+ "region": "local"
+}
diff --git
a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java
b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java
index c47b7c2347..77ecf6d69a 100644
---
a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java
+++
b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java
@@ -329,8 +329,10 @@ public class ProjectsGuiPlugin {
HopExtensionPoint.HopGuiProjectAfterEnabled.name(),
project);
- // Reset VFS filesystem to load additional configurations
- HopVfs.reset();
+ // The project that just opened has its own VFS connections: take its
namespace, and let go
+ // of the one of the project we came from. Not a full reset - in Hop Web
that would empty the
+ // file system manager of every other session as well.
+ hopGui.useVfsNamespaceOfOpenProject();
// Finally, warn about metadata elements in this project which we can't
load.
// They're ignored so the project itself opens just fine.
diff --git
a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/S3VfsPlugin.java
b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/S3VfsPlugin.java
index a886b8bb70..63d8fb76d7 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/S3VfsPlugin.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/S3VfsPlugin.java
@@ -43,10 +43,18 @@ public class S3VfsPlugin implements IVfs {
@Override
public Map<String, FileProvider> getProviders(IVariables variables) {
+ return getProviders(variables, null);
+ }
+
+ @Override
+ public Map<String, FileProvider> getProviders(
+ IVariables variables, IHopMetadataProvider executionMetadata) {
Map<String, FileProvider> providers = new HashMap<>();
try {
IHopMetadataProvider metadataProvider =
- HopMetadataUtil.getStandardHopMetadataProvider(variables);
+ executionMetadata != null
+ ? executionMetadata
+ : HopMetadataUtil.getStandardHopMetadataProvider(variables);
List<S3Meta> s3MetaList =
metadataProvider.getSerializer(S3Meta.class).loadAll();
for (S3Meta s3Meta : s3MetaList) {
providers.put(s3Meta.getName(), new S3FileProvider(variables, s3Meta));
diff --git
a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3Meta.java
b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3Meta.java
index 3f63f15b0c..f729072d97 100644
--- a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3Meta.java
+++ b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3Meta.java
@@ -37,7 +37,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
image = "s3.svg",
category = HopMetadataCategory.FILE_STORAGE,
documentationUrl = "/metadata-types/s3-connection.html",
- hopMetadataPropertyType = HopMetadataPropertyType.VFS_S3_CONNECTION)
+ hopMetadataPropertyType = HopMetadataPropertyType.VFS_S3_CONNECTION,
+ classLoaderGroup = "vfs-s3")
public class S3Meta extends HopMetadataBase implements Serializable,
IHopMetadata {
@HopMetadataProperty private String description;
diff --git
a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3MetaEditor.java
b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3MetaEditor.java
index b530a18a7e..800cb10973 100644
---
a/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3MetaEditor.java
+++
b/plugins/tech/aws/src/main/java/org/apache/hop/vfs/s3/metadata/S3MetaEditor.java
@@ -576,6 +576,6 @@ public class S3MetaEditor extends MetadataEditor<S3Meta> {
@Override
public void save() throws HopException {
super.save();
- HopVfs.reset();
+ HopVfs.refresh(hopGui.getVariables());
}
}
diff --git
a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureVfsPlugin.java
b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureVfsPlugin.java
index 04c31950a6..00a943c5af 100644
---
a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureVfsPlugin.java
+++
b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/AzureVfsPlugin.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.vfs2.provider.FileProvider;
+import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.vfs.plugin.IVfs;
import org.apache.hop.core.vfs.plugin.VfsPlugin;
@@ -43,10 +44,18 @@ public class AzureVfsPlugin implements IVfs {
@Override
public Map<String, FileProvider> getProviders(IVariables variables) {
+ return getProviders(variables, null);
+ }
+
+ @Override
+ public Map<String, FileProvider> getProviders(
+ IVariables variables, IHopMetadataProvider executionMetadata) {
Map<String, FileProvider> providers = new HashMap<>();
try {
IHopMetadataProvider metadataProvider =
- HopMetadataUtil.getStandardHopMetadataProvider(variables);
+ executionMetadata != null
+ ? executionMetadata
+ : HopMetadataUtil.getStandardHopMetadataProvider(variables);
List<AzureMetadataType> azureMetadataTypes =
metadataProvider.getSerializer(AzureMetadataType.class).loadAll();
for (AzureMetadataType azureMetadataType : azureMetadataTypes) {
@@ -54,7 +63,7 @@ public class AzureVfsPlugin implements IVfs {
azureMetadataType.getName(), new AzureFileProvider(variables,
azureMetadataType));
}
} catch (Exception e) {
- e.printStackTrace();
+ LogChannel.GENERAL.logError("Unable to load the Azure VFS providers", e);
}
return providers;
}
diff --git
a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataType.java
b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataType.java
index e6c82ee407..ae278fe3b4 100644
---
a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataType.java
+++
b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataType.java
@@ -33,7 +33,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
image = "ui/images/authentication.svg",
category = HopMetadataCategory.FILE_STORAGE,
documentationUrl = "/metadata-types/azure-authentication.html",
- hopMetadataPropertyType = HopMetadataPropertyType.VFS_AZURE_CONNECTION)
+ hopMetadataPropertyType = HopMetadataPropertyType.VFS_AZURE_CONNECTION,
+ classLoaderGroup = "vfs-azure")
@Getter
@Setter
public class AzureMetadataType extends HopMetadataBase implements
Serializable, IHopMetadata {
diff --git
a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataTypeEditor.java
b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataTypeEditor.java
index f0019bbbc0..46a9b0094a 100644
---
a/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataTypeEditor.java
+++
b/plugins/tech/azure/src/main/java/org/apache/hop/vfs/azure/metadatatype/AzureMetadataTypeEditor.java
@@ -301,6 +301,6 @@ public class AzureMetadataTypeEditor extends
MetadataEditor<AzureMetadataType> {
@Override
public void save() throws HopException {
super.save();
- HopVfs.reset();
+ HopVfs.refresh(hopGui.getVariables());
}
}
diff --git
a/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java
b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java
index 12418bf055..0836f89991 100644
---
a/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java
+++
b/plugins/tech/databricks/src/main/java/org/apache/hop/databricks/metadata/DatabricksConnection.java
@@ -44,7 +44,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
description = "i18n::DatabricksConnection.Description",
image = "databricks-connection.svg",
category = HopMetadataCategory.CONNECTIONS,
- documentationUrl = "/metadata-types/databricks-connection.html")
+ documentationUrl = "/metadata-types/databricks-connection.html",
+ classLoaderGroup = "vfs-databricks")
public class DatabricksConnection extends HopMetadataBase implements
Serializable, IHopMetadata {
private static final String PARENT =
DatabricksConnectionEditor.GUI_WIDGETS_PARENT_ID;
diff --git
a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksVfsPlugin.java
b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksVfsPlugin.java
index 2df9328b9d..4987ad5867 100644
---
a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksVfsPlugin.java
+++
b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/DatabricksVfsPlugin.java
@@ -55,11 +55,18 @@ public class DatabricksVfsPlugin implements IVfs {
@Override
public Map<String, FileProvider> getProviders(IVariables variables) {
+ return getProviders(variables, null);
+ }
+
+ @Override
+ public Map<String, FileProvider> getProviders(
+ IVariables variables, IHopMetadataProvider executionMetadata) {
Map<String, FileProvider> providers = new HashMap<>();
try {
- // Prefer the active runtime metadata (GUI project / MainSpark export).
Standard folders
- // alone are empty on Databricks (no hop-config metadata tree).
- IHopMetadataProvider metadataProvider =
HopMetadataInstance.getMetadataProvider();
+ IHopMetadataProvider metadataProvider = executionMetadata;
+ if (metadataProvider == null) {
+ metadataProvider = HopMetadataInstance.getMetadataProvider();
+ }
if (metadataProvider == null) {
metadataProvider =
HopMetadataUtil.getStandardHopMetadataProvider(variables);
}
diff --git
a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnection.java
b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnection.java
index 382cc33c98..7815fe572a 100644
---
a/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnection.java
+++
b/plugins/tech/databricks/src/main/java/org/apache/hop/vfs/databricks/metadata/DatabricksVfsConnection.java
@@ -46,7 +46,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
image = "databricks-connection.svg",
category = HopMetadataCategory.FILE_STORAGE,
documentationUrl = "/metadata-types/databricks-vfs-connection.html",
- hopMetadataPropertyType =
HopMetadataPropertyType.VFS_DATABRICKS_CONNECTION)
+ hopMetadataPropertyType =
HopMetadataPropertyType.VFS_DATABRICKS_CONNECTION,
+ classLoaderGroup = "vfs-databricks")
public class DatabricksVfsConnection extends HopMetadataBase implements
Serializable, IHopMetadata {
private static final String PARENT =
DatabricksVfsConnectionEditor.GUI_WIDGETS_PARENT_ID;
diff --git
a/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/FtpVfsPlugin.java
b/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/FtpVfsPlugin.java
index 6aabd18c11..a51d6cc082 100644
--- a/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/FtpVfsPlugin.java
+++ b/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/FtpVfsPlugin.java
@@ -59,10 +59,18 @@ public class FtpVfsPlugin implements IVfs {
@Override
public Map<String, FileProvider> getProviders(IVariables variables) {
+ return getProviders(variables, null);
+ }
+
+ @Override
+ public Map<String, FileProvider> getProviders(
+ IVariables variables, IHopMetadataProvider executionMetadata) {
Map<String, FileProvider> providers = new HashMap<>();
try {
IHopMetadataProvider metadataProvider =
- HopMetadataUtil.getStandardHopMetadataProvider(variables);
+ executionMetadata != null
+ ? executionMetadata
+ : HopMetadataUtil.getStandardHopMetadataProvider(variables);
List<FtpConnection> connections =
metadataProvider.getSerializer(FtpConnection.class).loadAll();
for (FtpConnection connection : connections) {
diff --git
a/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/metadata/FtpConnection.java
b/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/metadata/FtpConnection.java
index b65ad4429b..32dcc07214 100644
---
a/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/metadata/FtpConnection.java
+++
b/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/metadata/FtpConnection.java
@@ -47,7 +47,8 @@ import org.apache.hop.vfs.ftp.IFtpConnection;
category = HopMetadataCategory.FILE_STORAGE,
documentationUrl = "/metadata-types/ftp-connection.html",
hopMetadataPropertyType = HopMetadataPropertyType.VFS_FTP_CONNECTION,
- supportsGlobalReplace = true)
+ supportsGlobalReplace = true,
+ classLoaderGroup = "sftp")
public class FtpConnection extends HopMetadataBase
implements Serializable, IHopMetadata, IFtpConnection {
diff --git
a/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/metadata/FtpConnectionEditor.java
b/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/metadata/FtpConnectionEditor.java
index de110a9927..f8604f56d6 100644
---
a/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/metadata/FtpConnectionEditor.java
+++
b/plugins/tech/ftp/src/main/java/org/apache/hop/vfs/ftp/metadata/FtpConnectionEditor.java
@@ -665,6 +665,6 @@ public class FtpConnectionEditor extends
MetadataEditor<FtpConnection> {
// The name of a connection is a VFS scheme: re-register the providers so
the new or changed
// connection is picked up right away.
//
- HopVfs.reset();
+ HopVfs.refresh(hopGui.getVariables());
}
}
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageVfsPlugin.java
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageVfsPlugin.java
index 47e36aa00a..934fe9026e 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageVfsPlugin.java
+++
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/GoogleStorageVfsPlugin.java
@@ -43,10 +43,18 @@ public class GoogleStorageVfsPlugin implements IVfs {
@Override
public Map<String, FileProvider> getProviders(IVariables variables) {
+ return getProviders(variables, null);
+ }
+
+ @Override
+ public Map<String, FileProvider> getProviders(
+ IVariables variables, IHopMetadataProvider executionMetadata) {
Map<String, FileProvider> providers = new HashMap<>();
try {
IHopMetadataProvider metadataProvider =
- HopMetadataUtil.getStandardHopMetadataProvider(variables);
+ executionMetadata != null
+ ? executionMetadata
+ : HopMetadataUtil.getStandardHopMetadataProvider(variables);
List<GoogleStorageMetadataType> googleStorageMetadataTypes =
metadataProvider.getSerializer(GoogleStorageMetadataType.class).loadAll();
for (GoogleStorageMetadataType googleStorageMetadataType :
googleStorageMetadataTypes) {
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
index e60c2e048d..7499d73cd2 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
+++
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataType.java
@@ -33,7 +33,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
image = "ui/images/authentication.svg",
category = HopMetadataCategory.FILE_STORAGE,
documentationUrl = "/metadata-types/google-storage-authentication.html",
- hopMetadataPropertyType = HopMetadataPropertyType.VFS_GCP_CONNECTION)
+ hopMetadataPropertyType = HopMetadataPropertyType.VFS_GCP_CONNECTION,
+ classLoaderGroup = "vfs-google")
@Getter
@Setter
public class GoogleStorageMetadataType extends HopMetadataBase
diff --git
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataTypeEditor.java
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataTypeEditor.java
index 1197020e92..dc4bfbb513 100644
---
a/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataTypeEditor.java
+++
b/plugins/tech/google/src/main/java/org/apache/hop/vfs/gs/metadatatype/GoogleStorageMetadataTypeEditor.java
@@ -189,6 +189,6 @@ public class GoogleStorageMetadataTypeEditor extends
MetadataEditor<GoogleStorag
@Override
public void save() throws HopException {
super.save();
- HopVfs.reset();
+ HopVfs.refresh(hopGui.getVariables());
}
}
diff --git
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioVfsPlugin.java
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioVfsPlugin.java
index 573437a110..2acb0eacd1 100644
---
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioVfsPlugin.java
+++
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/MinioVfsPlugin.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.vfs2.provider.FileProvider;
+import org.apache.hop.core.logging.LogChannel;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.vfs.plugin.IVfs;
import org.apache.hop.core.vfs.plugin.VfsPlugin;
@@ -29,7 +30,7 @@ import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.util.HopMetadataUtil;
import org.apache.hop.vfs.minio.metadata.MinioMeta;
-@VfsPlugin(type = "minio", typeDescription = "S3 VFS plugin", classLoaderGroup
= "vfs-s3")
+@VfsPlugin(type = "minio", typeDescription = "Minio VFS plugin",
classLoaderGroup = "vfs-minio")
public class MinioVfsPlugin implements IVfs {
@Override
public String[] getUrlSchemes() {
@@ -44,16 +45,26 @@ public class MinioVfsPlugin implements IVfs {
@Override
public Map<String, FileProvider> getProviders(IVariables variables) {
+ return getProviders(variables, null);
+ }
+
+ @Override
+ public Map<String, FileProvider> getProviders(
+ IVariables variables, IHopMetadataProvider executionMetadata) {
Map<String, FileProvider> providers = new HashMap<>();
try {
IHopMetadataProvider metadataProvider =
- HopMetadataUtil.getStandardHopMetadataProvider(variables);
+ executionMetadata != null
+ ? executionMetadata
+ : HopMetadataUtil.getStandardHopMetadataProvider(variables);
List<MinioMeta> minioMetaTypes =
metadataProvider.getSerializer(MinioMeta.class).loadAll();
for (MinioMeta minioMeta : minioMetaTypes) {
providers.put(minioMeta.getName(), new MinioFileProvider(variables,
minioMeta));
}
} catch (Exception e) {
- // Ignore errors
+ // Never silently: an unreadable connection here means files resolved
through its scheme
+ // quietly land on the local disk instead of the object store.
+ LogChannel.GENERAL.logError("Unable to load the Minio VFS providers", e);
}
return providers;
}
diff --git
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMeta.java
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMeta.java
index 9be104c995..687be7e45a 100644
---
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMeta.java
+++
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMeta.java
@@ -39,7 +39,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
image = "minio.svg",
category = HopMetadataCategory.FILE_STORAGE,
documentationUrl = "/metadata-types/minio-connection.html",
- hopMetadataPropertyType = HopMetadataPropertyType.VFS_MINIO_CONNECTION)
+ hopMetadataPropertyType = HopMetadataPropertyType.VFS_MINIO_CONNECTION,
+ classLoaderGroup = "vfs-minio")
public class MinioMeta extends HopMetadataBase implements Serializable,
IHopMetadata {
private static final String WIDGET_ID_MINIO_DESCRIPTION =
"10000-minio-description";
private static final String WIDGET_ID_MINIO_ACCESS_KEY =
"10050-minio-access-key";
diff --git
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMetaEditor.java
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMetaEditor.java
index 667254c79f..8807d777e9 100644
---
a/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMetaEditor.java
+++
b/plugins/tech/minio/src/main/java/org/apache/hop/vfs/minio/metadata/MinioMetaEditor.java
@@ -154,6 +154,6 @@ public class MinioMetaEditor extends
MetadataEditor<MinioMeta> {
@Override
public void save() throws HopException {
super.save();
- HopVfs.reset();
+ HopVfs.refresh(hopGui.getVariables());
}
}
diff --git
a/plugins/tech/minio/src/test/java/org/apache/hop/vfs/minio/MinioVfsPluginTest.java
b/plugins/tech/minio/src/test/java/org/apache/hop/vfs/minio/MinioVfsPluginTest.java
index 4544f8f748..df28f92690 100644
---
a/plugins/tech/minio/src/test/java/org/apache/hop/vfs/minio/MinioVfsPluginTest.java
+++
b/plugins/tech/minio/src/test/java/org/apache/hop/vfs/minio/MinioVfsPluginTest.java
@@ -27,6 +27,7 @@ import org.apache.commons.vfs2.provider.FileProvider;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
import org.apache.hop.junit.rules.RestoreHopEnvironmentExtension;
+import org.apache.hop.vfs.minio.metadata.MinioMeta;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -98,7 +99,27 @@ class MinioVfsPluginTest {
assertNotNull(annotation, "VfsPlugin annotation should not be null");
assertEquals("minio", annotation.type(), "Plugin type should be 'minio'");
- assertEquals("S3 VFS plugin", annotation.typeDescription(), "Type
description should match");
+ assertEquals("Minio VFS plugin", annotation.typeDescription(), "Type
description should match");
+ }
+
+ @Test
+ void connectionMetadataSharesTheClassLoaderGroupOfThisPlugin() {
+ // The plugin reads MinioMeta objects it did not deserialize itself when
the connections come
+ // from a resource export.
+ String pluginGroup =
+ plugin
+ .getClass()
+ .getAnnotation(org.apache.hop.core.vfs.plugin.VfsPlugin.class)
+ .classLoaderGroup();
+ String metadataGroup =
+ MinioMeta.class
+ .getAnnotation(org.apache.hop.metadata.api.HopMetadata.class)
+ .classLoaderGroup();
+
+ assertEquals(
+ pluginGroup,
+ metadataGroup,
+ "MinioMeta and the Minio VFS plugin must share a class loader group");
}
@Test
diff --git
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpVfsPlugin.java
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpVfsPlugin.java
index 43743b42f7..5728cf7006 100644
--- a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpVfsPlugin.java
+++ b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpVfsPlugin.java
@@ -57,10 +57,18 @@ public class SftpVfsPlugin implements IVfs {
@Override
public Map<String, FileProvider> getProviders(IVariables variables) {
+ return getProviders(variables, null);
+ }
+
+ @Override
+ public Map<String, FileProvider> getProviders(
+ IVariables variables, IHopMetadataProvider executionMetadata) {
Map<String, FileProvider> providers = new HashMap<>();
try {
IHopMetadataProvider metadataProvider =
- HopMetadataUtil.getStandardHopMetadataProvider(variables);
+ executionMetadata != null
+ ? executionMetadata
+ : HopMetadataUtil.getStandardHopMetadataProvider(variables);
List<SftpConnection> connections =
metadataProvider.getSerializer(SftpConnection.class).loadAll();
for (SftpConnection connection : connections) {
diff --git
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/metadata/SftpConnection.java
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/metadata/SftpConnection.java
index 58655fdc68..f840c31708 100644
---
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/metadata/SftpConnection.java
+++
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/metadata/SftpConnection.java
@@ -44,7 +44,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
category = HopMetadataCategory.FILE_STORAGE,
documentationUrl = "/metadata-types/sftp-connection.html",
hopMetadataPropertyType = HopMetadataPropertyType.VFS_SFTP_CONNECTION,
- supportsGlobalReplace = true)
+ supportsGlobalReplace = true,
+ classLoaderGroup = "sftp")
public class SftpConnection extends HopMetadataBase implements Serializable,
IHopMetadata {
public static final int DEFAULT_PORT = 22;
diff --git
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/metadata/SftpConnectionEditor.java
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/metadata/SftpConnectionEditor.java
index 8ff3f1449f..b5d5809dd0 100644
---
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/metadata/SftpConnectionEditor.java
+++
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/metadata/SftpConnectionEditor.java
@@ -486,6 +486,6 @@ public class SftpConnectionEditor extends
MetadataEditor<SftpConnection> {
// The name of a connection is a VFS scheme: re-register the providers so
the new or changed
// connection is picked up right away.
//
- HopVfs.reset();
+ HopVfs.refresh(hopGui.getVariables());
}
}
diff --git
a/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/WebDavVfsPlugin.java
b/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/WebDavVfsPlugin.java
index 5f6d5a7017..64cf36d9f4 100644
---
a/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/WebDavVfsPlugin.java
+++
b/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/WebDavVfsPlugin.java
@@ -48,10 +48,18 @@ public class WebDavVfsPlugin implements IVfs {
@Override
public Map<String, FileProvider> getProviders(IVariables variables) {
+ return getProviders(variables, null);
+ }
+
+ @Override
+ public Map<String, FileProvider> getProviders(
+ IVariables variables, IHopMetadataProvider executionMetadata) {
Map<String, FileProvider> providers = new HashMap<>();
try {
IHopMetadataProvider metadataProvider =
- HopMetadataUtil.getStandardHopMetadataProvider(variables);
+ executionMetadata != null
+ ? executionMetadata
+ : HopMetadataUtil.getStandardHopMetadataProvider(variables);
List<WebDavConnection> connections =
metadataProvider.getSerializer(WebDavConnection.class).loadAll();
for (WebDavConnection connection : connections) {
diff --git
a/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnection.java
b/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnection.java
index 69b3461f6d..0c02264eab 100644
---
a/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnection.java
+++
b/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnection.java
@@ -41,7 +41,8 @@ import org.apache.hop.metadata.api.IHopMetadata;
image = "ui/images/authentication.svg",
category = HopMetadataCategory.FILE_STORAGE,
documentationUrl = "/metadata-types/webdav-connection.html",
- hopMetadataPropertyType = HopMetadataPropertyType.VFS_WEBDAV_CONNECTION)
+ hopMetadataPropertyType = HopMetadataPropertyType.VFS_WEBDAV_CONNECTION,
+ classLoaderGroup = "vfs-webdav")
public class WebDavConnection extends HopMetadataBase implements Serializable,
IHopMetadata {
@HopMetadataProperty private String description;
diff --git
a/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnectionEditor.java
b/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnectionEditor.java
index 6f6c9d6fbc..3f74627921 100644
---
a/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnectionEditor.java
+++
b/plugins/tech/webdav/src/main/java/org/apache/hop/vfs/webdav/metadata/WebDavConnectionEditor.java
@@ -228,6 +228,6 @@ public class WebDavConnectionEditor extends
MetadataEditor<WebDavConnection> {
@Override
public void save() throws HopException {
super.save();
- HopVfs.reset();
+ HopVfs.refresh(hopGui.getVariables());
}
}
diff --git a/rap/src/main/java/org/apache/hop/ui/hopgui/HopWeb.java
b/rap/src/main/java/org/apache/hop/ui/hopgui/HopWeb.java
index 7dc75d8ef6..eb3ec1c19c 100644
--- a/rap/src/main/java/org/apache/hop/ui/hopgui/HopWeb.java
+++ b/rap/src/main/java/org/apache/hop/ui/hopgui/HopWeb.java
@@ -45,8 +45,10 @@ import org.apache.hop.core.plugins.PluginRegistry;
import org.apache.hop.core.svg.SvgCache;
import org.apache.hop.core.svg.SvgCacheEntry;
import org.apache.hop.core.svg.SvgFile;
+import org.apache.hop.core.vfs.HopVfsNamespaces;
import org.apache.hop.core.xml.XmlHandler;
import org.apache.hop.metadata.plugin.MetadataPluginType;
+import org.apache.hop.metadata.util.HopMetadataInstance;
import org.apache.hop.ui.hopgui.canvas.CanvasRenderServiceHandler;
import org.apache.hop.ui.hopgui.perspective.HopPerspectivePluginType;
import org.eclipse.rap.rwt.application.Application;
@@ -79,6 +81,10 @@ public class HopWeb implements ApplicationConfiguration {
@Override
public void configure(Application application) {
+ HopMetadataInstance.setScope(
+ new RapSessionScope<>(RapSessionScope.MetadataProviderHolder.class));
+ HopVfsNamespaces.setScope(new
RapSessionScope<>(RapSessionScope.VfsNamespaceHolder.class));
+
try {
// Hop initialization is already done here.
// This means we can simply ask the gui registry for the toolbar images
to register.
diff --git a/rap/src/main/java/org/apache/hop/ui/hopgui/HopWebEntryPoint.java
b/rap/src/main/java/org/apache/hop/ui/hopgui/HopWebEntryPoint.java
index ce87d5e55c..b61e7ecec1 100644
--- a/rap/src/main/java/org/apache/hop/ui/hopgui/HopWebEntryPoint.java
+++ b/rap/src/main/java/org/apache/hop/ui/hopgui/HopWebEntryPoint.java
@@ -263,7 +263,13 @@ public class HopWebEntryPoint extends AbstractEntryPoint {
public void beforeDestroy(UISessionEvent event) {
try {
HopGui hopGui = HopGui.getInstance();
- if (hopGui == null || hopGui.auditDelegate == null) {
+ if (hopGui == null) {
+ return;
+ }
+ // Let go of this session's VFS namespace: it closes once
nothing is using it,
+ // and the sessions still running keep theirs.
+ hopGui.releaseVfsNamespace();
+ if (hopGui.auditDelegate == null) {
return;
}
if (hopGui.getShell() != null &&
hopGui.getShell().isDisposed()) {
diff --git a/rap/src/main/java/org/apache/hop/ui/hopgui/RapSessionScope.java
b/rap/src/main/java/org/apache/hop/ui/hopgui/RapSessionScope.java
new file mode 100644
index 0000000000..15ba916158
--- /dev/null
+++ b/rap/src/main/java/org/apache/hop/ui/hopgui/RapSessionScope.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.hopgui;
+
+import org.apache.hop.core.scope.IHopScope;
+import org.eclipse.rap.rwt.SingletonUtil;
+
+/**
+ * Holds one piece of Hop's ambient state per RAP session, so that one user's
project does not
+ * become another's. This is what {@link
org.apache.hop.metadata.util.HopMetadataInstance} and
+ * {@link org.apache.hop.core.vfs.HopVfsNamespaces} keep their "current" value
in when Hop runs as
+ * Hop Web.
+ *
+ * <p>Not every thread that asks has a RAP session: an execution runs on
threads Hop started itself,
+ * and those have no request and no {@code UISession}. So the value is kept
twice - in the session
+ * for whoever is serving a request, and in a thread local inherited by
threads an execution starts.
+ * A pipeline launched from the GUI therefore keeps resolving in the session
that launched it, while
+ * a second session serving a request alongside it gets its own.
+ *
+ * <p>One instance per kind of state, each with its own holder class: {@link
SingletonUtil} keys the
+ * per session instance on that class.
+ *
+ * @param <T> the type of the value held
+ */
+public class RapSessionScope<T> implements IHopScope<T> {
+
+ /** Kept per RAP session by {@link SingletonUtil}. One subclass per kind of
state. */
+ public static class Holder {
+ Object value;
+ }
+
+ /** The session slot for the metadata provider in use. */
+ public static class MetadataProviderHolder extends Holder {}
+
+ /** The session slot for the VFS namespace files are resolved in. */
+ public static class VfsNamespaceHolder extends Holder {}
+
+ private final Class<? extends Holder> holderType;
+
+ /** For the threads of an execution, which have no session to ask. */
+ private final InheritableThreadLocal<T> outsideASession = new
InheritableThreadLocal<>();
+
+ public RapSessionScope(Class<? extends Holder> holderType) {
+ this.holderType = holderType;
+ }
+
+ @Override
+ public T get() {
+ return valueOf(holder());
+ }
+
+ /**
+ * The answer for a given session slot. Kept apart from {@link #get()} so it
can be exercised
+ * without a RAP session, which is the only way to reach the case where a
session exists but has
+ * nothing bound.
+ *
+ * @param holder the session's slot, or null when this thread is serving no
session
+ * @return the value in scope
+ */
+ @SuppressWarnings("unchecked")
+ T valueOf(Holder holder) {
+ if (holder != null) {
+ // Whether this session has anything bound or not, the answer is its
own. Falling back to the
+ // thread here would hand it whatever the previous session on this
pooled request thread
+ // left behind.
+ return (T) holder.value;
+ }
+ return outsideASession.get();
+ }
+
+ @Override
+ public void set(T value) {
+ Holder holder = holder();
+ if (holder != null) {
+ holder.value = value;
+ }
+ // Also outside the session: an execution started from here runs on its
own threads, and they
+ // have to keep resolving the way the session that launched them does.
+ outsideASession.set(value);
+ }
+
+ @Override
+ public void remove() {
+ Holder holder = holder();
+ if (holder != null) {
+ holder.value = null;
+ }
+ outsideASession.remove();
+ }
+
+ /** The session's holder, or null when this thread is not serving a session.
*/
+ private Holder holder() {
+ try {
+ return SingletonUtil.getSessionInstance(holderType);
+ } catch (Exception e) {
+ // No UISession on this thread: an execution thread, or a shutdown hook.
+ return null;
+ }
+ }
+}
diff --git
a/rap/src/test/java/org/apache/hop/ui/hopgui/RapSessionScopeTest.java
b/rap/src/test/java/org/apache/hop/ui/hopgui/RapSessionScopeTest.java
new file mode 100644
index 0000000000..5ab823435a
--- /dev/null
+++ b/rap/src/test/java/org/apache/hop/ui/hopgui/RapSessionScopeTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.hopgui;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Hop Web serves many people from one JVM on a pool of request threads, so
the same thread serves
+ * one session and then another. What a session sees has to be its own, and
only a thread with no
+ * session at all - an execution thread - may fall back to what it inherited.
+ */
+class RapSessionScopeTest {
+
+ private RapSessionScope<String> scope() {
+ return new RapSessionScope<>(RapSessionScope.VfsNamespaceHolder.class);
+ }
+
+ @Test
+ @DisplayName("A session with nothing bound gets nothing, not the previous
session's value")
+ void aSessionNeverInheritsFromThePooledThread() {
+ RapSessionScope<String> scope = scope();
+ RapSessionScope.Holder sessionA = new RapSessionScope.VfsNamespaceHolder();
+ RapSessionScope.Holder sessionB = new RapSessionScope.VfsNamespaceHolder();
+
+ // Session A is served on this thread and binds something. set() also
writes the thread local,
+ // so that executions it starts inherit it.
+ sessionA.value = "namespace of A";
+ scope.set("namespace of A");
+
+ assertEquals("namespace of A", scope.valueOf(sessionA));
+ assertNull(
+ scope.valueOf(sessionB),
+ "Session B has opened nothing and must not be handed what session A
left on this thread");
+ }
+
+ @Test
+ @DisplayName("A thread with no session at all falls back to what it
inherited")
+ void anExecutionThreadFallsBackToTheThread() {
+ RapSessionScope<String> scope = scope();
+ scope.set("namespace of the session that started this execution");
+
+ assertEquals(
+ "namespace of the session that started this execution",
+ scope.valueOf(null),
+ "An execution runs on threads with no UISession and has to keep
resolving as its session does");
+ }
+}
diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/HopGui.java
b/ui/src/main/java/org/apache/hop/ui/hopgui/HopGui.java
index 3e76385473..cbda06fa44 100644
--- a/ui/src/main/java/org/apache/hop/ui/hopgui/HopGui.java
+++ b/ui/src/main/java/org/apache/hop/ui/hopgui/HopGui.java
@@ -81,9 +81,13 @@ import org.apache.hop.core.util.TranslateUtil;
import org.apache.hop.core.variables.DescribedVariable;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.core.variables.Variables;
+import org.apache.hop.core.vfs.HopVfs;
+import org.apache.hop.core.vfs.HopVfsNamespace;
+import org.apache.hop.core.vfs.HopVfsNamespaces;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.i18n.LanguageChoice;
import org.apache.hop.metadata.api.IHasHopMetadataProvider;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
import org.apache.hop.metadata.util.HopMetadataInstance;
import org.apache.hop.metadata.util.HopMetadataUtil;
@@ -378,7 +382,7 @@ public class HopGui
this.id = UUID.randomUUID().toString();
commandLineArguments = new ArrayList<>();
- variables = Variables.getADefaultVariableSpace();
+ setVariables(Variables.getADefaultVariableSpace());
loggingObject = new LoggingObject(APP_NAME);
log = new LogChannel(APP_NAME);
@@ -420,6 +424,77 @@ public class HopGui
PROVIDER = (ISingletonProvider)
ImplementationLoader.newInstance(HopGui.class);
}
+ /**
+ * Sits behind this GUI's variables and answers with the metadata of the
project it has open, so
+ * that everything resolving a file through those variables lands in the
right VFS namespace. In
+ * Hop Web every session has its own HopGui, so this is also what keeps one
user's named VFS
+ * connections apart from another's. See issue #8106.
+ *
+ * <p>It is reached by walking the parent chain, never for looking a
variable value up - {@link
+ * Variables#getVariable(String)} reads its own properties only - so
variable inheritance is
+ * untouched.
+ */
+ private final IVariables metadataAnchor =
+ new Variables() {
+ @Override
+ public IHopMetadataProvider getMetadataProvider() {
+ return HopGui.this.metadataProvider;
+ }
+ };
+
+ /** The metadata this GUI took a VFS namespace for, so it can let go of it
again. */
+ private IHopMetadataProvider vfsNamespaceProvider;
+
+ /**
+ * Take the VFS namespace of the project this GUI now has open, and let go
of the previous one.
+ *
+ * <p>Only does anything where tenants share the JVM: in Hop Web this is
what gives each session
+ * its own named VFS connections, so two people can both have a connection
called {@code mydata}
+ * pointing somewhere different. On the desktop there is one project in the
process and the
+ * process wide file system manager already holds its connections, so this
reads them again
+ * instead. See Apache Hop issue #8106.
+ */
+ public void useVfsNamespaceOfOpenProject() {
+ IHopMetadataProvider previousProvider = vfsNamespaceProvider;
+ HopVfsNamespace namespace =
+ HopVfsNamespaces.acquire(variables, metadataProvider, "Hop GUI " + id);
+ vfsNamespaceProvider = namespace == null ? null : metadataProvider;
+
+ // Bind it for this session, so the call sites that resolve a file with no
variables in hand
+ // land in it too. Deliberately not restored afterwards: it stays for as
long as the session.
+ HopVfsNamespaces.bindThread(namespace);
+
+ if (previousProvider != null) {
+ HopVfsNamespaces.release(previousProvider);
+ }
+ if (namespace == null) {
+ HopVfs.refresh(variables);
+ }
+ }
+
+ /** Let go of the VFS namespace of this GUI, when its session ends. */
+ public void releaseVfsNamespace() {
+ if (vfsNamespaceProvider != null) {
+ HopVfsNamespaces.release(vfsNamespaceProvider);
+ vfsNamespaceProvider = null;
+ }
+ // Put back nothing: this session is going away.
+ HopVfsNamespaces.restoreThread(null);
+ }
+
+ /**
+ * Give this GUI a new set of variables, as loading a project does. The
metadata of the open
+ * project stays reachable from them either way.
+ *
+ * @param variables the variables to use
+ */
+ public void setVariables(IVariables variables) {
+ this.variables = variables;
+ if (variables != null) {
+ variables.setParentVariables(metadataAnchor);
+ }
+ }
+
public static HopGui getInstance() {
return (HopGui) PROVIDER.getInstanceInternal();
}
