This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 9875b48005 minor improvements to vfs (test & docs), fixes #7148 (#7149)
9875b48005 is described below
commit 9875b480050c4711284825486f190b1a6473a58f
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Wed May 20 21:55:47 2026 +0200
minor improvements to vfs (test & docs), fixes #7148 (#7149)
---
core/pom.xml | 21 ++
.../main/java/org/apache/hop/core/vfs/HopVfs.java | 5 +-
.../hop/core/vfs/HopVfsNetworkProvidersTest.java | 346 +++++++++++++++++++++
.../apache/hop/core/vfs/HopVfsProvidersTest.java | 257 +++++++++++++++
docs/hop-user-manual/modules/ROOT/pages/vfs.adoc | 90 +++---
5 files changed, 664 insertions(+), 55 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index afe26470d8..2919637317 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -462,6 +462,27 @@
<scope>test</scope>
</dependency>
+ <!-- Embedded servers used by HopVfsNetworkProvidersTest to verify the
FTP/FTPS/SFTP
+ VFS providers actually exchange bytes. Test scope only. -->
+ <dependency>
+ <groupId>org.apache.ftpserver</groupId>
+ <artifactId>ftpserver-core</artifactId>
+ <version>1.2.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.sshd</groupId>
+ <artifactId>sshd-core</artifactId>
+ <version>2.16.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.sshd</groupId>
+ <artifactId>sshd-sftp</artifactId>
+ <version>2.16.0</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.javassist</groupId>
<artifactId>javassist</artifactId>
diff --git a/core/src/main/java/org/apache/hop/core/vfs/HopVfs.java
b/core/src/main/java/org/apache/hop/core/vfs/HopVfs.java
index 71c48bbde1..7f94a04370 100644
--- a/core/src/main/java/org/apache/hop/core/vfs/HopVfs.java
+++ b/core/src/main/java/org/apache/hop/core/vfs/HopVfs.java
@@ -166,7 +166,10 @@ public class HopVfs {
fsm.addMimeTypeMap("application/x-gzip", "gz");
fsm.addMimeTypeMap("application/zip", "zip");
fsm.setFileContentInfoFactory(new FileContentInfoFilenameFactory());
- fsm.setReplicator(new DefaultFileReplicator());
+
+ DefaultFileReplicator replicator = new DefaultFileReplicator();
+ fsm.setReplicator(replicator);
+ fsm.setTemporaryFileStore(replicator);
fsm.setFilesCache(new SoftRefFilesCache());
fsm.setCacheStrategy(CacheStrategy.ON_RESOLVE);
diff --git
a/core/src/test/java/org/apache/hop/core/vfs/HopVfsNetworkProvidersTest.java
b/core/src/test/java/org/apache/hop/core/vfs/HopVfsNetworkProvidersTest.java
new file mode 100644
index 0000000000..928730fba1
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/core/vfs/HopVfsNetworkProvidersTest.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.core.vfs;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.sun.net.httpserver.HttpServer;
+import com.sun.net.httpserver.HttpsConfigurator;
+import com.sun.net.httpserver.HttpsServer;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.KeyStore;
+import java.security.SecureRandom;
+import java.util.Collections;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.commons.vfs2.provider.ftp.FtpFileSystemConfigBuilder;
+import org.apache.commons.vfs2.provider.ftps.FtpsDataChannelProtectionLevel;
+import org.apache.commons.vfs2.provider.ftps.FtpsFileSystemConfigBuilder;
+import org.apache.ftpserver.FtpServer;
+import org.apache.ftpserver.FtpServerFactory;
+import org.apache.ftpserver.listener.Listener;
+import org.apache.ftpserver.listener.ListenerFactory;
+import org.apache.ftpserver.ssl.SslConfigurationFactory;
+import org.apache.ftpserver.usermanager.PropertiesUserManagerFactory;
+import org.apache.ftpserver.usermanager.impl.BaseUser;
+import org.apache.ftpserver.usermanager.impl.WritePermission;
+import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
+import org.apache.sshd.server.SshServer;
+import org.apache.sshd.server.auth.password.AcceptAllPasswordAuthenticator;
+import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
+import org.apache.sshd.sftp.server.SftpSubsystemFactory;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Round-trip tests for the network VFS providers Hop registers by default:
{@code http}, {@code
+ * https}, {@code ftp}, {@code ftps}, {@code sftp}. Each is exercised against
an embedded server so
+ * we don't need external resources.
+ */
+class HopVfsNetworkProvidersTest {
+
+ private static final String KEYSTORE_PASSWORD = "hop-test-pass";
+ private static final String FTP_USER = "tester";
+ private static final String FTP_PASS = "secret";
+ private static final String SFTP_USER = "alice";
+ private static final String SFTP_PASS = "secret";
+
+ @TempDir static Path sharedRoot;
+
+ private static HttpServer httpServer;
+ private static int httpPort;
+
+ private static HttpsServer httpsServer;
+ private static int httpsPort;
+ private static Path keyStorePath;
+ private static String previousTrustStoreProperty;
+ private static String previousTrustStorePasswordProperty;
+
+ private static FtpServer ftpServer;
+ private static Listener ftpListener;
+ private static int ftpPort;
+
+ private static FtpServer ftpsServer;
+ private static Listener ftpsListener;
+ private static int ftpsPort;
+
+ private static SshServer sshServer;
+ private static int sftpPort;
+
+ @BeforeAll
+ static void startServers() throws Exception {
+ keyStorePath = generateTestKeyStore();
+
+ // HTTP
+ httpServer = HttpServer.create(new InetSocketAddress("localhost", 0), 0);
+ httpPort = httpServer.getAddress().getPort();
+ httpServer.createContext("/payload.txt", new
FixedPayloadHandler("http-payload"));
+ httpServer.start();
+
+ // HTTPS
+ SSLContext sslContext = buildServerSslContext(keyStorePath);
+ httpsServer = HttpsServer.create(new InetSocketAddress("localhost", 0), 0);
+ httpsServer.setHttpsConfigurator(new HttpsConfigurator(sslContext));
+ httpsServer.createContext("/secure.txt", new
FixedPayloadHandler("https-payload"));
+ httpsServer.start();
+ httpsPort = httpsServer.getAddress().getPort();
+
+ // Make Hop's HTTPS provider trust the self-signed cert via the JVM-wide
trust store.
+ previousTrustStoreProperty =
System.getProperty("javax.net.ssl.trustStore");
+ previousTrustStorePasswordProperty =
System.getProperty("javax.net.ssl.trustStorePassword");
+ System.setProperty("javax.net.ssl.trustStore", keyStorePath.toString());
+ System.setProperty("javax.net.ssl.trustStorePassword", KEYSTORE_PASSWORD);
+
+ // FTP
+ Path ftpHome = sharedRoot.resolve("ftp");
+ Files.createDirectories(ftpHome);
+ Files.writeString(ftpHome.resolve("greeting.txt"), "ftp-payload");
+ FtpServerStart ftp = startFtp(ftpHome, false);
+ ftpServer = ftp.server;
+ ftpListener = ftp.listener;
+ ftpPort = ftpListener.getPort();
+
+ // FTPS
+ Path ftpsHome = sharedRoot.resolve("ftps");
+ Files.createDirectories(ftpsHome);
+ Files.writeString(ftpsHome.resolve("greeting.txt"), "ftps-payload");
+ FtpServerStart ftps = startFtp(ftpsHome, true);
+ ftpsServer = ftps.server;
+ ftpsListener = ftps.listener;
+ ftpsPort = ftpsListener.getPort();
+
+ // SFTP
+ Path sftpHome = sharedRoot.resolve("sftp");
+ Files.createDirectories(sftpHome);
+ Files.writeString(sftpHome.resolve("greeting.txt"), "sftp-payload");
+ sshServer = startSftp(sftpHome);
+ sftpPort = sshServer.getPort();
+ }
+
+ @AfterAll
+ static void stopServers() throws Exception {
+ if (httpServer != null) httpServer.stop(0);
+ if (httpsServer != null) httpsServer.stop(0);
+ if (ftpServer != null) ftpServer.stop();
+ if (ftpsServer != null) ftpsServer.stop();
+ if (sshServer != null) sshServer.stop();
+ restoreSystemProperty("javax.net.ssl.trustStore",
previousTrustStoreProperty);
+ restoreSystemProperty("javax.net.ssl.trustStorePassword",
previousTrustStorePasswordProperty);
+ if (keyStorePath != null) Files.deleteIfExists(keyStorePath);
+ }
+
+ @Test
+ @DisplayName("http:// fetches a payload from an embedded HttpServer")
+ void httpProviderReadsFromEmbeddedServer() throws Exception {
+ assertEquals("http-payload", readToString("http://localhost:"; + httpPort +
"/payload.txt"));
+ }
+
+ @Test
+ @DisplayName("https:// fetches a payload over TLS from an embedded
HttpsServer")
+ void httpsProviderReadsFromEmbeddedServer() throws Exception {
+ assertEquals("https-payload", readToString("https://localhost:"; +
httpsPort + "/secure.txt"));
+ }
+
+ @Test
+ @DisplayName("ftp:// fetches a payload from an embedded Apache FtpServer")
+ void ftpProviderReadsFromEmbeddedServer() throws Exception {
+ String url = "ftp://"; + FTP_USER + ":" + FTP_PASS + "@localhost:" +
ftpPort + "/greeting.txt";
+ FileSystemOptions opts = new FileSystemOptions();
+ FtpFileSystemConfigBuilder.getInstance().setPassiveMode(opts, true);
+ assertEquals("ftp-payload", readWithOptions(url, opts));
+ }
+
+ @Test
+ @DisplayName("ftps:// fetches a payload over TLS from an embedded Apache
FtpServer")
+ void ftpsProviderReadsFromEmbeddedServer() throws Exception {
+ String url = "ftps://" + FTP_USER + ":" + FTP_PASS + "@localhost:" +
ftpsPort + "/greeting.txt";
+ FileSystemOptions opts = new FileSystemOptions();
+ FtpsFileSystemConfigBuilder ftps =
FtpsFileSystemConfigBuilder.getInstance();
+ ftps.setPassiveMode(opts, true);
+ ftps.setDataChannelProtectionLevel(opts, FtpsDataChannelProtectionLevel.P);
+ assertEquals("ftps-payload", readWithOptions(url, opts));
+ }
+
+ @Test
+ @DisplayName("sftp:// fetches a payload from an embedded Apache MINA SSHD
server")
+ void sftpProviderReadsFromEmbeddedServer() throws Exception {
+ String url =
+ "sftp://"; + SFTP_USER + ":" + SFTP_PASS + "@localhost:" + sftpPort +
"/greeting.txt";
+ assertEquals("sftp-payload", readToString(url));
+ }
+
+ // --- helpers
---------------------------------------------------------------------------
+
+ private static String readToString(String url) throws Exception {
+ try (InputStream in =
HopVfs.getFileObject(url).getContent().getInputStream()) {
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+
+ private static String readWithOptions(String url, FileSystemOptions opts)
throws Exception {
+ FileObject obj = HopVfs.getFileSystemManager().resolveFile(url, opts);
+ try (InputStream in = obj.getContent().getInputStream()) {
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+
+ /** Generates a fresh PKCS12 keystore containing a self-signed cert for
CN=localhost. */
+ private static Path generateTestKeyStore() throws Exception {
+ Path path = Files.createTempFile("hopvfs-network-", ".p12");
+ Files.deleteIfExists(path);
+ String keytool = Path.of(System.getProperty("java.home"), "bin",
"keytool").toString();
+ Process process =
+ new ProcessBuilder(
+ keytool,
+ "-genkeypair",
+ "-alias",
+ "hopvfs-test",
+ "-keyalg",
+ "RSA",
+ "-keysize",
+ "2048",
+ "-validity",
+ "1",
+ "-storetype",
+ "PKCS12",
+ "-keystore",
+ path.toString(),
+ "-storepass",
+ KEYSTORE_PASSWORD,
+ "-keypass",
+ KEYSTORE_PASSWORD,
+ "-dname",
+ "CN=localhost, OU=Hop, O=Apache, L=Test, S=Test, C=US",
+ "-ext",
+ "SAN=DNS:localhost,IP:127.0.0.1",
+ "-noprompt")
+ .redirectErrorStream(true)
+ .start();
+ int exit = process.waitFor();
+ if (exit != 0) {
+ String stderr = new String(process.getInputStream().readAllBytes(),
StandardCharsets.UTF_8);
+ throw new IOException("keytool failed (exit " + exit + "): " + stderr);
+ }
+ return path;
+ }
+
+ private static SSLContext buildServerSslContext(Path ks) throws Exception {
+ KeyStore keyStore = KeyStore.getInstance("PKCS12");
+ try (InputStream in = Files.newInputStream(ks)) {
+ keyStore.load(in, KEYSTORE_PASSWORD.toCharArray());
+ }
+ KeyManagerFactory kmf =
KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+ kmf.init(keyStore, KEYSTORE_PASSWORD.toCharArray());
+
+ TrustManagerFactory tmf =
+
TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ tmf.init(keyStore);
+
+ SSLContext ctx = SSLContext.getInstance("TLS");
+ ctx.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());
+ return ctx;
+ }
+
+ /** Holder so callers can recover the dynamic port from the {@link
Listener}. */
+ private record FtpServerStart(FtpServer server, Listener listener) {}
+
+ private static FtpServerStart startFtp(Path home, boolean tls) throws
Exception {
+ FtpServerFactory serverFactory = new FtpServerFactory();
+ ListenerFactory listenerFactory = new ListenerFactory();
+ listenerFactory.setPort(0);
+ if (tls) {
+ SslConfigurationFactory ssl = new SslConfigurationFactory();
+ ssl.setKeystoreFile(keyStorePath.toFile());
+ ssl.setKeystorePassword(KEYSTORE_PASSWORD);
+ ssl.setKeyPassword(KEYSTORE_PASSWORD);
+ listenerFactory.setSslConfiguration(ssl.createSslConfiguration());
+ // Explicit FTPS (AUTH TLS) — matches commons-vfs2's default
FtpsMode.EXPLICIT.
+ listenerFactory.setImplicitSsl(false);
+ }
+ Listener listener = listenerFactory.createListener();
+ serverFactory.addListener("default", listener);
+
+ PropertiesUserManagerFactory userManagerFactory = new
PropertiesUserManagerFactory();
+ BaseUser user = new BaseUser();
+ user.setName(FTP_USER);
+ user.setPassword(FTP_PASS);
+ user.setHomeDirectory(home.toString());
+ user.setAuthorities(Collections.singletonList(new WritePermission()));
+ serverFactory.setUserManager(userManagerFactory.createUserManager());
+ serverFactory.getUserManager().save(user);
+
+ FtpServer server = serverFactory.createServer();
+ server.start();
+ return new FtpServerStart(server, listener);
+ }
+
+ private static SshServer startSftp(Path home) throws IOException {
+ SshServer sshd = SshServer.setUpDefaultServer();
+ sshd.setHost("localhost");
+ sshd.setPort(0);
+ sshd.setKeyPairProvider(new
SimpleGeneratorHostKeyProvider(sharedRoot.resolve("hostkey.ser")));
+ sshd.setPasswordAuthenticator(AcceptAllPasswordAuthenticator.INSTANCE);
+ sshd.setFileSystemFactory(new VirtualFileSystemFactory(home));
+ sshd.setSubsystemFactories(Collections.singletonList(new
SftpSubsystemFactory()));
+ // commons-vfs2's SFTP client doesn't actively close the session when the
FileObject
+ // is closed; the server's default 10-minute IDLE_TIMEOUT would otherwise
hold the
+ // test open. Drop it to a few seconds so the read returns promptly.
+ org.apache.sshd.core.CoreModuleProperties.IDLE_TIMEOUT.set(
+ sshd, java.time.Duration.ofSeconds(5));
+ sshd.start();
+ return sshd;
+ }
+
+ private static void restoreSystemProperty(String key, String previousValue) {
+ if (previousValue == null) {
+ System.clearProperty(key);
+ } else {
+ System.setProperty(key, previousValue);
+ }
+ }
+
+ /** Sends a fixed string body with a 200 response. */
+ private static final class FixedPayloadHandler implements
com.sun.net.httpserver.HttpHandler {
+ private final byte[] body;
+
+ FixedPayloadHandler(String body) {
+ this.body = body.getBytes(StandardCharsets.UTF_8);
+ }
+
+ @Override
+ public void handle(com.sun.net.httpserver.HttpExchange exchange) throws
IOException {
+ exchange.getResponseHeaders().add("Content-Type", "text/plain;
charset=UTF-8");
+ exchange.sendResponseHeaders(200, body.length);
+ try (OutputStream out = exchange.getResponseBody()) {
+ out.write(body);
+ }
+ exchange.close();
+ }
+ }
+}
diff --git
a/core/src/test/java/org/apache/hop/core/vfs/HopVfsProvidersTest.java
b/core/src/test/java/org/apache/hop/core/vfs/HopVfsProvidersTest.java
new file mode 100644
index 0000000000..042a03b802
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/core/vfs/HopVfsProvidersTest.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.core.vfs;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Coverage check for the VFS providers Hop registers by default in {@link
HopVfs}. Smoke-tests
+ * every scheme is registered; for providers that don't need a network or
credentials, also
+ * round-trips a real file to confirm the provider is functional and not just
resolvable.
+ */
+class HopVfsProvidersTest {
+
+ /** Schemes registered unconditionally in {@code
HopVfs.createFileSystemManager}. */
+ private static final List<String> BUILTIN_SCHEMES =
+ List.of(
+ "ram",
+ "file",
+ "res",
+ "zip",
+ "gz",
+ "jar",
+ "http",
+ "https",
+ "ftp",
+ "ftps",
+ "sftp",
+ "war",
+ "par",
+ "ear",
+ "sar",
+ "ejb3",
+ "tmp",
+ "tar",
+ "tbz2",
+ "tgz",
+ "bz2",
+ "files-cache");
+
+ @Test
+ @DisplayName("All built-in providers are registered")
+ void allBuiltinSchemesAreRegistered() {
+ DefaultFileSystemManager fsm = HopVfs.getFileSystemManager();
+ Set<String> registered = new HashSet<>(Arrays.asList(fsm.getSchemes()));
+
+ for (String scheme : BUILTIN_SCHEMES) {
+ assertTrue(
+ fsm.hasProvider(scheme),
+ "VFS provider missing for scheme '" + scheme + "'. Registered: " +
registered);
+ }
+ }
+
+ @Test
+ @DisplayName("file:// round-trip: write, read back, exists, delete")
+ void fileProviderRoundTrips(@TempDir Path tmp) throws Exception {
+ Path path = tmp.resolve("hello.txt");
+ String url = path.toUri().toString();
+ writeBytes(url, "file-content".getBytes(StandardCharsets.UTF_8));
+ assertEquals("file-content", readString(url));
+ assertTrue(HopVfs.fileExists(url));
+ HopVfs.getFileObject(url).delete();
+ }
+
+ @Test
+ @DisplayName("ram:// round-trip")
+ void ramProviderRoundTrips() throws Exception {
+ String url = "ram:///" + uniqueName(".txt");
+ writeBytes(url, "ram-content".getBytes(StandardCharsets.UTF_8));
+ assertEquals("ram-content", readString(url));
+ }
+
+ @Test
+ @DisplayName("tmp:// round-trip")
+ void tmpProviderRoundTrips() throws Exception {
+ String url = "tmp:///" + uniqueName(".txt");
+ writeBytes(url, "tmp-content".getBytes(StandardCharsets.UTF_8));
+ assertEquals("tmp-content", readString(url));
+ }
+
+ @Test
+ @DisplayName("files-cache:// resolves to the same temp provider as tmp://")
+ void filesCacheProviderResolves() throws Exception {
+ String url = "files-cache:///" + uniqueName(".txt");
+ writeBytes(url, "cache".getBytes(StandardCharsets.UTF_8));
+ assertEquals("cache", readString(url));
+ }
+
+ @Test
+ @DisplayName("res:// reads a classpath resource")
+ void resProviderReadsClasspathResource() throws Exception {
+ // Use a resource that's guaranteed to be on the test classpath.
+ String url = "res:" + getClass().getName().replace('.', '/') + ".class";
+ FileObject obj = HopVfs.getFileObject(url);
+ assertTrue(obj.exists(), url + " should exist on the classpath");
+ assertTrue(obj.getContent().getSize() > 0);
+ }
+
+ @Test
+ @DisplayName("zip:// reads entries from a real zip file")
+ void zipProviderReadsEntries(@TempDir Path tmp) throws Exception {
+ Path archive = tmp.resolve("archive.zip");
+ try (java.util.zip.ZipOutputStream out =
+ new java.util.zip.ZipOutputStream(Files.newOutputStream(archive))) {
+ out.putNextEntry(new java.util.zip.ZipEntry("entry.txt"));
+ out.write("zip-entry".getBytes(StandardCharsets.UTF_8));
+ out.closeEntry();
+ }
+ String url = "zip:" + archive.toUri() + "!/entry.txt";
+ assertEquals("zip-entry", readString(url));
+ }
+
+ @Test
+ @DisplayName("jar:// reads entries the same way zip:// does")
+ void jarProviderReadsEntries(@TempDir Path tmp) throws Exception {
+ Path archive = tmp.resolve("archive.jar");
+ try (java.util.zip.ZipOutputStream out =
+ new java.util.zip.ZipOutputStream(Files.newOutputStream(archive))) {
+ out.putNextEntry(new java.util.zip.ZipEntry("payload.txt"));
+ out.write("jar-entry".getBytes(StandardCharsets.UTF_8));
+ out.closeEntry();
+ }
+ String url = "jar:" + archive.toUri() + "!/payload.txt";
+ assertEquals("jar-entry", readString(url));
+ }
+
+ @Test
+ @DisplayName("war/par/ear/sar/ejb3 share the jar provider")
+ void jarAliasesShareTheJarProvider(@TempDir Path tmp) throws Exception {
+ Path archive = tmp.resolve("archive.zip");
+ try (java.util.zip.ZipOutputStream out =
+ new java.util.zip.ZipOutputStream(Files.newOutputStream(archive))) {
+ out.putNextEntry(new java.util.zip.ZipEntry("inside.txt"));
+ out.write("hi".getBytes(StandardCharsets.UTF_8));
+ out.closeEntry();
+ }
+ for (String alias : new String[] {"war", "par", "ear", "sar", "ejb3"}) {
+ String url = alias + ":" + archive.toUri() + "!/inside.txt";
+ assertEquals("hi", readString(url), alias + " should read the entry");
+ }
+ }
+
+ @Test
+ @DisplayName("gz:// decompresses a gzipped file")
+ void gzProviderDecompresses(@TempDir Path tmp) throws Exception {
+ Path gz = tmp.resolve("payload.gz");
+ byte[] payload = "gzip-payload".getBytes(StandardCharsets.UTF_8);
+ try (java.util.zip.GZIPOutputStream out =
+ new java.util.zip.GZIPOutputStream(Files.newOutputStream(gz))) {
+ out.write(payload);
+ }
+ String url = "gz:" + gz.toUri();
+ try (InputStream in =
HopVfs.getFileObject(url).getContent().getInputStream()) {
+ assertArrayEquals(payload, in.readAllBytes());
+ }
+ }
+
+ @Test
+ @DisplayName("bz2:// decompresses a bzip2 file")
+ void bz2ProviderDecompresses(@TempDir Path tmp) throws Exception {
+ Path bz2 = tmp.resolve("payload.bz2");
+ byte[] payload = "bz2-payload".getBytes(StandardCharsets.UTF_8);
+ try
(org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream out =
+ new
org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream(
+ Files.newOutputStream(bz2))) {
+ out.write(payload);
+ }
+ String url = "bz2:" + bz2.toUri();
+ try (InputStream in =
HopVfs.getFileObject(url).getContent().getInputStream()) {
+ assertArrayEquals(payload, in.readAllBytes());
+ }
+ }
+
+ @Test
+ @DisplayName("tar:// reads entries from a tar archive")
+ void tarProviderReadsEntries(@TempDir Path tmp) throws Exception {
+ Path tar = tmp.resolve("archive.tar");
+ try (org.apache.commons.compress.archivers.tar.TarArchiveOutputStream out =
+ new org.apache.commons.compress.archivers.tar.TarArchiveOutputStream(
+ Files.newOutputStream(tar))) {
+ byte[] payload = "tar-entry".getBytes(StandardCharsets.UTF_8);
+ org.apache.commons.compress.archivers.tar.TarArchiveEntry entry =
+ new
org.apache.commons.compress.archivers.tar.TarArchiveEntry("entry.txt");
+ entry.setSize(payload.length);
+ out.putArchiveEntry(entry);
+ out.write(payload);
+ out.closeArchiveEntry();
+ }
+ String url = "tar:" + tar.toUri() + "!/entry.txt";
+ assertEquals("tar-entry", readString(url));
+ }
+
+ @Test
+ @DisplayName(
+ "Remaining network providers are registered (round-trip in
HopVfsNetworkProvidersTest)")
+ void remainingNetworkProvidersAreRegistered() {
+ // http/https/ftp/ftps/sftp need real servers; HopVfsNetworkProvidersTest
spins up embedded
+ // ones for those. This case just checks they're present (the round-trip
tests would fail
+ // earlier if they weren't).
+ DefaultFileSystemManager fsm = HopVfs.getFileSystemManager();
+ for (String scheme : new String[] {"http", "https", "ftp", "ftps",
"sftp"}) {
+ assertTrue(fsm.hasProvider(scheme), scheme + " provider must be
registered");
+ }
+ }
+
+ // --- helpers
-----------------------------------------------------------------------------
+
+ private static String uniqueName(String suffix) {
+ return "hopvfs-test-" + System.nanoTime() + suffix;
+ }
+
+ private static void writeBytes(String url, byte[] bytes) throws Exception {
+ FileObject obj = HopVfs.getFileObject(url);
+ try (OutputStream out = obj.getContent().getOutputStream()) {
+ out.write(bytes);
+ }
+ }
+
+ private static String readString(String url) throws Exception {
+ try (InputStream in =
HopVfs.getFileObject(url).getContent().getInputStream()) {
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ throw e;
+ }
+ }
+}
diff --git a/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc
b/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc
index 846ff157da..91acb32492 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc
@@ -38,12 +38,12 @@ Click the File system name to access more detailed file
system documentation.
|===
|File System|Description|URI Format
|xref:vfs/aws-s3-vfs.adoc[AWS S3]|Provides access to Amazon S3 Buckets|`s3://`
-|xref:vfs/azure-blob-storage-vfs.adoc[Azure Blob Storage]|Provides access to
Azure Blob Storage|`azure://`
+|xref:vfs/azure-blob-storage-vfs.adoc[Azure Blob Storage]|Provides access to
Azure Blob Storage|`azure://` (alias: `azfs://`)
|xref:vfs/dropbox-vfs.adoc[Dropbox]|Provides access to Dropbox|`dropbox://`
|xref:vfs/google-cloud-storage-vfs.adoc[Google Cloud Storage]|Provides access
to Google Cloud Storage buckets|`gs://`
|xref:vfs/google-drive-vfs.adoc[Google Drive]|Provides access to Google Drive
folders|`googledrive://`
-|xref:metadata-types/minio-connection.adoc[Minio connection]|Provides access
to S3 endpoints using a Minio client|`any://`
-|xref:metadata-types/webdav-connection.adoc[WebDAV Connection]|Provides access
to WebDAV servers via a named connection (metadata)|`+connectionName://+`
+|xref:metadata-types/minio-connection.adoc[Minio connection]|Provides access
to S3-compatible endpoints via a named Minio connection
(metadata)|`+<connectionName>://+`
+|xref:metadata-types/webdav-connection.adoc[WebDAV Connection]|Provides access
to WebDAV servers, either via the static schemes or a named connection
(metadata)|`webdav4://`, `webdav4s://`, or `+<connectionName>://+`
|===
== Apache VFS File System Types
@@ -72,10 +72,6 @@ Examples
* `+gz:/my/gz/file.gz+`
-//
-// CIFS
-//
-|CIFS*||
|File|Provides access to the files on the local physical file system.
a|URI Format
@@ -100,7 +96,7 @@ Examples
|FTP|Provides access to the files on an FTP server.
a|URI Format
-`+tp://[ username[: password]@] hostname[: port][ relative-path]+`
+`+ftp://[ username[: password]@] hostname[: port][ relative-path]+`
Examples
@@ -127,21 +123,6 @@ Examples
// GZIP
//
|GZIP|see 'bzip2'|
-//
-// HDFS
-//
-|HDFS|Provides access to files in an Apache Hadoop File System (HDFS).
-On Windows the integration test is disabled by default, as it requires
binaries.
-a|
-URI Format
-
-`+hdfs:// hostname[: port][ absolute-path]+`
-
-Examples
-
-* `+hdfs://somehost:8080/downloads/some_dir+`
-* `+hdfs://somehost:8080/downloads/some_file.ext+`
-
//
// HTTP
//
@@ -176,6 +157,7 @@ Examples
// Jar, Zip and Tar
//
|Jar, Zip and Tar|Provides read-only access to the contents of Zip, Jar and
Tar files.
+The `jar` provider is also registered under the `war`, `par`, `ear`, `sar`,
and `ejb3` schemes for convenience when working with Java EE archive types.
a|
URI Format
@@ -202,23 +184,6 @@ Examples
* `+tar:gz:http://anyhost/dir/mytar.tar.gz!/mytar.tar!/path/in/tar/README.txt+`
* `+tgz:file://anyhost/dir/mytar.tgz!/somepath/somefile+`
-//
-// mime
-//
-|mime*|This (sandbox) filesystem can read mails and its attachements like
archives.
-If a part in the parsed mail has no name, a dummy name will be generated.
-The dummy name is: _body_part_X where X will be replaced by the part number.
-a|
-URI Format
-
-`+mime:// mime-file-uri[! absolute-path]+`
-
-Examples
-
-* `+mime:file:///your/path/mail/anymail.mime!/+`
-* `+mime:file:///your/path/mail/anymail.mime!/filename.pdf+`
-* `+mime:file:///your/path/mail/anymail.mime!/_body_part_0+`
-
//
// RAM
//
@@ -236,19 +201,6 @@ Examples
* `+ram:///any/path/to/file.txt+`
-//
-// RES
-//
-|RES|This is not really a filesystem, it just tries to lookup a resource using
javas ClassLoader.getResource() and creates a VFS url for further processing.
-a|
-URI Format
-
-`+res://[ path]+`
-
-Examples
-
-* `+res://path/in/classpath/image.png` might result in
`jar:file://my/path/to/images.jar!/path/in/classpath/image.png+`
-
//
// SFTP
//
@@ -311,4 +263,34 @@ Examples
|Zip|see 'jar'|
|===
-*) VFS file system type in development
+== Supported operations
+
+The matrix below shows which operations each registered provider exposes,
taken from the capability set each provider declares in code.
+Capabilities can drift between commons-vfs2 releases, so the authoritative
reference is the `capabilities` collection on each `FileProvider` /
`FileSystem` class (in `commons-vfs2` for the standard providers, and in
`plugins/tech/<plugin>` for the Hop-managed ones).
+A ✓ means the capability is declared by the provider; ✗ means it isn't.
+Server-side restrictions (read-only mounts, bucket policies, account
permissions, ...) can still block individual operations at runtime.
+
+[options="header",cols="2,1,1,1,1,1,1,1,1,1"]
+|===
+|Scheme|Read|Write|Append|List|Create/Delete|Rename|Random read|Random
write|Last-modified
+
+// Hop-managed providers
+|`s3` |✓|✓|✗|✓|✓|✓|✓|✗|read
+|`azure` |✓|✓|✗|✓|✓|✓|✗|✗|read
+|`gs` |✓|✓|✗|✓|✓|✗|✗|✗|read/set
+|`googledrive` |✓|✓|✗|✓|✓|✓|✓|✗|read
+|`dropbox` |✓|✓|✗|✓|✓|✓|✗|✗|read
+|`+<minio-conn>+` |✓|✓|✗|✓|✓|✓|✓|✗|read
+|`webdav4`, `webdav4s`, named WebDAV connection |✓|✓|✗|✓|✓|✓|✓|✗|read
+
+// Apache VFS providers
+|`file`, `tmp`, `files-cache` |✓|✓|✓|✓|✓|✓|✓|✓|read/set
+|`ram` |✓|✓|✓|✓|✓|✓|✓|✓|read/set
+|`zip` |✓|✗|✗|✓|✗|✗|✗|✗|read
+|`jar` (and `war`/`par`/`ear`/`sar`/`ejb3`) |✓|✗|✗|✓|✗|✗|✗|✗|read
+|`tar` (and `tbz2`/`tgz`) |✓|✗|✗|✓|✗|✗|✗|✗|read
+|`gz`, `bz2` |✓|✓|✗|✓|✗|✗|✗|✗|read
+|`http`, `https` |✓|✗|✗|✗|✗|✗|✓|✗|read
+|`ftp`, `ftps` |✓|✓|✓|✓|✓|✓|✓|✗|read
+|`sftp` |✓|✓|✓|✓|✓|✓|✓|✗|read/set
+|===
