This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new 807debeea3 allow // as valid path for SFTP targets, fixes #7930 (#8886)
807debeea3 is described below
commit 807debeea340451da6f9cb6fa376848841e606f2
Author: Hans Van Akelyen <[email protected]>
AuthorDate: Tue Oct 13 16:58:19 2026 +0200
allow // as valid path for SFTP targets, fixes #7930 (#8886)
---
.../apache/hop/core/vfs/HopVfsGetFilenameTest.java | 123 ++++++
.../hop/core/vfs/LocalFileNameParserShapeTest.java | 277 +++++++++++++
.../org/apache/hop/junit/vfs/VfsPathShapes.java | 240 +++++++++++
.../org/apache/hop/vfs/ftp/FtpPathShapeTest.java | 105 +++++
.../hop/vfs/sftp/SftpConnectionFileName.java | 31 +-
.../hop/vfs/sftp/SftpConnectionFileNameParser.java | 29 +-
.../org/apache/hop/vfs/sftp/SftpConnections.java | 14 +-
.../hop/vfs/sftp/provider/IRootPrefixFileName.java | 40 ++
.../apache/hop/vfs/sftp/provider/SftpFileName.java | 79 ++++
.../hop/vfs/sftp/provider/SftpFileNameParser.java | 88 ++++
.../hop/vfs/sftp/provider/SftpFileObject.java | 27 +-
.../hop/vfs/sftp/provider/SftpFileSystem.java | 9 +-
.../apache/hop/vfs/sftp/SftpConnectionsTest.java | 21 +
.../org/apache/hop/vfs/sftp/SftpPathShapeTest.java | 189 +++++++++
.../vfs/sftp/metadata/SftpConnectionJsonTest.java | 100 +++++
.../vfs/sftp/provider/SftpDoubleSlashPathTest.java | 441 +++++++++++++++++++++
16 files changed, 1792 insertions(+), 21 deletions(-)
diff --git
a/core/src/test/java/org/apache/hop/core/vfs/HopVfsGetFilenameTest.java
b/core/src/test/java/org/apache/hop/core/vfs/HopVfsGetFilenameTest.java
new file mode 100644
index 0000000000..62a5ac3a2e
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/core/vfs/HopVfsGetFilenameTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.core.vfs;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.provider.VfsComponentContext;
+import org.apache.commons.vfs2.provider.local.GenericFileNameParser;
+import org.apache.commons.vfs2.provider.local.LocalFileNameParser;
+import org.apache.commons.vfs2.provider.local.WindowsFileNameParser;
+import org.apache.hop.core.Const;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The four branches of {@link HopVfs#getFilename(FileObject)}, which turns a
file object back into
+ * something to hand to the operating system.
+ *
+ * <p>It is worth pinning all four before touching any of them. Around 140
call sites across 65
+ * files go through this method and its {@link HopVfs#normalize(String)}
wrapper, no test called it
+ * directly until this one, and its single biggest caller - the VFS file
dialog - is excluded from
+ * the coverage report by the {@code **}{@code /*Dialog*} jacoco exclusion. So
nothing in the build
+ * would notice if a branch changed.
+ *
+ * <p>Note that the contract is not uniform: two branches return a
<em>URI</em> and two return an
+ * <em>operating system path</em>. Anything handing the result to {@code new
File(...)} therefore
+ * behaves differently for a UNC share than for a drive letter.
+ */
+class HopVfsGetFilenameTest {
+
+ /** A file object is only ever asked for its name, so a name is all a
stand-in needs. */
+ private static FileObject fileObjectNamed(FileName name) {
+ FileObject fileObject = mock(FileObject.class);
+ when(fileObject.getName()).thenReturn(name);
+ return fileObject;
+ }
+
+ private static FileName parse(LocalFileNameParser parser, String uri) throws
Exception {
+ FileSystemManager fsm = mock(FileSystemManager.class);
+ when(fsm.getSchemes()).thenReturn(new String[] {"file"});
+ VfsComponentContext context = mock(VfsComponentContext.class);
+ when(context.getFileSystemManager()).thenReturn(fsm);
+ return parser.parseUri(context, null, uri);
+ }
+
+ /** The separators the caller's platform expects, for the branches returning
an OS path. */
+ private static String osPath(String slashPath) {
+ return "/".equals(Const.FILE_SEPARATOR)
+ ? slashPath
+ : Const.replace(slashPath, "/", Const.FILE_SEPARATOR);
+ }
+
+ @Test
+ @DisplayName("a non-file scheme comes back as its URI, untouched")
+ void nonFileSchemeReturnsTheUri() {
+ FileName name = mock(FileName.class);
+ when(name.getRootURI()).thenReturn("sftp://user@host/";);
+ when(name.getURI()).thenReturn("sftp://user@host/inbox/customers.csv";);
+
+ assertEquals("sftp://user@host/inbox/customers.csv";,
HopVfs.getFilename(fileObjectNamed(name)));
+ }
+
+ @Test
+ @DisplayName("a UNC share comes back as its URI, all four slashes intact")
+ void uncShareReturnsTheUri() throws Exception {
+ FileName name = parse(new WindowsFileNameParser(),
"file:////server/share/dir/file.txt");
+
+ // Guard the precondition this branch keys off, so a change of root shows
up here too.
+ assertEquals("file:////server/share/", name.getRootURI());
+ assertEquals("file:////server/share/dir/file.txt",
HopVfs.getFilename(fileObjectNamed(name)));
+ }
+
+ @Test
+ @DisplayName("a Windows drive comes back as an OS path, drive letter
included")
+ void windowsDriveReturnsAnOsPath() throws Exception {
+ FileName name = parse(new WindowsFileNameParser(),
"file:///C:/temp/file.txt");
+
+ assertEquals("file:///C:/", name.getRootURI());
+ assertEquals(osPath("C:/temp/file.txt"),
HopVfs.getFilename(fileObjectNamed(name)));
+ }
+
+ @Test
+ @DisplayName("everywhere else comes back as an OS path with no root prefix")
+ void posixReturnsAnOsPath() throws Exception {
+ FileName name = parse(new GenericFileNameParser(),
"file:///home/me/file.txt");
+
+ assertEquals("file:///", name.getRootURI());
+ assertEquals(osPath("/home/me/file.txt"),
HopVfs.getFilename(fileObjectNamed(name)));
+ }
+
+ /**
+ * A UNC-looking name parsed by the generic parser is an ordinary path, so
it takes the OS-path
+ * branch rather than the URI one. Same input, different answer per platform
- which is the thing
+ * to keep in mind when changing any of this.
+ */
+ @Test
+ @DisplayName("a UNC-looking name under the generic parser is just a path")
+ void uncUnderTheGenericParserIsJustAPath() throws Exception {
+ FileName name = parse(new GenericFileNameParser(),
"file:////server/share/dir/file.txt");
+
+ assertEquals("file:///", name.getRootURI());
+ assertEquals(osPath("/server/share/dir/file.txt"),
HopVfs.getFilename(fileObjectNamed(name)));
+ }
+}
diff --git
a/core/src/test/java/org/apache/hop/core/vfs/LocalFileNameParserShapeTest.java
b/core/src/test/java/org/apache/hop/core/vfs/LocalFileNameParserShapeTest.java
new file mode 100644
index 0000000000..52cdb53659
--- /dev/null
+++
b/core/src/test/java/org/apache/hop/core/vfs/LocalFileNameParserShapeTest.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.core.vfs;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.provider.VfsComponentContext;
+import org.apache.commons.vfs2.provider.local.GenericFileNameParser;
+import org.apache.commons.vfs2.provider.local.LocalFileNameParser;
+import org.apache.commons.vfs2.provider.local.WindowsFileNameParser;
+import org.apache.hop.junit.vfs.VfsPathShapes;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+
+/**
+ * What the local file name parsers make of the awkward path shapes in {@link
VfsPathShapes}.
+ *
+ * <p>Both parsers are exercised by name rather than through {@link HopVfs},
so this pins Windows
+ * behaviour on every platform. That matters more here than anywhere else: the
CI workflows all run
+ * on {@code ubuntu-latest}, so nothing else in the build would notice a
Windows path regression,
+ * and UNC is the one place in Commons VFS where a leading double slash is
meant to survive.
+ */
+class LocalFileNameParserShapeTest {
+
+ private static VfsPathShapes.NameParser parserFor(LocalFileNameParser
parser) {
+ FileSystemManager fsm = mock(FileSystemManager.class);
+ when(fsm.getSchemes()).thenReturn(new String[] {"file"});
+ VfsComponentContext context = mock(VfsComponentContext.class);
+ when(context.getFileSystemManager()).thenReturn(fsm);
+ return uri -> parser.parseUri(context, null, uri);
+ }
+
+ @Nested
+ @DisplayName("Windows, drive letter")
+ class WindowsDrive {
+
+ private static final String BASE = "file:///C:";
+ private final VfsPathShapes.NameParser parser = parserFor(new
WindowsFileNameParser());
+
+ @Test
+ void paths() {
+ VfsPathShapes.assertPaths(
+ BASE,
+ parser,
+ """
+ plain /dir/file.txt
+ leading-double-slash /dir/file.txt
+ leading-triple-slash /dir/file.txt
+ interior-double-slash /dir/file.txt
+ trailing-slash /dir
+ root-only /
+ no-path error # "file:///C:" has no
separator after the drive
+ dot-segment /dir/file.txt
+ parent-segment /dir/file.txt
+ parent-above-root error
+ encoded-slash /dir/file.txt # %2F decodes before it is
normalised away
+ space /dir/my file.txt
+ encoded-space /dir/my file.txt
+ hash /dir/file#1.txt
+ question-mark /dir/file?1.txt
+ plus /dir/file+1.txt
+ backslash /dir/file.txt # a backslash becomes a
separator
+ leading-dots-in-name /dir/..hidden
+ unicode /dir/naïve.txt
+ """);
+ }
+
+ @Test
+ void theDriveStaysInTheRoot() throws Exception {
+ assertEquals("file:///C:/", parser.parse(BASE +
"/dir/file.txt").getRootURI());
+ }
+
+ @Test
+ void uriRoundTrips() {
+ VfsPathShapes.assertUriRoundTrips(BASE, parser);
+ }
+ }
+
+ /**
+ * The one shape which survives anywhere in Commons VFS: {@code
//server/share} is hoisted into
+ * the root by {@link WindowsFileNameParser}, so it is never handed to the
normaliser as a path.
+ * Every URI below keeps all four of its leading slashes, and that is what a
UNC fix must not
+ * break.
+ */
+ @Nested
+ @DisplayName("Windows, UNC share")
+ class WindowsUnc {
+
+ private static final String BASE = "file:////server/share";
+ private final VfsPathShapes.NameParser parser = parserFor(new
WindowsFileNameParser());
+
+ private FileName parse(String uri) throws Exception {
+ return parser.parse(uri);
+ }
+
+ @Test
+ void paths() {
+ VfsPathShapes.assertPaths(
+ BASE,
+ parser,
+ """
+ plain /dir/file.txt
+ leading-double-slash /dir/file.txt
+ leading-triple-slash /dir/file.txt
+ interior-double-slash /dir/file.txt
+ trailing-slash /dir
+ root-only /
+ no-path /
+ dot-segment /dir/file.txt
+ parent-segment /dir/file.txt
+ parent-above-root error
+ encoded-slash /dir/file.txt
+ space /dir/my file.txt
+ encoded-space /dir/my file.txt
+ hash /dir/file#1.txt
+ question-mark /dir/file?1.txt
+ plus /dir/file+1.txt
+ backslash /dir/file.txt
+ leading-dots-in-name /dir/..hidden
+ unicode /dir/naïve.txt
+ """);
+ }
+
+ /**
+ * The root swallows exactly two segments, whatever they were meant to be:
in {@code
+ * file:////servername/path/on/server} the share is {@code path}, not
{@code servername}, and
+ * the file is {@code /on/server} beneath it.
+ */
+ @Test
+ void theRootIsHostAndShare() throws Exception {
+ final FileName name = parse("file:////servername/path/on/server");
+
+ assertEquals("file:////servername/path/", name.getRootURI());
+ assertEquals("/on/server", name.getPath());
+ assertEquals("file:////servername/path/on/server", name.getURI());
+ }
+
+ /**
+ * Two, three and four slashes all mean the same UNC share, because the
parser skips up to four
+ * and only then looks for a drive letter. So {@code
file:///servername/path} - which on any
+ * other platform is the plain local path {@code /servername/path} - is
the share {@code
+ * \\servername\path} here. Nothing in CI would notice: every workflow
runs on Linux.
+ */
+ @Test
+ void twoThreeAndFourSlashesAllMeanTheSameShare() throws Exception {
+ for (String uri :
+ new String[] {
+ "file://servername/path/on/server",
+ "file:///servername/path/on/server",
+ "file:////servername/path/on/server"
+ }) {
+ assertEquals("file:////servername/path/", parse(uri).getRootURI(),
uri);
+ assertEquals("/on/server", parse(uri).getPath(), uri);
+ }
+ }
+
+ /** A drive letter still wins over the UNC reading. */
+ @Test
+ void aDriveLetterIsNotAShare() throws Exception {
+ assertEquals("file:///C:/", parse("file:///C:/temp/x.txt").getRootURI());
+ }
+
+ /** A host on its own is not a share, and is refused rather than guessed
at. */
+ @Test
+ void aHostWithoutAShareIsRejected() {
+ final FileSystemException e =
+ assertThrows(FileSystemException.class, () ->
parse("file:////servername"));
+ assertTrue(
+ e.getMessage().contains("Share name missing"), "unexpected message:
" + e.getMessage());
+ }
+
+ /** The share is part of the root, which is why it survives normalisation.
*/
+ @Test
+ void theShareStaysInTheRoot() throws Exception {
+ assertEquals("file:////server/share/", parser.parse(BASE +
"/dir/file.txt").getRootURI());
+ assertEquals(
+ "file:////server/share/dir/file.txt", parser.parse(BASE +
"/dir/file.txt").getURI());
+ }
+
+ @Test
+ void uriRoundTrips() {
+ VfsPathShapes.assertUriRoundTrips(BASE, parser);
+ }
+ }
+
+ /**
+ * The parser every non-Windows platform gets. It has no root prefix to
hoist anything into, so a
+ * leading double slash is flattened into the path and axed - the same
collapse which makes a
+ * mainframe SFTP data set unaddressable.
+ */
+ @Nested
+ @DisplayName("Generic (non-Windows)")
+ class Generic {
+
+ private static final String BASE = "file://";
+ private final VfsPathShapes.NameParser parser = parserFor(new
GenericFileNameParser());
+
+ private FileName parse(String uri) throws Exception {
+ return parser.parse(uri);
+ }
+
+ @Test
+ void paths() {
+ VfsPathShapes.assertPaths(
+ BASE,
+ parser,
+ """
+ plain /dir/file.txt
+ leading-double-slash /dir/file.txt # no hoist, so the share is
flattened away
+ leading-triple-slash /dir/file.txt
+ interior-double-slash /dir/file.txt
+ trailing-slash /dir
+ root-only /
+ no-path /
+ dot-segment /dir/file.txt
+ parent-segment /dir/file.txt
+ parent-above-root error
+ encoded-slash /dir/file.txt
+ space /dir/my file.txt
+ encoded-space /dir/my file.txt
+ hash /dir/file#1.txt
+ question-mark /dir/file?1.txt
+ plus /dir/file+1.txt
+ backslash /dir/file.txt
+ leading-dots-in-name /dir/..hidden
+ unicode /dir/naïve.txt
+ """);
+ }
+
+ /**
+ * The same URI the Windows parser reads as a share is an ordinary local
path here - so {@code
+ * file:////servername/path/on/server} names two different files depending
on the platform.
+ */
+ @Test
+ void aShareIsJustAPathHere() throws Exception {
+ final FileName name = parse("file:////servername/path/on/server");
+
+ assertEquals("file:///", name.getRootURI());
+ assertEquals("/servername/path/on/server", name.getPath());
+ assertEquals("file:///servername/path/on/server", name.getURI());
+ }
+
+ /** Everything lands under a bare root, UNC-looking names included. */
+ @Test
+ void thereIsNoRootPrefixToHoistInto() throws Exception {
+ assertEquals("file:///", parser.parse(BASE +
"//server/share/x").getRootURI());
+ assertEquals("/server/share/x", parser.parse(BASE +
"//server/share/x").getPath());
+ }
+
+ @Test
+ void uriRoundTrips() {
+ VfsPathShapes.assertUriRoundTrips(BASE, parser);
+ }
+ }
+}
diff --git a/core/src/test/java/org/apache/hop/junit/vfs/VfsPathShapes.java
b/core/src/test/java/org/apache/hop/junit/vfs/VfsPathShapes.java
new file mode 100644
index 0000000000..e2b2a5cd1e
--- /dev/null
+++ b/core/src/test/java/org/apache/hop/junit/vfs/VfsPathShapes.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.junit.vfs;
+
+import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import org.apache.commons.vfs2.FileName;
+import org.junit.jupiter.api.function.Executable;
+
+/**
+ * One shared set of awkward path shapes to run through every VFS file name
parser, so what a parser
+ * makes of them is written down instead of assumed.
+ *
+ * <p>A file name parser is a handful of lines of its own on top of a Commons
VFS base class, so
+ * line coverage says nothing useful about one: {@code SftpFileNameParser}
sits at 100% and still
+ * cannot address a mainframe data set. What matters is which path shapes
survive it, which is what
+ * this pins down. Expectations are written as a table, one shape per line,
name and expected value
+ * separated by two or more spaces:
+ *
+ * <pre>
+ * VfsPathShapes.assertPaths("sftp://user@host:22";, manager::resolveURI, """
+ * plain /dir/file.txt
+ * leading-double-slash /dir/file.txt # collapsed, see issue
+ * ...
+ * """);
+ * </pre>
+ *
+ * <p>Every shape in {@link #SHAPES} has to appear in the table and vice
versa, so a shape added
+ * here surfaces as a failure in each parser's test rather than quietly going
unchecked.
+ *
+ * @see #assertUriRoundTrips(String, NameParser) the invariant a fix has to
keep
+ */
+public final class VfsPathShapes {
+
+ /** Expected value for a shape the parser rejects. */
+ public static final String ERROR = "error";
+
+ /** A path shape: a name to refer to it by, and the path to append to a base
URI. */
+ public record Shape(String name, String suffix) {}
+
+ /**
+ * The shapes. Appended to a base URI such as {@code sftp://user@host:22} or
{@code file://}, so
+ * each one starts at the root of the file system.
+ */
+ public static final List<Shape> SHAPES =
+ List.of(
+ new Shape("plain", "/dir/file.txt"),
+ // A mainframe data set and a Windows UNC share both live or die by
these two.
+ new Shape("leading-double-slash", "//dir/file.txt"),
+ new Shape("leading-triple-slash", "///dir/file.txt"),
+ // What a base folder ending in "/" plus a child starting with "/"
concatenates to.
+ new Shape("interior-double-slash", "/dir//file.txt"),
+ new Shape("trailing-slash", "/dir/"),
+ new Shape("root-only", "/"),
+ new Shape("no-path", ""),
+ new Shape("dot-segment", "/dir/./file.txt"),
+ new Shape("parent-segment", "/dir/sub/../file.txt"),
+ new Shape("parent-above-root", "/../file.txt"),
+ new Shape("encoded-slash", "/dir/%2Ffile.txt"),
+ new Shape("space", "/dir/my file.txt"),
+ new Shape("encoded-space", "/dir/my%20file.txt"),
+ new Shape("hash", "/dir/file#1.txt"),
+ new Shape("question-mark", "/dir/file?1.txt"),
+ new Shape("plus", "/dir/file+1.txt"),
+ new Shape("backslash", "/dir\\file.txt"),
+ new Shape("leading-dots-in-name", "/dir/..hidden"),
+ new Shape("unicode", "/dir/naïve.txt"));
+
+ /** Parses a URI into a file name, the way the provider under test would. */
+ @FunctionalInterface
+ public interface NameParser {
+ FileName parse(String uri) throws Exception;
+ }
+
+ /**
+ * Run every shape through the parser.
+ *
+ * @return shape name to the resulting path, or {@link #ERROR} if the parser
rejected it
+ */
+ public static Map<String, String> paths(String baseUri, NameParser parser) {
+ return collect(baseUri, parser, FileName::getPath);
+ }
+
+ /**
+ * Run every shape through the parser.
+ *
+ * @return shape name to the resulting URI, or {@link #ERROR} if the parser
rejected it
+ */
+ public static Map<String, String> uris(String baseUri, NameParser parser) {
+ return collect(baseUri, parser, FileName::getURI);
+ }
+
+ private static Map<String, String> collect(
+ String baseUri, NameParser parser, Function<FileName, String> render) {
+ Map<String, String> actual = new LinkedHashMap<>();
+ for (Shape shape : SHAPES) {
+ try {
+ actual.put(shape.name(), render.apply(parser.parse(baseUri +
shape.suffix())));
+ } catch (Exception e) {
+ actual.put(shape.name(), ERROR);
+ }
+ }
+ return actual;
+ }
+
+ /** Assert the path each shape parses to, reporting every shape which moved
at once. */
+ public static void assertPaths(String baseUri, NameParser parser, String
expectedTable) {
+ assertEvery(paths(baseUri, parser), expectedTable, "path");
+ }
+
+ /** Assert the URI each shape parses to, reporting every shape which moved
at once. */
+ public static void assertUris(String baseUri, NameParser parser, String
expectedTable) {
+ assertEvery(uris(baseUri, parser), expectedTable, "URI");
+ }
+
+ /**
+ * One shape per line: the shape name, two or more spaces, the expected
value. Blank lines are
+ * ignored, and so is a trailing note introduced by a {@code #} which is
itself preceded by two or
+ * more spaces - the same separator the columns use, so that a {@code #}
inside an expected value
+ * stays part of the value.
+ */
+ private static Map<String, String> parseTable(String table) {
+ Map<String, String> expected = new LinkedHashMap<>();
+ for (String rawLine : table.split("\n")) {
+ String line = rawLine.replaceAll("\\s{2,}#.*$", "").strip();
+ if (line.isEmpty() || line.startsWith("#")) {
+ continue;
+ }
+ String[] cells = line.split("\\s{2,}", 2);
+ if (cells.length != 2) {
+ fail(
+ "table line needs a name and a value separated by two or more
spaces: \""
+ + line
+ + "\"");
+ }
+ expected.put(cells[0].strip(), cells[1].strip());
+ }
+ return expected;
+ }
+
+ private static void assertEvery(Map<String, String> actual, String table,
String what) {
+ Map<String, String> expected = parseTable(table);
+
+ List<String> unknown = new ArrayList<>(expected.keySet());
+ unknown.removeAll(actual.keySet());
+ if (!unknown.isEmpty()) {
+ fail("table has rows for shapes which do not exist: " + unknown);
+ }
+
+ List<Executable> checks = new ArrayList<>();
+ for (Shape shape : SHAPES) {
+ String name = shape.name();
+ if (!expected.containsKey(name)) {
+ checks.add(
+ () ->
+ fail(
+ "no expected "
+ + what
+ + " for shape \""
+ + name
+ + "\" ("
+ + shape.suffix()
+ + "), which parses to: "
+ + actual.get(name)));
+ continue;
+ }
+ checks.add(
+ () ->
+ assertEquals(
+ expected.get(name),
+ actual.get(name),
+ "shape \"" + name + "\" (" + shape.suffix() + ")"));
+ }
+ assertAll(what + " per shape", checks);
+ }
+
+ /**
+ * Assert that every shape which parses survives a trip through its own URI
unchanged, for the
+ * name itself and for every one of its parents.
+ *
+ * <p>This is the invariant a fix has to keep, and the one most easily lost.
A file name carries
+ * its root and its path separately, and only the root is handed on
untouched: {@code
+ * createName(absPath, type)} is called with an already normalised path on
every child listing and
+ * every {@link FileName#getParent()}. So anything a parser defends inside
the <em>path</em> is
+ * re-normalised away the moment a folder is listed or a parent is walked
to, while anything it
+ * puts in the <em>root</em> survives - which is why the Windows parser
hoists a UNC share into
+ * the root rather than guarding it in the path.
+ */
+ public static void assertUriRoundTrips(String baseUri, NameParser parser) {
+ List<Executable> checks = new ArrayList<>();
+ for (Shape shape : SHAPES) {
+ FileName name;
+ try {
+ name = parser.parse(baseUri + shape.suffix());
+ } catch (Exception e) {
+ continue; // a shape the parser rejects has nothing to round trip
+ }
+ for (FileName each = name; each != null; each = each.getParent()) {
+ FileName toCheck = each;
+ String role = each == name ? "" : " (parent of " + name.getURI() + ")";
+ checks.add(
+ () -> {
+ FileName reparsed = parser.parse(toCheck.getURI());
+ assertEquals(
+ toCheck.getPath(),
+ reparsed.getPath(),
+ "path of \"" + shape.name() + "\"" + role + " changed on
reparsing its URI");
+ assertEquals(
+ toCheck.getURI(),
+ reparsed.getURI(),
+ "URI of \"" + shape.name() + "\"" + role + " changed on
reparsing itself");
+ });
+ }
+ }
+ assertAll("URI round trip per shape", checks);
+ }
+
+ private VfsPathShapes() {}
+}
diff --git
a/plugins/tech/ftp/src/test/java/org/apache/hop/vfs/ftp/FtpPathShapeTest.java
b/plugins/tech/ftp/src/test/java/org/apache/hop/vfs/ftp/FtpPathShapeTest.java
new file mode 100644
index 0000000000..d838de7ece
--- /dev/null
+++
b/plugins/tech/ftp/src/test/java/org/apache/hop/vfs/ftp/FtpPathShapeTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.vfs.ftp;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.provider.VfsComponentContext;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.junit.vfs.VfsPathShapes;
+import org.apache.hop.vfs.ftp.metadata.FtpConnection;
+import org.junit.jupiter.api.Test;
+
+/**
+ * What the named FTP connection parser makes of the awkward path shapes in
{@link VfsPathShapes}.
+ *
+ * <p>This is the FTP twin of {@code SftpPathShapeTest}, and the two are
deliberately out of step
+ * for now: SFTP keeps a path rooted at a second slash, so that a z/OS MVS
data set can be
+ * addressed, and FTP still collapses every leading slash into one. z/OS FTP
names data sets the
+ * same {@code //A.B.C} way, so the same change belongs here - but the SFTP
fix reaches the wire
+ * through Hop's own fork of the Commons VFS SFTP provider, and there is no
such fork for FTP: the
+ * path sent to the server is built in {@code FtpFileObject}'s constructor,
which Hop does not own.
+ * The rows below say what FTP does today, so that whatever is done about that
is measurable.
+ */
+class FtpPathShapeTest {
+
+ private static final String BASE = "prod:/";
+
+ private final VfsPathShapes.NameParser parser = connectionParser();
+
+ private VfsPathShapes.NameParser connectionParser() {
+ FtpConnection connection = new FtpConnection();
+ connection.setName("prod");
+ connection.setServerName("mainframe.example.com");
+ connection.setServerPort("21");
+ connection.setUserName("hop");
+
+ FileSystemManager fsm = mock(FileSystemManager.class);
+ when(fsm.getSchemes()).thenReturn(new String[] {"prod"});
+ VfsComponentContext context = mock(VfsComponentContext.class);
+ when(context.getFileSystemManager()).thenReturn(fsm);
+
+ FtpConnectionFileNameParser nameParser =
+ new FtpConnectionFileNameParser(new Variables(), connection);
+ return uri -> nameParser.parseUri(context, null, uri);
+ }
+
+ @Test
+ void paths() {
+ VfsPathShapes.assertPaths(
+ BASE,
+ parser,
+ """
+ plain /dir/file.txt
+ leading-double-slash /dir/file.txt # an MVS data set, collapsed
- unlike SFTP
+ leading-triple-slash /dir/file.txt
+ interior-double-slash /dir/file.txt
+ trailing-slash /dir
+ root-only /
+ no-path /
+ dot-segment /dir/file.txt
+ parent-segment /dir/file.txt
+ parent-above-root error
+ encoded-slash /dir/file.txt
+ space /dir/my file.txt
+ encoded-space /dir/my file.txt
+ hash /dir/file#1.txt
+ question-mark /dir/file?1.txt
+ plus /dir/file+1.txt
+ backslash /dir/file.txt
+ leading-dots-in-name /dir/..hidden
+ unicode /dir/naïve.txt
+ """);
+ }
+
+ /** Every leading slash count lands on the same root, which is what SFTP no
longer does. */
+ @Test
+ void everySlashCountMeansTheSameRoot() throws Exception {
+ assertEquals("prod:///", parser.parse("prod://dir/file.txt").getRootURI());
+ assertEquals("prod:///",
parser.parse("prod:///dir/file.txt").getRootURI());
+ assertEquals("prod:///",
parser.parse("prod:////dir/file.txt").getRootURI());
+ assertEquals("prod:///dir/file.txt",
parser.parse("prod:////dir/file.txt").getURI());
+ }
+
+ @Test
+ void uriRoundTrips() {
+ VfsPathShapes.assertUriRoundTrips(BASE, parser);
+ }
+}
diff --git
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnectionFileName.java
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnectionFileName.java
index b79ecfe95a..7c4da2bdfc 100644
---
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnectionFileName.java
+++
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnectionFileName.java
@@ -20,6 +20,7 @@ import org.apache.commons.vfs2.FileName;
import org.apache.commons.vfs2.FileType;
import org.apache.commons.vfs2.provider.GenericFileName;
import org.apache.hop.vfs.sftp.metadata.SftpConnection;
+import org.apache.hop.vfs.sftp.provider.IRootPrefixFileName;
/**
* The name of a file behind a named SFTP connection.
@@ -30,7 +31,9 @@ import org.apache.hop.vfs.sftp.metadata.SftpConnection;
* point at today. That also means a URI taken from a file object can be
handed straight back to
* VFS.
*/
-public class SftpConnectionFileName extends GenericFileName {
+public class SftpConnectionFileName extends GenericFileName implements
IRootPrefixFileName {
+
+ private final String rootPrefix;
protected SftpConnectionFileName(
String scheme,
@@ -39,7 +42,8 @@ public class SftpConnectionFileName extends GenericFileName {
String userName,
String password,
String path,
- FileType type) {
+ FileType type,
+ String rootPrefix) {
super(
scheme,
hostName,
@@ -49,17 +53,34 @@ public class SftpConnectionFileName extends GenericFileName
{
password,
path,
type == null ? FileType.IMAGINARY : type);
+ this.rootPrefix = rootPrefix == null ? "" : rootPrefix;
+ }
+
+ @Override
+ public String getRootPrefix() {
+ return rootPrefix;
}
- /** {@code <connection name>://}, without server or credentials. */
+ /**
+ * {@code <connection name>://}, without server or credentials, plus the
second slash of a path
+ * rooted at one - so {@code prod:////A.B.C} reads back as it was typed.
+ */
@Override
protected void appendRootUri(StringBuilder buffer, boolean addPassword) {
- buffer.append(getScheme()).append("://");
+ buffer.append(getScheme()).append("://").append(rootPrefix);
}
+ /** Children and parents keep the root they were reached through. */
@Override
public FileName createName(String absPath, FileType type) {
return new SftpConnectionFileName(
- getScheme(), getHostName(), getPort(), getUserName(), getPassword(),
absPath, type);
+ getScheme(),
+ getHostName(),
+ getPort(),
+ getUserName(),
+ getPassword(),
+ absPath,
+ type,
+ rootPrefix);
}
}
diff --git
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnectionFileNameParser.java
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnectionFileNameParser.java
index fb62cb654e..5b197c9a69 100644
---
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnectionFileNameParser.java
+++
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnectionFileNameParser.java
@@ -27,6 +27,7 @@ import org.apache.hop.core.Const;
import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.vfs.sftp.metadata.SftpConnection;
+import org.apache.hop.vfs.sftp.provider.SftpFileNameParser;
/**
* Turns {@code <connection name>://path} (any number of slashes after the
scheme) into a {@link
@@ -51,6 +52,7 @@ public class SftpConnectionFileNameParser extends
AbstractFileNameParser {
}
StringBuilder buffer = new StringBuilder();
+ String rootPrefix = "";
String scheme =
UriParser.extractScheme(context.getFileSystemManager().getSchemes(),
uri, buffer);
if (scheme == null) {
@@ -68,15 +70,17 @@ public class SftpConnectionFileNameParser extends
AbstractFileNameParser {
+ scheme
+ "\"");
} else {
- // Everything after the scheme is a path on the server: prod://tmp/x,
prod:///tmp/x and
- // prod:////tmp/x all point at /tmp/x.
+ // Everything after the scheme is a path on the server. The two slashes
behind the scheme are
+ // the separator, as in any URI, and what is left is the path:
prod://tmp/x and prod:///tmp/x
+ // both point at /tmp/x. A path which then still starts with two slashes
keeps the second one
+ // - prod:////A.B.C is the MVS data set //A.B.C, the same way
file:////server/share is a UNC
+ // share - and three or more collapse to one, as they do everywhere else.
//
- int slashes = 0;
- while (slashes < buffer.length() && buffer.charAt(slashes) == '/') {
- slashes++;
+ buffer.delete(0, Math.min(2, leadingSlashes(buffer)));
+ if (buffer.isEmpty() || buffer.charAt(0) != '/') {
+ buffer.insert(0, '/');
}
- buffer.delete(0, slashes);
- buffer.insert(0, '/');
+ rootPrefix = SftpFileNameParser.resolveRootPrefix(buffer, uri, base);
}
UriParser.canonicalizePath(buffer, 0, buffer.length(), this);
@@ -94,6 +98,15 @@ public class SftpConnectionFileNameParser extends
AbstractFileNameParser {
String user = variables.resolve(connection.getUsername());
String password = Utils.resolvePassword(variables,
connection.getPassword());
- return new SftpConnectionFileName(connectionName, host, port, user,
password, path, fileType);
+ return new SftpConnectionFileName(
+ connectionName, host, port, user, password, path, fileType,
rootPrefix);
+ }
+
+ private static int leadingSlashes(StringBuilder buffer) {
+ int slashes = 0;
+ while (slashes < buffer.length() && buffer.charAt(slashes) == '/') {
+ slashes++;
+ }
+ return slashes;
}
}
diff --git
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnections.java
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnections.java
index 3ffca04ae4..e119596e47 100644
---
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnections.java
+++
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/SftpConnections.java
@@ -61,7 +61,7 @@ public final class SftpConnections {
/**
* Build the URI of a file behind a named connection. Both the folder and
the file name are
- * optional.
+ * optional. A path rooted at a second slash, such as the z/OS data set
{@code //A.B.C}, keeps it.
*/
public static String buildUri(SftpConnection connection, String folder,
String filename) {
StringBuilder uri = new StringBuilder(getBaseUri(connection));
@@ -72,10 +72,16 @@ public final class SftpConnections {
}
path += filename.replace('\\', '/').trim();
}
- while (path.startsWith("/")) {
- path = path.substring(1);
+ // One leading slash or three and more mean the same, but exactly two is a
path of its own,
+ // which the parser reads from prod:////A.B.C.
+ int slashes = 0;
+ while (slashes < path.length() && path.charAt(slashes) == '/') {
+ slashes++;
}
- return uri.append(path).toString();
+ if (slashes == 2) {
+ uri.append("//");
+ }
+ return uri.append(path.substring(slashes)).toString();
}
/**
diff --git
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/IRootPrefixFileName.java
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/IRootPrefixFileName.java
new file mode 100644
index 0000000000..7bbd1a0c9a
--- /dev/null
+++
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/IRootPrefixFileName.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.vfs.sftp.provider;
+
+/**
+ * A file name which remembers that its path began with a second slash.
+ *
+ * <p>POSIX gives a path starting with exactly two slashes a meaning of its
own, and z/OS uses that
+ * meaning: {@code /A.B.C} is a USS path and {@code //A.B.C} is an MVS data
set. Three or more
+ * slashes are the same as one, again per POSIX.
+ *
+ * <p>The extra slash cannot be kept in the path. Commons VFS normalises a
path element by element
+ * and drops the empty element a {@code //} amounts to, and it does that again
on every child
+ * listing and every {@link org.apache.commons.vfs2.FileName#getParent()},
because {@code
+ * createName(absPath, type)} is handed an already normalised path. Only the
root survives being
+ * passed along, which is why the Windows parser hoists a UNC share into the
root instead of
+ * guarding it in the path. This is the same hoist: the path stays an ordinary
{@code /A.B.C} and
+ * the prefix rides along in the root.
+ */
+public interface IRootPrefixFileName {
+
+ /**
+ * @return {@code "/"} when the path is rooted at a second slash, {@code ""}
otherwise
+ */
+ String getRootPrefix();
+}
diff --git
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileName.java
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileName.java
new file mode 100644
index 0000000000..f8a648a3c6
--- /dev/null
+++
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileName.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.vfs.sftp.provider;
+
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileType;
+import org.apache.commons.vfs2.provider.URLFileName;
+
+/**
+ * The name of a file on an SFTP server, which remembers whether the user
rooted it at a second
+ * slash: {@code sftp://host/A.B.C} and {@code sftp://host//A.B.C} are two
different files, and on
+ * z/OS the second is an MVS data set.
+ *
+ * @see IRootPrefixFileName
+ */
+public class SftpFileName extends URLFileName implements IRootPrefixFileName {
+
+ private final String rootPrefix;
+
+ public SftpFileName(
+ String scheme,
+ String hostName,
+ int port,
+ int defaultPort,
+ String userName,
+ String password,
+ String path,
+ FileType type,
+ String queryString,
+ String rootPrefix) {
+ super(scheme, hostName, port, defaultPort, userName, password, path, type,
queryString);
+ this.rootPrefix = rootPrefix == null ? "" : rootPrefix;
+ }
+
+ @Override
+ public String getRootPrefix() {
+ return rootPrefix;
+ }
+
+ /** Children and parents keep the root they were reached through. */
+ @Override
+ public FileName createName(String absPath, FileType type) {
+ return new SftpFileName(
+ getScheme(),
+ getHostName(),
+ getPort(),
+ getDefaultPort(),
+ getUserName(),
+ getPassword(),
+ absPath,
+ type,
+ getQueryString(),
+ rootPrefix);
+ }
+
+ /**
+ * The extra slash belongs to the root, so it lands between the host and the
path and the URI
+ * reads back as the user typed it: {@code sftp://host//A.B.C}.
+ */
+ @Override
+ protected void appendRootUri(StringBuilder buffer, boolean addPassword) {
+ super.appendRootUri(buffer, addPassword);
+ buffer.append(rootPrefix);
+ }
+}
diff --git
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileNameParser.java
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileNameParser.java
index 7d85b65821..ca677bc6a3 100644
---
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileNameParser.java
+++
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileNameParser.java
@@ -16,8 +16,13 @@
*/
package org.apache.hop.vfs.sftp.provider;
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileType;
import org.apache.commons.vfs2.provider.FileNameParser;
import org.apache.commons.vfs2.provider.URLFileNameParser;
+import org.apache.commons.vfs2.provider.UriParser;
+import org.apache.commons.vfs2.provider.VfsComponentContext;
/** Implementation for SFTP. Sets the default port to 22. */
public class SftpFileNameParser extends URLFileNameParser {
@@ -39,4 +44,87 @@ public class SftpFileNameParser extends URLFileNameParser {
public SftpFileNameParser() {
super(DEFAULT_PORT);
}
+
+ /**
+ * As {@link URLFileNameParser#parseUri}, but a path rooted at a second
slash keeps it.
+ *
+ * <p>The prefix is taken off before the path is canonicalized and
normalised, because
+ * normalisation is what would drop it, and it is put in the root of a
{@link SftpFileName} rather
+ * than left in the path so that it survives every child and every parent
derived from this name.
+ */
+ @Override
+ public FileName parseUri(VfsComponentContext context, FileName base, String
fileName)
+ throws FileSystemException {
+ StringBuilder name = new StringBuilder();
+
+ Authority auth = extractToPath(context, fileName, name);
+ String queryString = UriParser.extractQueryString(name);
+ String rootPrefix = resolveRootPrefix(name, fileName, base);
+
+ UriParser.canonicalizePath(name, 0, name.length(), this);
+ UriParser.fixSeparators(name);
+ FileType fileType = UriParser.normalisePath(name);
+
+ return new SftpFileName(
+ auth.getScheme(),
+ auth.getHostName(),
+ auth.getPort(),
+ getDefaultPort(),
+ auth.getUserName(),
+ auth.getPassword(),
+ name.toString(),
+ fileType,
+ queryString,
+ rootPrefix);
+ }
+
+ /**
+ * The prefix for a path, taken from the base name whenever this URI was
built out of it.
+ *
+ * <p>The textual rule cannot be trusted on a derived name. {@link
+ * org.apache.commons.vfs2.impl.DefaultFileSystemManager#resolveName}
rebuilds every child, every
+ * parent and every root lookup as {@code base.getRootURI() + path}, and
since the root URI ends
+ * in a separator and the path begins with one, that concatenation
manufactures a {@code //} at
+ * the boundary which is indistinguishable from one a user typed. It is also
the reason Commons
+ * VFS collapses {@code //} at all. A URI built that way starts with the
root URI of the base it
+ * came from, and the base already knows what it is rooted at, so ask it.
+ *
+ * <p>The one thing this cannot tell apart is a full URI handed to {@code
+ * FileObject.resolveFile(String)} on a file of the same file system, which
looks exactly like a
+ * derived name. Resolving from a base is for relative paths; an absolute
URI belongs to {@code
+ * HopVfs.getFileObject}, which has no base and is read as written.
+ *
+ * @param path the path, with the scheme and the authority already taken off
+ * @param uri the whole URI being parsed
+ * @param base the name this one is being derived from, null if there is none
+ */
+ public static String resolveRootPrefix(StringBuilder path, String uri,
FileName base) {
+ if (base instanceof IRootPrefixFileName prefixed &&
uri.startsWith(base.getRootURI())) {
+ return prefixed.getRootPrefix();
+ }
+ return extractRootPrefix(path);
+ }
+
+ /**
+ * Takes the second slash off a path which starts with exactly two, leaving
an ordinary absolute
+ * path behind.
+ *
+ * <p>Exactly two is what POSIX gives a meaning of its own and what z/OS
reads as an MVS data set;
+ * three or more mean the same as one, so those are left for the normaliser
to collapse. Only a
+ * literal slash counts - a {@code %2F} is an escaped slash, which is to say
not a separator at
+ * all.
+ *
+ * @param path the path, with the scheme and the authority already taken off
+ * @return {@code "/"} if a second slash was taken off, {@code ""} otherwise
+ */
+ public static String extractRootPrefix(StringBuilder path) {
+ if (path.length() >= 2
+ && path.charAt(0) == '/'
+ && path.charAt(1) == '/'
+ && (path.length() == 2 || path.charAt(2) != '/')) {
+ path.deleteCharAt(0);
+ return "/";
+ }
+ return "";
+ }
}
diff --git
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileObject.java
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileObject.java
index d7e611126d..b17101d953 100644
---
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileObject.java
+++
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileObject.java
@@ -104,7 +104,28 @@ public class SftpFileObject extends
AbstractFileObject<SftpFileSystem> {
protected SftpFileObject(final AbstractFileName fileName, final
SftpFileSystem fileSystem)
throws FileSystemException {
super(fileName, fileSystem);
- relPath =
UriParser.decode(fileSystem.getRootName().getRelativeName(fileName));
+ relPath = remotePath(fileName, fileSystem);
+ }
+
+ /**
+ * The path to ask the server about. Normally relative to the working
directory of the channel,
+ * which is what every other SFTP server expects.
+ *
+ * <p>A name rooted at a second slash - see {@link IRootPrefixFileName} -
instead goes out
+ * absolute and with that slash put back, so {@code
sftp://host//EDIJP.NAB.CRERESP/AAMIPI} asks
+ * for {@code //EDIJP.NAB.CRERESP/AAMIPI} and a z/OS server reads an MVS
data set rather than a
+ * USS path. Absolute is deliberate: jsch passes a path starting with a
slash to the server
+ * untouched, while a relative one is resolved against a working directory
the server would have
+ * normalised for us.
+ */
+ private static String remotePath(final AbstractFileName fileName, final
SftpFileSystem fileSystem)
+ throws FileSystemException {
+ final String rootPrefix =
+ fileName instanceof IRootPrefixFileName prefixed ?
prefixed.getRootPrefix() : "";
+ if (!rootPrefix.isEmpty()) {
+ return rootPrefix + UriParser.decode(fileName.getPath());
+ }
+ return
UriParser.decode(fileSystem.getRootName().getRelativeName(fileName));
}
/** Creates this file as a folder. */
@@ -493,9 +514,9 @@ public class SftpFileObject extends
AbstractFileObject<SftpFileSystem> {
final ChannelSftp channel = getAbstractFileSystem().getChannel();
// Using InputStream directly from the channel
// is much faster than the memory method.
+ // relPath, like every other request: the path of the name has lost a
second slash.
try {
- return new SftpInputStream(
- channel, channel.get(getName().getPathDecoded(), null, filePointer));
+ return new SftpInputStream(channel, channel.get(relPath, null,
filePointer));
} catch (final SftpException e) {
putChannel(channel);
throw new FileSystemException(e);
diff --git
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileSystem.java
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileSystem.java
index da9816d182..54e0f69127 100644
---
a/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileSystem.java
+++
b/plugins/tech/sftp/src/main/java/org/apache/hop/vfs/sftp/provider/SftpFileSystem.java
@@ -216,8 +216,15 @@ public class SftpFileSystem extends AbstractFileSystem {
channel.connect(DurationUtils.toMillisInt(connectTimeout));
final Boolean userDirIsRoot =
SftpFileSystemConfigBuilder.getInstance().getUserDirIsRoot(getFileSystemOptions());
+ // A file system rooted at a second slash sends every path absolute,
so the working
+ // directory plays no part - and a "cd /" is a USS path a z/OS server
need not have at all.
+ final boolean rootedAtSecondSlash =
+ getRootName() instanceof IRootPrefixFileName prefixed
+ && !prefixed.getRootPrefix().isEmpty();
final String workingDirectory = getRootName().getPath();
- if (workingDirectory != null && (userDirIsRoot == null ||
!userDirIsRoot.booleanValue())) {
+ if (workingDirectory != null
+ && !rootedAtSecondSlash
+ && (userDirIsRoot == null || !userDirIsRoot.booleanValue())) {
try {
channel.cd(workingDirectory);
} catch (final SftpException e) {
diff --git
a/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/SftpConnectionsTest.java
b/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/SftpConnectionsTest.java
index cf85c8d1fb..b540d20523 100644
---
a/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/SftpConnectionsTest.java
+++
b/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/SftpConnectionsTest.java
@@ -54,6 +54,27 @@ class SftpConnectionsTest {
SftpConnections.buildUri(connection, "\\tmp\\upload", "file.txt"));
}
+ /**
+ * Exactly two leading slashes is a path of its own - on z/OS {@code
//A.B.C} is an MVS data set
+ * and {@code /A.B.C} a USS path - so it reaches the parser as {@code
prod:////A.B.C}. Three or
+ * more still mean the same as one.
+ */
+ @Test
+ void testBuildUriKeepsAPathRootedAtASecondSlash() {
+ assertEquals(
+ "prod:////EDIJP.NAB.CRERESP/AAMIPI",
+ SftpConnections.buildUri(connection, "//EDIJP.NAB.CRERESP", "AAMIPI"));
+ assertEquals(
+ "prod:////EDIJP.NAB.CRERESP",
+ SftpConnections.buildUri(connection, "//EDIJP.NAB.CRERESP", null));
+ assertEquals(
+ "prod:////EDIJP.NAB.CRERESP/AAMIPI",
+ SftpConnections.buildUri(connection, null,
"//EDIJP.NAB.CRERESP/AAMIPI"));
+ assertEquals(
+ "prod://tmp/upload/file.txt",
+ SftpConnections.buildUri(connection, "///tmp/upload", "file.txt"));
+ }
+
private SftpConnection connection() {
SftpConnection sftpConnection = new SftpConnection();
sftpConnection.setName("prod");
diff --git
a/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/SftpPathShapeTest.java
b/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/SftpPathShapeTest.java
new file mode 100644
index 0000000000..889f6e7230
--- /dev/null
+++
b/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/SftpPathShapeTest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hop.vfs.sftp;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.commons.vfs2.provider.VfsComponentContext;
+import org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.junit.vfs.VfsPathShapes;
+import org.apache.hop.vfs.sftp.metadata.SftpConnection;
+import org.apache.hop.vfs.sftp.provider.SftpFileProvider;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+
+/**
+ * What the two SFTP file name parsers make of the awkward path shapes in
{@link VfsPathShapes}.
+ *
+ * <p>Neither parser needs a server: this is name parsing only, so the tables
below say what a path
+ * turns into before anything is ever sent over the wire.
+ */
+class SftpPathShapeTest {
+
+ private static DefaultFileSystemManager manager;
+
+ @BeforeAll
+ static void setUp() throws Exception {
+ manager = new DefaultFileSystemManager();
+ manager.addProvider("sftp", new SftpFileProvider());
+ manager.addProvider("file", new DefaultLocalFileProvider());
+ manager.init();
+ }
+
+ @AfterAll
+ static void tearDown() {
+ if (manager != null) {
+ manager.close();
+ }
+ }
+
+ /**
+ * The parser behind a plain {@code sftp://} URI. The path of a name is
always an ordinary
+ * absolute path: a path rooted at a second slash keeps it in the
<em>root</em>, so none of the
+ * rows below move when one is used.
+ */
+ @Nested
+ @DisplayName("sftp:// URIs")
+ class RawUris {
+
+ private static final String BASE = "sftp://user@host:22";;
+ private final VfsPathShapes.NameParser parser = uri ->
manager.resolveURI(uri);
+
+ @Test
+ void paths() {
+ VfsPathShapes.assertPaths(
+ BASE,
+ parser,
+ """
+ plain /dir/file.txt
+ leading-double-slash /dir/file.txt # the data set: the extra
slash is in the root
+ leading-triple-slash /dir/file.txt
+ interior-double-slash /dir/file.txt
+ trailing-slash /dir
+ root-only /
+ no-path /
+ dot-segment /dir/file.txt
+ parent-segment /dir/file.txt
+ parent-above-root error
+ encoded-slash /dir/file.txt # %2F is decoded before it
is normalised away
+ space /dir/my file.txt
+ encoded-space /dir/my file.txt
+ hash /dir/file#1.txt
+ question-mark /dir/file # "?1.txt" is taken for a
URL query
+ plus /dir/file+1.txt
+ backslash /dir/file.txt
+ leading-dots-in-name /dir/..hidden
+ unicode /dir/naïve.txt
+ """);
+ }
+
+ /** The second slash rides in the root, which is what lets it survive
being derived from. */
+ @Test
+ void theSecondSlashIsInTheRoot() throws Exception {
+ assertEquals("sftp://user@host/";, parser.parse(BASE +
"/dir/file.txt").getRootURI());
+ assertEquals("sftp://user@host//";, parser.parse(BASE +
"//dir/file.txt").getRootURI());
+ assertEquals(
+ "sftp://user@host//dir/file.txt";, parser.parse(BASE +
"//dir/file.txt").getURI());
+ }
+
+ @Test
+ void uriRoundTrips() {
+ VfsPathShapes.assertUriRoundTrips(BASE, parser);
+ }
+ }
+
+ /**
+ * The parser behind a named SFTP connection, which is Hop's own code. The
two slashes behind the
+ * scheme are the separator, so {@code prod://tmp/x} and {@code
prod:///tmp/x} both mean {@code
+ * /tmp/x}; a third pair makes {@code prod:////A.B.C} the data set {@code
//A.B.C}.
+ */
+ @Nested
+ @DisplayName("named connection URIs")
+ class NamedConnection {
+
+ private static final String BASE = "prod:/";
+
+ private final VfsPathShapes.NameParser parser = namedConnectionParser();
+
+ private VfsPathShapes.NameParser namedConnectionParser() {
+ SftpConnection connection = new SftpConnection();
+ connection.setName("prod");
+ connection.setServerName("mainframe.example.com");
+ connection.setServerPort("22");
+ connection.setUsername("hop");
+
+ FileSystemManager fsm = mock(FileSystemManager.class);
+ when(fsm.getSchemes()).thenReturn(new String[] {"prod"});
+ VfsComponentContext context = mock(VfsComponentContext.class);
+ when(context.getFileSystemManager()).thenReturn(fsm);
+
+ SftpConnectionFileNameParser nameParser =
+ new SftpConnectionFileNameParser(new Variables(), connection);
+ return uri -> nameParser.parseUri(context, null, uri);
+ }
+
+ @Test
+ void paths() {
+ VfsPathShapes.assertPaths(
+ BASE,
+ parser,
+ """
+ plain /dir/file.txt
+ leading-double-slash /dir/file.txt # the data set: the extra
slash is in the root
+ leading-triple-slash /dir/file.txt
+ interior-double-slash /dir/file.txt
+ trailing-slash /dir
+ root-only /
+ no-path /
+ dot-segment /dir/file.txt
+ parent-segment /dir/file.txt
+ parent-above-root error
+ encoded-slash /dir/file.txt
+ space /dir/my file.txt
+ encoded-space /dir/my file.txt
+ hash /dir/file#1.txt
+ question-mark /dir/file?1.txt # kept, unlike the sftp://
parser above
+ plus /dir/file+1.txt
+ backslash /dir/file.txt
+ leading-dots-in-name /dir/..hidden
+ unicode /dir/naïve.txt
+ """);
+ }
+
+ /** The server stays out of the URI, so the connection name is the whole
root. */
+ @Test
+ void theConnectionNameIsTheRoot() throws Exception {
+ assertEquals("prod:///", parser.parse(BASE +
"/dir/file.txt").getRootURI());
+ assertEquals("prod:///dir/file.txt", parser.parse(BASE +
"/dir/file.txt").getURI());
+ assertEquals("prod:////", parser.parse(BASE +
"///dir/file.txt").getRootURI());
+ assertEquals("prod:////dir/file.txt", parser.parse(BASE +
"///dir/file.txt").getURI());
+ }
+
+ @Test
+ void uriRoundTrips() {
+ VfsPathShapes.assertUriRoundTrips(BASE, parser);
+ }
+ }
+}
diff --git
a/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/metadata/SftpConnectionJsonTest.java
b/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/metadata/SftpConnectionJsonTest.java
new file mode 100644
index 0000000000..0119640065
--- /dev/null
+++
b/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/metadata/SftpConnectionJsonTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.vfs.sftp.metadata;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Path;
+import org.apache.hop.core.Const;
+import org.apache.hop.core.encryption.Encr;
+import org.apache.hop.core.encryption.HopTwoWayPasswordEncoder;
+import org.apache.hop.core.encryption.TwoWayPasswordEncoderPlugin;
+import org.apache.hop.core.encryption.TwoWayPasswordEncoderPluginType;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.util.HopMetadataUtil;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * A connection has to survive the trip through the JSON files it actually
lives in. The in-memory
+ * provider the other tests use is not the same thing: it hands the object
straight back, so it
+ * agrees with whatever the class does.
+ */
+class SftpConnectionJsonTest {
+
+ @TempDir private Path metadataFolder;
+
+ private IVariables variables;
+
+ @BeforeAll
+ static void initEncryption() throws Exception {
+ PluginRegistry.getInstance()
+ .registerPluginClass(
+ HopTwoWayPasswordEncoder.class.getName(),
+ TwoWayPasswordEncoderPluginType.class,
+ TwoWayPasswordEncoderPlugin.class);
+ Encr.init("Hop");
+ }
+
+ @BeforeEach
+ void pointAtTheMetadataFolder() {
+ variables = new Variables();
+ variables.setVariable(Const.HOP_METADATA_FOLDER,
metadataFolder.toString());
+ }
+
+ @Test
+ @DisplayName("Every setting survives a save and a load")
+ void everySettingRoundTrips() throws Exception {
+ SftpConnection connection = connection();
+ connection.setUserDirIsRoot(true);
+ connection.setCompression("zlib");
+
+ SftpConnection loaded = saveAndLoad(connection);
+
+ assertEquals("mainframe.example.com", loaded.getServerName());
+ assertEquals("22", loaded.getServerPort());
+ assertEquals("hop", loaded.getUsername());
+ assertTrue(loaded.isUserDirIsRoot());
+ assertEquals("zlib", loaded.getCompression());
+ }
+
+ private SftpConnection connection() {
+ SftpConnection connection = new SftpConnection();
+ connection.setName("mainframe");
+ connection.setServerName("mainframe.example.com");
+ connection.setServerPort("22");
+ connection.setUsername("hop");
+ return connection;
+ }
+
+ private SftpConnection saveAndLoad(SftpConnection connection) throws
Exception {
+ IHopMetadataProvider provider =
HopMetadataUtil.getStandardHopMetadataProvider(variables);
+ provider.getSerializer(SftpConnection.class).save(connection);
+ SftpConnection loaded =
provider.getSerializer(SftpConnection.class).load(connection.getName());
+ assertNotNull(loaded, "the connection could not be read back");
+ return loaded;
+ }
+}
diff --git
a/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/provider/SftpDoubleSlashPathTest.java
b/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/provider/SftpDoubleSlashPathTest.java
new file mode 100644
index 0000000000..59588fab99
--- /dev/null
+++
b/plugins/tech/sftp/src/test/java/org/apache/hop/vfs/sftp/provider/SftpDoubleSlashPathTest.java
@@ -0,0 +1,441 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.vfs.sftp.provider;
+
+import static org.apache.commons.vfs2.VfsTestUtils.getTestDirectoryFile;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.jcraft.jsch.TestIdentityRepositoryFactory;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.commons.vfs2.NameScope;
+import org.apache.commons.vfs2.RandomAccessContent;
+import org.apache.commons.vfs2.auth.StaticUserAuthenticator;
+import org.apache.commons.vfs2.impl.DefaultFileSystemConfigBuilder;
+import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
+import org.apache.commons.vfs2.provider.VfsComponentContext;
+import org.apache.commons.vfs2.provider.local.DefaultLocalFileProvider;
+import org.apache.commons.vfs2.provider.local.GenericFileNameParser;
+import org.apache.commons.vfs2.provider.local.LocalFileNameParser;
+import org.apache.commons.vfs2.provider.local.WindowsFileNameParser;
+import org.apache.commons.vfs2.util.RandomAccessMode;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.vfs.sftp.SftpConnectionFileNameParser;
+import org.apache.hop.vfs.sftp.SftpConnectionFileProvider;
+import org.apache.hop.vfs.sftp.SftpConnections;
+import org.apache.hop.vfs.sftp.metadata.SftpConnection;
+import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
+import org.apache.sshd.server.SshServer;
+import org.apache.sshd.server.channel.ChannelSession;
+import org.apache.sshd.server.command.Command;
+import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
+import org.apache.sshd.sftp.server.SftpSubsystem;
+import org.apache.sshd.sftp.server.SftpSubsystemFactory;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+/**
+ * A mainframe SFTP server (z/OS) distinguishes a USS path {@code /A.B.C} from
an MVS data set
+ * {@code //A.B.C}: the leading double slash is part of the name, not
redundant punctuation. Commons
+ * VFS collapses it, so such a data set can not be addressed from Hop at all.
+ *
+ * <p>The collapse happens in {@link
org.apache.commons.vfs2.provider.UriParser#normalisePath} which
+ * every {@link org.apache.commons.vfs2.provider.AbstractFileNameParser}
calls: it walks the path
+ * element by element and "axes" every empty element, which is exactly what a
{@code //} is. The
+ * local provider escapes this for Windows UNC names ({@code \\host\share})
because {@link
+ * org.apache.commons.vfs2.provider.local.WindowsFileNameParser} pulls {@code
//host/share} out of
+ * the path and into the <em>root prefix</em> before normalisation runs. The
SFTP name parser has no
+ * such escape: {@link SftpFileNameParser} extends {@link
+ * org.apache.commons.vfs2.provider.URLFileNameParser}, which puts only the
host and the port in the
+ * root, so the whole {@code //A.B.C} is handed to the normaliser as a path.
+ *
+ * <p>Hop keeps the second slash by hoisting it into the <em>root</em> of the
file name, the same
+ * escape the Windows parser uses for a UNC share, so the path itself stays an
ordinary {@code
+ * /A.B.C} and survives every child and every parent derived from it.
+ *
+ * @see <a
href="https://www.ibm.com/docs/en/zos/latest?topic=ftp-mvs-data-sets";>z/OS:
naming MVS
+ * data sets</a>
+ */
+class SftpDoubleSlashPathTest {
+
+ private static final String USER = "testuser";
+ private static final String PASSWORD = "testpass";
+
+ /** The data set the user is after, in the four spellings that were tried. */
+ private static final String DATA_SET = "//EDIJP.NAB.CRERESP/AAMIPI";
+
+ /** The scheme of the named connection pointing at the test server. */
+ private static final String NAMED_CONNECTION = "mainframe";
+
+ /** Raw remote paths as the SFTP server received them, oldest first. */
+ private static final List<String> REQUESTED_PATHS =
+ Collections.synchronizedList(new ArrayList<>());
+
+ private static SshServer sshServer;
+ private static int serverPort;
+ private static DefaultFileSystemManager manager;
+
+ @BeforeAll
+ static void setUp() throws Exception {
+ sshServer = SshServer.setUpDefaultServer();
+ sshServer.setPort(0);
+ final Path hostKey = Files.createTempFile("sshd-double-slash-key", ".ser");
+ hostKey.toFile().deleteOnExit();
+ final SimpleGeneratorHostKeyProvider keyProvider = new
SimpleGeneratorHostKeyProvider(hostKey);
+ keyProvider.setAlgorithm("RSA");
+ sshServer.setKeyPairProvider(keyProvider);
+ sshServer.setPasswordAuthenticator(
+ (user, pass, session) -> USER.equals(user) && PASSWORD.equals(pass));
+ sshServer.setSubsystemFactories(Collections.singletonList(new
RecordingSftpSubsystemFactory()));
+ final File homeDir = getTestDirectoryFile();
+ sshServer.setFileSystemFactory(new
VirtualFileSystemFactory(homeDir.toPath().toAbsolutePath()));
+ sshServer.start();
+ serverPort = sshServer.getPort();
+
+ manager = new DefaultFileSystemManager();
+ manager.addProvider("sftp", new SftpFileProvider());
+ manager.addProvider("file", new DefaultLocalFileProvider());
+ manager.addProvider(
+ NAMED_CONNECTION, new SftpConnectionFileProvider(new Variables(),
namedConnection()));
+ manager.init();
+ }
+
+ /** A named connection to the test server; the password comes from {@link
#authOptions()}. */
+ private static SftpConnection namedConnection() {
+ final SftpConnection connection = new SftpConnection();
+ connection.setName(NAMED_CONNECTION);
+ connection.setServerName("localhost");
+ connection.setServerPort(Integer.toString(serverPort));
+ connection.setUsername(USER);
+ return connection;
+ }
+
+ @AfterAll
+ static void tearDown() {
+ if (manager != null) {
+ manager.close();
+ }
+ if (sshServer != null) {
+ final Thread stopper = new Thread(() -> IOUtils.closeQuietly(sshServer),
"sshd-stop");
+ stopper.setDaemon(true);
+ stopper.start();
+ try {
+ stopper.join(5000);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ private static String baseUri() {
+ return String.format("sftp://%s@localhost:%d";, USER, serverPort);
+ }
+
+ private static FileSystemOptions authOptions() throws FileSystemException {
+ final FileSystemOptions opts = new FileSystemOptions();
+ DefaultFileSystemConfigBuilder.getInstance()
+ .setUserAuthenticator(opts, new StaticUserAuthenticator(null, USER,
PASSWORD));
+ final SftpFileSystemConfigBuilder builder =
SftpFileSystemConfigBuilder.getInstance();
+ builder.setStrictHostKeyChecking(opts, "no");
+ builder.setUserInfo(opts, new TrustEveryoneUserInfo());
+ builder.setIdentityRepositoryFactory(opts, new
TestIdentityRepositoryFactory());
+ builder.setConnectTimeout(opts, Duration.ofSeconds(60));
+ builder.setSessionTimeout(opts, Duration.ofSeconds(60));
+ return opts;
+ }
+
+ /** Resolve a name through the SFTP provider without connecting to anything.
*/
+ private static FileName parse(String uri) throws FileSystemException {
+ return manager.resolveURI(uri);
+ }
+
+ /** Ask the server for a file and return every raw path it was asked about.
*/
+ private static List<String> pathsSentToServer(String uri) throws
FileSystemException {
+ REQUESTED_PATHS.clear();
+ try (FileObject file = manager.resolveFile(uri, authOptions())) {
+ file.exists();
+ }
+ return new ArrayList<>(REQUESTED_PATHS);
+ }
+
+ @Nested
+ @DisplayName("An ordinary path is left alone")
+ class OrdinaryPaths {
+
+ /**
+ * The path of a name is always an ordinary absolute path, the data set
spelling included - the
+ * second slash lives in the root, not here. Three slashes, a {@code %2F}
and a {@code .}
+ * segment all mean the plain USS path.
+ */
+ @ParameterizedTest(name = "{0}")
+ @ValueSource(
+ strings = {
+ "//EDIJP.NAB.CRERESP/AAMIPI",
+ "///EDIJP.NAB.CRERESP/AAMIPI",
+ "/\\/\\/EDIJP.NAB.CRERESP/AAMIPI",
+ "/%2F%2FEDIJP.NAB.CRERESP/AAMIPI",
+ "/./EDIJP.NAB.CRERESP/AAMIPI"
+ })
+ void thePathIsAlwaysAnOrdinaryAbsolutePath(String path) throws Exception {
+ assertEquals("/EDIJP.NAB.CRERESP/AAMIPI", parse(baseUri() +
path).getPath());
+ }
+
+ /** The relative spelling does not even parse. */
+ @Test
+ void parentOfTheRootIsRejected() {
+ final FileSystemException e =
+ assertThrows(
+ FileSystemException.class, () -> parse(baseUri() +
"/../EDIJP.NAB.CRERESP/AAMIPI"));
+ assertTrue(
+ e.getMessage().contains("Invalid relative file name"),
+ "unexpected message: " + e.getMessage());
+ }
+
+ /** A named SFTP connection reads the same way. */
+ @Test
+ void namedConnectionReadsTheSameWay() throws Exception {
+ assertEquals("/EDIJP.NAB.CRERESP/AAMIPI", parseNamedConnection("prod:" +
DATA_SET));
+ }
+
+ /** One slash on the way in is one slash on the wire: nothing gained,
nothing lost. */
+ @Test
+ void theServerIsAskedForTheSingleSlashPath() throws Exception {
+ final List<String> paths = pathsSentToServer(baseUri() +
"/read-tests/file1.txt");
+
+ assertTrue(paths.contains("/read-tests/file1.txt"), "paths asked for: "
+ paths);
+ assertTrue(
+ paths.stream().noneMatch(p -> p.startsWith("//")),
+ "no request should carry a double slash, but got: " + paths);
+ }
+ }
+
+ /**
+ * A path rooted at a second slash keeps it, all the way to the wire. The
slash rides in the root
+ * of the file name rather than in its path - see {@link
IRootPrefixFileName}.
+ */
+ @Nested
+ @DisplayName("Addressing a z/OS data set")
+ class MainframeDataSets {
+
+ /** The whole point: the server is asked for the data set, not for the USS
path. */
+ @Test
+ void theServerIsAskedForTheDataSet() throws Exception {
+ final List<String> paths = pathsSentToServer(baseUri() +
"//read-tests/file1.txt");
+
+ assertTrue(
+ paths.contains("//read-tests/file1.txt"),
+ "the server should have been asked for a //-path, but got: " +
paths);
+ }
+
+ /**
+ * The SFTP Put transform builds its target out of a named connection and
a remote folder, so
+ * the data set has to make it through {@link SftpConnections#buildUri} as
well.
+ */
+ @Test
+ void aRemoteFolderOfANamedConnectionReachesTheDataSet() throws Exception {
+ final String uri = SftpConnections.buildUri(namedConnection(),
"//read-tests", "file1.txt");
+ final List<String> paths = pathsSentToServer(uri);
+
+ assertTrue(
+ paths.contains("//read-tests/file1.txt"),
+ "the server should have been asked for a //-path, but got: " +
paths);
+ }
+
+ /** A random-access read opens the same file a plain read does: the data
set. */
+ @Test
+ void aRandomAccessReadAsksForTheDataSet() throws Exception {
+ final String expected =
+ Files.readString(
+ new File(getTestDirectoryFile(),
"read-tests/file1.txt").toPath(),
+ StandardCharsets.UTF_8);
+
+ REQUESTED_PATHS.clear();
+ final String content;
+ try (FileObject file =
+ manager.resolveFile(baseUri() + "//read-tests/file1.txt",
authOptions());
+ RandomAccessContent randomAccess =
+ file.getContent().getRandomAccessContent(RandomAccessMode.READ);
+ InputStream inputStream = randomAccess.getInputStream()) {
+ content = new String(inputStream.readAllBytes(),
StandardCharsets.UTF_8);
+ }
+ final List<String> paths = new ArrayList<>(REQUESTED_PATHS);
+
+ assertEquals(expected, content);
+ assertFalse(
+ paths.contains("/read-tests/file1.txt"),
+ "the random-access read went to the single-slash path: " + paths);
+ }
+
+ /** One slash is still one slash: an ordinary path is untouched by any of
this. */
+ @Test
+ void aSingleSlashIsStillAUssPath() throws Exception {
+ final List<String> paths = pathsSentToServer(baseUri() +
"/read-tests/file1.txt");
+
+ assertTrue(
+ paths.stream().noneMatch(p -> p.startsWith("//")),
+ "an ordinary path should not have gained a slash, but got: " +
paths);
+ }
+
+ /** Three or more mean the same as one, as POSIX says they do. */
+ @Test
+ void threeSlashesCollapseToOne() throws Exception {
+ final List<String> paths = pathsSentToServer(baseUri() +
"///read-tests/file1.txt");
+
+ assertTrue(
+ paths.stream().noneMatch(p -> p.startsWith("//")),
+ "three slashes should have collapsed, but got: " + paths);
+ }
+
+ /** The URI reads back exactly as it was typed, so it survives a save and
a reload. */
+ @Test
+ void theUriKeepsTheSecondSlash() throws Exception {
+ final FileName name = parse(baseUri() + "//EDIJP.NAB.CRERESP/AAMIPI");
+
+ assertEquals(baseUri() + "//EDIJP.NAB.CRERESP/AAMIPI", name.getURI());
+ assertEquals("/EDIJP.NAB.CRERESP/AAMIPI", name.getPath());
+ assertEquals(baseUri() + "//", name.getRootURI());
+ }
+
+ /** And it is a different file from the USS path of the same name. */
+ @Test
+ void aDataSetIsNotTheUssPath() throws Exception {
+ assertNotEquals(
+ parse(baseUri() + "/EDIJP.NAB.CRERESP").getURI(),
+ parse(baseUri() + "//EDIJP.NAB.CRERESP").getURI());
+ }
+
+ /**
+ * The prefix has to survive being derived from, which is the whole reason
it lives in the root.
+ */
+ @Test
+ void childrenAndParentsKeepTheSecondSlash() throws Exception {
+ final FileName name = parse(baseUri() + "//EDIJP.NAB.CRERESP/AAMIPI");
+
+ assertEquals(baseUri() + "//EDIJP.NAB.CRERESP",
name.getParent().getURI());
+ assertEquals(
+ baseUri() + "//EDIJP.NAB.CRERESP/AAMIPI/SUB",
+ manager.resolveName(name, "SUB", NameScope.CHILD).getURI());
+ }
+ }
+
+ /**
+ * Windows UNC names ({@code \\server\share\...}) are the one place where
Commons VFS keeps a
+ * leading double slash, and the way it does that is the way out for the
mainframe as well: the
+ * Windows name parser hoists {@code //server/share} into the <em>root
prefix</em> before {@link
+ * org.apache.commons.vfs2.provider.UriParser#normalisePath} ever sees it.
Take that hoist away
+ * and UNC breaks exactly like the SFTP data set does — which is what the
generic (non-Windows)
+ * parser below shows.
+ */
+ @Nested
+ @DisplayName("Why UNC survives and a data set does not")
+ class UncKeepsItsDoubleSlash {
+
+ private static final String UNC = "file:////server/share/dir/file.txt";
+
+ /** Hoisted into the root, so it survives, and the URI round-trips with
all four slashes. */
+ @Test
+ void windowsParserHoistsTheShareIntoTheRoot() throws Exception {
+ final FileName name = parseLocal(new WindowsFileNameParser(), UNC);
+
+ assertEquals("file:////server/share/", name.getRootURI());
+ assertEquals("/dir/file.txt", name.getPath());
+ assertEquals(UNC, name.getURI());
+ }
+
+ /** No hoist, so the slashes end up in the path and the normaliser axes
them. */
+ @Test
+ void genericParserCollapsesTheShareLikeTheDataSet() throws Exception {
+ final FileName name = parseLocal(new GenericFileNameParser(), UNC);
+
+ assertEquals("file:///", name.getRootURI());
+ assertEquals("/server/share/dir/file.txt", name.getPath());
+ assertEquals("file:///server/share/dir/file.txt", name.getURI());
+ }
+
+ private FileName parseLocal(LocalFileNameParser parser, String uri) throws
FileSystemException {
+ final FileSystemManager fsm = mock(FileSystemManager.class);
+ when(fsm.getSchemes()).thenReturn(new String[] {"file"});
+ final VfsComponentContext context = mock(VfsComponentContext.class);
+ when(context.getFileSystemManager()).thenReturn(fsm);
+
+ return parser.parseUri(context, null, uri);
+ }
+ }
+
+ /** Parse a {@code <connection>:<path>} URI and return the path it resolves
to. */
+ private static String parseNamedConnection(String uri) throws
FileSystemException {
+ final SftpConnection connection = new SftpConnection();
+ connection.setName("prod");
+ connection.setServerName("mainframe.example.com");
+ connection.setServerPort("22");
+ connection.setUsername(USER);
+
+ final IVariables variables = new Variables();
+ final FileSystemManager fsm = mock(FileSystemManager.class);
+ when(fsm.getSchemes()).thenReturn(new String[] {"prod"});
+ final VfsComponentContext context = mock(VfsComponentContext.class);
+ when(context.getFileSystemManager()).thenReturn(fsm);
+
+ return new SftpConnectionFileNameParser(variables, connection)
+ .parseUri(context, null, uri)
+ .getPath();
+ }
+
+ /** An SFTP subsystem that records the raw remote paths the client asks
about. */
+ private static class RecordingSftpSubsystemFactory extends
SftpSubsystemFactory {
+
+ @Override
+ public Command createSubsystem(final ChannelSession channel) throws
IOException {
+ return new SftpSubsystem(channel, this) {
+ @Override
+ protected Path resolveFile(final String remotePath)
+ throws IOException, InvalidPathException {
+ REQUESTED_PATHS.add(remotePath);
+ return super.resolveFile(remotePath);
+ }
+ };
+ }
+ }
+}
