This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 7b624468652 branch-4.0: [fix](multi-catalog) OSS bucket endpoint path
normalization (#64943)
7b624468652 is described below
commit 7b624468652aef0d74cd42c56239878f65f8e97f
Author: Socrates <[email protected]>
AuthorDate: Tue Jun 30 11:06:13 2026 +0800
branch-4.0: [fix](multi-catalog) OSS bucket endpoint path normalization
(#64943)
## Summary
- Normalize remote paths in SwitchingFileSystem before delegating
operations.
- Select upload filesystem from the remote path instead of the local
path.
- Add an OSS bucket endpoint delegation test.
## Root Cause
OSS bucket-domain-name paths such as `oss://bucket.endpoint/path` were
normalized when selecting the filesystem, but the original unnormalized
path was still passed to object storage operations. The lower
S3-compatible parser could then treat `bucket.endpoint` as the bucket
name, breaking Hive insert overwrite cleanup/list/delete paths.
## Validation
- `DISABLE_BUILD_UI=ON ./build.sh --fe`
- `./run-fe-ut.sh --run
org.apache.doris.fs.remote.SwitchingFileSystemTest`
---
.../doris/fs/remote/SwitchingFileSystem.java | 56 +++++--
.../doris/fs/remote/SwitchingFileSystemTest.java | 170 +++++++++++++++++++++
2 files changed, 210 insertions(+), 16 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java
index a6545f2db80..6f5b658760d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java
@@ -48,77 +48,95 @@ public class SwitchingFileSystem implements FileSystem {
@Override
public Status exists(String remotePath) {
- return fileSystem(remotePath).exists(remotePath);
+ String normalizedPath = normalizeLocation(remotePath);
+ return fileSystem(normalizedPath).exists(normalizedPath);
}
@Override
public Status directoryExists(String dir) {
- return fileSystem(dir).directoryExists(dir);
+ String normalizedPath = normalizeLocation(dir);
+ return fileSystem(normalizedPath).directoryExists(normalizedPath);
}
@Override
public Status downloadWithFileSize(String remoteFilePath, String
localFilePath, long fileSize) {
- return fileSystem(remoteFilePath).downloadWithFileSize(remoteFilePath,
localFilePath, fileSize);
+ String normalizedPath = normalizeLocation(remoteFilePath);
+ return fileSystem(normalizedPath).downloadWithFileSize(normalizedPath,
localFilePath, fileSize);
}
@Override
public Status upload(String localPath, String remotePath) {
- return fileSystem(localPath).upload(localPath, remotePath);
+ String normalizedPath = normalizeLocation(remotePath);
+ return fileSystem(normalizedPath).upload(localPath, normalizedPath);
}
@Override
public Status directUpload(String content, String remoteFile) {
- return fileSystem(remoteFile).directUpload(content, remoteFile);
+ String normalizedPath = normalizeLocation(remoteFile);
+ return fileSystem(normalizedPath).directUpload(content,
normalizedPath);
}
@Override
public Status rename(String origFilePath, String destFilePath) {
- return fileSystem(origFilePath).rename(origFilePath, destFilePath);
+ String normalizedOrigPath = normalizeLocation(origFilePath);
+ String normalizedDestPath = normalizeLocation(destFilePath);
+ return fileSystem(normalizedOrigPath).rename(normalizedOrigPath,
normalizedDestPath);
}
@Override
public Status renameDir(String origFilePath, String destFilePath) {
- return fileSystem(origFilePath).renameDir(origFilePath, destFilePath);
+ String normalizedOrigPath = normalizeLocation(origFilePath);
+ String normalizedDestPath = normalizeLocation(destFilePath);
+ return fileSystem(normalizedOrigPath).renameDir(normalizedOrigPath,
normalizedDestPath);
}
@Override
public Status renameDir(String origFilePath, String destFilePath, Runnable
runWhenPathNotExist) {
- return fileSystem(origFilePath).renameDir(origFilePath, destFilePath,
runWhenPathNotExist);
+ String normalizedOrigPath = normalizeLocation(origFilePath);
+ String normalizedDestPath = normalizeLocation(destFilePath);
+ return fileSystem(normalizedOrigPath).renameDir(normalizedOrigPath,
normalizedDestPath, runWhenPathNotExist);
}
@Override
public Status delete(String remotePath) {
- return fileSystem(remotePath).delete(remotePath);
+ String normalizedPath = normalizeLocation(remotePath);
+ return fileSystem(normalizedPath).delete(normalizedPath);
}
@Override
public Status deleteDirectory(String absolutePath) {
- return fileSystem(absolutePath).deleteDirectory(absolutePath);
+ String normalizedPath = normalizeLocation(absolutePath);
+ return fileSystem(normalizedPath).deleteDirectory(normalizedPath);
}
@Override
public Status makeDir(String remotePath) {
- return fileSystem(remotePath).makeDir(remotePath);
+ String normalizedPath = normalizeLocation(remotePath);
+ return fileSystem(normalizedPath).makeDir(normalizedPath);
}
@Override
public Status listFiles(String remotePath, boolean recursive,
List<RemoteFile> result) {
- return fileSystem(remotePath).listFiles(remotePath, recursive, result);
+ String normalizedPath = normalizeLocation(remotePath);
+ return fileSystem(normalizedPath).listFiles(normalizedPath, recursive,
result);
}
@Override
public Status globList(String remotePath, List<RemoteFile> result) {
- return fileSystem(remotePath).globList(remotePath, result);
+ String normalizedPath = normalizeLocation(remotePath);
+ return fileSystem(normalizedPath).globList(normalizedPath, result);
}
@Override
public Status globList(String remotePath, List<RemoteFile> result, boolean
fileNameOnly) {
- return fileSystem(remotePath).globList(remotePath, result,
fileNameOnly);
+ String normalizedPath = normalizeLocation(remotePath);
+ return fileSystem(normalizedPath).globList(normalizedPath, result,
fileNameOnly);
}
@Override
public Status listDirectories(String remotePath, Set<String> result) {
- return fileSystem(remotePath).listDirectories(remotePath, result);
+ String normalizedPath = normalizeLocation(remotePath);
+ return fileSystem(normalizedPath).listDirectories(normalizedPath,
result);
}
public FileSystem fileSystem(String location) {
@@ -128,5 +146,11 @@ public class SwitchingFileSystem implements FileSystem {
);
return
extMetaCacheMgr.getFsCache().getRemoteFileSystem(fileSystemCacheKey);
}
-}
+ private String normalizeLocation(String location) {
+ if (storagePropertiesMap == null) {
+ return location;
+ }
+ return LocationPath.of(location,
storagePropertiesMap).getNormalizedLocation();
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/fs/remote/SwitchingFileSystemTest.java
b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/SwitchingFileSystemTest.java
new file mode 100644
index 00000000000..3fbcec8715a
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/SwitchingFileSystemTest.java
@@ -0,0 +1,170 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.fs.remote;
+
+import org.apache.doris.backup.Status;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.doris.fs.FileSystem;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class SwitchingFileSystemTest {
+
+ @Test
+ public void testNormalizeOssBucketEndpointPathBeforeDelegating() {
+ RecordingFileSystem delegate = new RecordingFileSystem();
+ SwitchingFileSystem fs = new TestSwitchingFileSystem(delegate,
createOssStorageProperties());
+
+ String sourcePath =
"oss://my-bucket.oss-cn-beijing-internal.aliyuncs.com/path/to/source";
+ String destPath =
"oss://my-bucket.oss-cn-beijing-internal.aliyuncs.com/path/to/dest";
+
+ fs.deleteDirectory(sourcePath);
+ Assertions.assertEquals("s3://my-bucket/path/to/source",
delegate.remotePath);
+ Assertions.assertEquals("s3://my-bucket/path/to/source",
delegate.selectedLocation);
+
+ fs.listFiles(sourcePath, true, Collections.emptyList());
+ Assertions.assertEquals("s3://my-bucket/path/to/source",
delegate.remotePath);
+ Assertions.assertEquals("s3://my-bucket/path/to/source",
delegate.selectedLocation);
+
+ fs.rename(sourcePath, destPath);
+ Assertions.assertEquals("s3://my-bucket/path/to/source",
delegate.remotePath);
+ Assertions.assertEquals("s3://my-bucket/path/to/dest",
delegate.destPath);
+ Assertions.assertEquals("s3://my-bucket/path/to/source",
delegate.selectedLocation);
+
+ fs.upload("/tmp/local-file", destPath);
+ Assertions.assertEquals("/tmp/local-file", delegate.localPath);
+ Assertions.assertEquals("s3://my-bucket/path/to/dest",
delegate.remotePath);
+ Assertions.assertEquals("s3://my-bucket/path/to/dest",
delegate.selectedLocation);
+ }
+
+ private static Map<StorageProperties.Type, StorageProperties>
createOssStorageProperties() {
+ Map<String, String> origProps = new HashMap<>();
+ origProps.put("oss.endpoint", "oss-cn-beijing-internal.aliyuncs.com");
+ origProps.put("oss.access_key", "ak");
+ origProps.put("oss.secret_key", "sk");
+ origProps.put(StorageProperties.FS_OSS_SUPPORT, "true");
+
+ Map<StorageProperties.Type, StorageProperties> storageProperties = new
HashMap<>();
+ storageProperties.put(StorageProperties.Type.OSS,
StorageProperties.createPrimary(origProps));
+ return storageProperties;
+ }
+
+ private static class TestSwitchingFileSystem extends SwitchingFileSystem {
+ private final RecordingFileSystem delegate;
+
+ TestSwitchingFileSystem(RecordingFileSystem delegate,
+ Map<StorageProperties.Type, StorageProperties>
storagePropertiesMap) {
+ super(null, storagePropertiesMap);
+ this.delegate = delegate;
+ }
+
+ @Override
+ public FileSystem fileSystem(String location) {
+ delegate.selectedLocation = location;
+ return delegate;
+ }
+ }
+
+ private static class RecordingFileSystem implements FileSystem {
+ private String selectedLocation;
+ private String localPath;
+ private String remotePath;
+ private String destPath;
+
+ @Override
+ public Map<String, String> getProperties() {
+ return null;
+ }
+
+ @Override
+ public Status exists(String remotePath) {
+ this.remotePath = remotePath;
+ return Status.OK;
+ }
+
+ @Override
+ public Status downloadWithFileSize(String remoteFilePath, String
localFilePath, long fileSize) {
+ this.remotePath = remoteFilePath;
+ this.localPath = localFilePath;
+ return Status.OK;
+ }
+
+ @Override
+ public Status upload(String localPath, String remotePath) {
+ this.localPath = localPath;
+ this.remotePath = remotePath;
+ return Status.OK;
+ }
+
+ @Override
+ public Status directUpload(String content, String remoteFile) {
+ this.remotePath = remoteFile;
+ return Status.OK;
+ }
+
+ @Override
+ public Status rename(String origFilePath, String destFilePath) {
+ this.remotePath = origFilePath;
+ this.destPath = destFilePath;
+ return Status.OK;
+ }
+
+ @Override
+ public Status delete(String remotePath) {
+ this.remotePath = remotePath;
+ return Status.OK;
+ }
+
+ @Override
+ public Status deleteDirectory(String dir) {
+ this.remotePath = dir;
+ return Status.OK;
+ }
+
+ @Override
+ public Status makeDir(String remotePath) {
+ this.remotePath = remotePath;
+ return Status.OK;
+ }
+
+ @Override
+ public Status listFiles(String remotePath, boolean recursive,
List<RemoteFile> result) {
+ this.remotePath = remotePath;
+ return Status.OK;
+ }
+
+ @Override
+ public Status globList(String remotePath, List<RemoteFile> result,
boolean fileNameOnly) {
+ this.remotePath = remotePath;
+ return Status.OK;
+ }
+
+ @Override
+ public Status listDirectories(String remotePath, Set<String> result) {
+ this.remotePath = remotePath;
+ return Status.OK;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
