This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-fs-spi in repository https://gitbox.apache.org/repos/asf/doris.git
commit 53c8e47eeb99c1a8c287e77b87ff0b9eae8f4546 Author: morningman <[email protected]> AuthorDate: Mon Mar 30 12:02:13 2026 +0800 [refactor](fe) Phase 0: Prerequisite decoupling for filesystem SPI ### What problem does this PR solve? Issue Number: N/A Problem Summary: Before splitting filesystem implementations into independent Maven modules (Phase 3), several compile-time couplings must be eliminated. This commit completes all Phase 0 prerequisite decoupling tasks: - P0.1: Introduce FsStorageType enum in fe-foundation (zero-dep module) to replace StorageBackend.StorageType (Thrift-generated) in PersistentFileSystem. Add FsStorageTypeAdapter for bidirectional Thrift conversion. Update all subclasses and callers (Repository, BackupJob, RestoreJob, CloudRestoreJob). - P0.2: Add IOException-based default bridge methods to ObjStorage interface (checkObjectExists, getObjectChecked, putObjectChecked, deleteObjectChecked, deleteObjectsChecked, copyObjectChecked, listObjectsChecked). Add ObjStorageStatusAdapter for Status→IOException conversion. Zero changes to existing implementations. - P0.3: Decouple SwitchingFileSystem from ExternalMetaCacheMgr via new FileSystemLookup functional interface. FileSystemProviderImpl passes a lambda. - P0.4: Extract MultipartUploadCapable interface from ObjFileSystem, removing the forced abstract method. S3FileSystem and AzureFileSystem implement it. HMSTransaction now uses instanceof check instead of ObjFileSystem cast. - P0.5: Introduce FileSystemDescriptor POJO for Repository metadata serialization, replacing direct PersistentFileSystem subclass serialization. Migrate GsonUtils to string-based Class.forName() reflection for legacy format backward compat, removing 7 compile-time imports of concrete filesystem classes. - P0.6: Add FileSystemSpiProvider interface skeleton in fs/spi/ as the future ServiceLoader contract for Phase 3 module split. ### Release note None ### Check List (For Author) - Test: No need to test (pure refactor; all changes are backward compatible; three successful FE builds verified during development) - Behavior changed: No - Does this need documentation: No Co-authored-by: Copilot <[email protected]> --- .../java/org/apache/doris/backup/BackupJob.java | 2 +- .../java/org/apache/doris/backup/Repository.java | 45 ++++++++++--- .../java/org/apache/doris/backup/RestoreJob.java | 2 +- .../apache/doris/cloud/backup/CloudRestoreJob.java | 2 +- .../doris/datasource/hive/HMSTransaction.java | 4 +- .../org/apache/doris/fs/FileSystemDescriptor.java | 77 ++++++++++++++++++++++ ...SSHdfsFileSystem.java => FileSystemLookup.java} | 19 +++--- .../apache/doris/fs/FileSystemProviderImpl.java | 4 +- .../org/apache/doris/fs/FsStorageTypeAdapter.java | 59 +++++++++++++++++ .../org/apache/doris/fs/PersistentFileSystem.java | 12 +++- .../java/org/apache/doris/fs/obj/ObjStorage.java | 48 ++++++++++++++ .../doris/fs/obj/ObjStorageStatusAdapter.java | 68 +++++++++++++++++++ .../apache/doris/fs/remote/AzureFileSystem.java | 6 +- .../apache/doris/fs/remote/BrokerFileSystem.java | 4 +- .../doris/fs/remote/MultipartUploadCapable.java | 53 +++++++++++++++ .../org/apache/doris/fs/remote/ObjFileSystem.java | 23 +------ .../apache/doris/fs/remote/RemoteFileSystem.java | 4 +- .../org/apache/doris/fs/remote/S3FileSystem.java | 6 +- .../doris/fs/remote/SwitchingFileSystem.java | 10 +-- .../apache/doris/fs/remote/dfs/DFSFileSystem.java | 6 +- .../apache/doris/fs/remote/dfs/JFSFileSystem.java | 4 +- .../apache/doris/fs/remote/dfs/OFSFileSystem.java | 4 +- .../doris/fs/remote/dfs/OSSHdfsFileSystem.java | 4 +- .../apache/doris/fs/spi/FileSystemSpiProvider.java | 59 +++++++++++++++++ .../org/apache/doris/persist/gson/GsonUtils.java | 51 +++++++++----- .../doris/fs/remote/RemoteFileSystemTest.java | 4 +- .../apache/doris/foundation/fs/FsStorageType.java} | 23 +++++-- 27 files changed, 508 insertions(+), 95 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index a64a4a98f5d..b345969997a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -822,7 +822,7 @@ public class BackupJob extends AbstractJob implements GsonPostProcessable { UploadTask task = new UploadTask(null, beId, signature, jobId, dbId, srcToDest, brokers.get(0), repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties(), - repo.getRemoteFileSystem().getStorageType(), repo.getLocation()); + repo.getRemoteFileSystem().getThriftStorageType(), repo.getLocation()); batchTask.addTask(task); unfinishedTaskIds.put(signature, beId); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index cac9b9ae1d4..a70489d4321 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -17,7 +17,6 @@ package org.apache.doris.backup; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.backup.Status.ErrCode; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FsBroker; @@ -31,6 +30,8 @@ import org.apache.doris.common.util.DatasourcePrintableMap; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.property.storage.BrokerProperties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.foundation.fs.FsStorageType; +import org.apache.doris.fs.FileSystemDescriptor; import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.fs.PersistentFileSystem; import org.apache.doris.fs.remote.BrokerFileSystem; @@ -129,8 +130,17 @@ public class Repository implements Writable, GsonPostProcessable { @SerializedName("lo") private String location; + /** Legacy field: kept for backward-compatible deserialization of old metadata. */ + @Deprecated @SerializedName("fs") - private PersistentFileSystem fileSystem; + private PersistentFileSystem legacyFileSystem; + + /** New field: lightweight descriptor used for new metadata serialization. */ + @SerializedName("fs_descriptor") + private FileSystemDescriptor fileSystemDescriptor; + + /** Live filesystem instance; transient — rebuilt in {@link #gsonPostProcess()}. */ + private transient PersistentFileSystem fileSystem; public PersistentFileSystem getFileSystem() { return fileSystem; @@ -146,6 +156,7 @@ public class Repository implements Writable, GsonPostProcessable { this.isReadOnly = isReadOnly; this.location = location; this.fileSystem = fileSystem; + this.fileSystemDescriptor = FileSystemDescriptor.fromPersistentFileSystem(fileSystem); this.createTime = System.currentTimeMillis(); } @@ -201,8 +212,22 @@ public class Repository implements Writable, GsonPostProcessable { @Override public void gsonPostProcess() { + // Determine source of name+properties: prefer new descriptor, fall back to legacy field. + String fsName; + Map<String, String> fsProps; + if (fileSystemDescriptor != null) { + fsName = fileSystemDescriptor.getName(); + fsProps = fileSystemDescriptor.getProperties(); + } else if (legacyFileSystem != null) { + fsName = legacyFileSystem.name; + fsProps = legacyFileSystem.properties; + // Migrate to new descriptor so the next write uses the new format. + fileSystemDescriptor = FileSystemDescriptor.fromPersistentFileSystem(legacyFileSystem); + } else { + return; + } try { - StorageProperties storageProperties = StorageProperties.createPrimary(this.fileSystem.properties); + StorageProperties storageProperties = StorageProperties.createPrimary(fsProps); this.fileSystem = FileSystemFactory.get(storageProperties); } catch (RuntimeException exception) { LOG.warn("File system initialization failed due to incompatible configuration parameters. " @@ -212,7 +237,7 @@ public class Repository implements Writable, GsonPostProcessable { + "Please review the configuration and update it to match the supported parameter" + " format. Root cause: {}", ExceptionUtils.getRootCause(exception), exception); - BrokerProperties brokerProperties = BrokerProperties.of(this.fileSystem.name, this.fileSystem.properties); + BrokerProperties brokerProperties = BrokerProperties.of(fsName, fsProps); this.fileSystem = FileSystemFactory.get(brokerProperties); } } @@ -635,7 +660,7 @@ public class Repository implements Writable, GsonPostProcessable { + "failed to send upload snapshot task"); } // only Broker storage backend need to get broker addr, other type return a fake one; - if (fileSystem.getStorageType() != StorageBackend.StorageType.BROKER) { + if (fileSystem.getStorageType() != FsStorageType.BROKER) { brokerAddrs.add(new FsBroker("127.0.0.1", 0)); return Status.OK; } @@ -664,7 +689,7 @@ public class Repository implements Writable, GsonPostProcessable { info.add(TimeUtils.longToTimeString(createTime)); info.add(String.valueOf(isReadOnly)); info.add(location); - info.add(fileSystem.getStorageType() != StorageBackend.StorageType.BROKER ? "-" : fileSystem.getName()); + info.add(fileSystem.getStorageType() != FsStorageType.BROKER ? "-" : fileSystem.getName()); info.add(fileSystem.getStorageType().name()); info.add(errMsg == null ? FeConstants.null_string : errMsg); return info; @@ -704,12 +729,12 @@ public class Repository implements Writable, GsonPostProcessable { stmtBuilder.append("REPOSITORY "); stmtBuilder.append(this.name); stmtBuilder.append(" \nWITH "); - StorageBackend.StorageType storageType = this.fileSystem.getStorageType(); - if (storageType == StorageBackend.StorageType.S3) { + FsStorageType storageType = this.fileSystem.getStorageType(); + if (storageType == FsStorageType.S3) { stmtBuilder.append(" S3 "); - } else if (storageType == StorageBackend.StorageType.HDFS) { + } else if (storageType == FsStorageType.HDFS) { stmtBuilder.append(" HDFS "); - } else if (storageType == StorageBackend.StorageType.BROKER) { + } else if (storageType == FsStorageType.BROKER) { stmtBuilder.append(" BROKER "); stmtBuilder.append(this.fileSystem.getName()); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 0395378de47..a0bd223da72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -2003,7 +2003,7 @@ public class RestoreJob extends AbstractJob implements GsonPostProcessable { Map<String, String> srcToDest, FsBroker brokerAddr) { return new DownloadTask(null, beId, signature, jobId, dbId, srcToDest, brokerAddr, repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties(), - repo.getRemoteFileSystem().getStorageType(), repo.getLocation(), ""); + repo.getRemoteFileSystem().getThriftStorageType(), repo.getLocation(), ""); } // Get the id mapping for snapshot, user should hold the lock of table. diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java index 980e75ee2b5..59f40833a5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/backup/CloudRestoreJob.java @@ -293,7 +293,7 @@ public class CloudRestoreJob extends RestoreJob { Preconditions.checkState(storageVaultId != null, "Storage vault ID cannot be null"); return new DownloadTask(null, beId, signature, jobId, dbId, srcToDest, brokerAddr, repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties(), - repo.getRemoteFileSystem().getStorageType(), repo.getLocation(), storageVaultId); + repo.getRemoteFileSystem().getThriftStorageType(), repo.getLocation(), storageVaultId); } public void downloadLocalSnapshots() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java index 0a7fe103427..dda3ae0c1d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java @@ -31,7 +31,7 @@ import org.apache.doris.foundation.util.PathUtils; import org.apache.doris.fs.FileSystem; import org.apache.doris.fs.FileSystemProvider; import org.apache.doris.fs.FileSystemUtil; -import org.apache.doris.fs.remote.ObjFileSystem; +import org.apache.doris.fs.remote.MultipartUploadCapable; import org.apache.doris.fs.remote.RemoteFile; import org.apache.doris.fs.remote.S3FileSystem; import org.apache.doris.fs.remote.SwitchingFileSystem; @@ -1850,7 +1850,7 @@ public class HMSTransaction implements Transaction { if (isMockedPartitionUpdate) { return; } - ObjFileSystem fileSystem = (ObjFileSystem) ((SwitchingFileSystem) fs).fileSystem(path); + MultipartUploadCapable fileSystem = (MultipartUploadCapable) ((SwitchingFileSystem) fs).fileSystem(path); for (TS3MPUPendingUpload s3MPUPendingUpload : s3MpuPendingUploads) { asyncFileSystemTaskFutures.add(CompletableFuture.runAsync(() -> { if (fileSystemTaskCancelled.get()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDescriptor.java new file mode 100644 index 00000000000..2231299c85d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDescriptor.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fs; + +import org.apache.doris.foundation.fs.FsStorageType; + +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; + +import java.util.Map; + +/** + * A lightweight POJO that describes a persistent file system configuration. + * + * <p>This class replaces direct serialization of concrete {@link PersistentFileSystem} + * subclasses (S3FileSystem, DFSFileSystem, etc.) in the backup/restore metadata. + * By separating the description (type + properties) from the live object, + * {@code GsonUtils} no longer needs compile-time references to concrete implementation classes. + * + * <p>Serialized format (JSON): + * <pre>{@code + * { + * "fs_type": "S3", + * "fs_name": "my-s3-repo", + * "fs_props": { "AWS_ACCESS_KEY": "...", "AWS_SECRET_KEY": "..." } + * } + * }</pre> + */ +public class FileSystemDescriptor { + + @SerializedName("fs_type") + private final FsStorageType storageType; + + @SerializedName("fs_name") + private final String name; + + @SerializedName("fs_props") + private final Map<String, String> properties; + + public FileSystemDescriptor(FsStorageType storageType, String name, Map<String, String> properties) { + this.storageType = storageType; + this.name = name; + this.properties = Maps.newHashMap(properties); + } + + public FsStorageType getStorageType() { + return storageType; + } + + public String getName() { + return name; + } + + public Map<String, String> getProperties() { + return properties; + } + + /** Creates a FileSystemDescriptor from an existing PersistentFileSystem (migration helper). */ + public static FileSystemDescriptor fromPersistentFileSystem(PersistentFileSystem fs) { + return new FileSystemDescriptor(fs.getStorageType(), fs.getName(), fs.getProperties()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemLookup.java similarity index 66% copy from fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java copy to fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemLookup.java index 636cb6ecb84..09f3a453ce4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemLookup.java @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.fs.remote.dfs; +package org.apache.doris.fs; -import org.apache.doris.analysis.StorageBackend; -import org.apache.doris.datasource.property.storage.OSSHdfsProperties; - -public class OSSHdfsFileSystem extends DFSFileSystem { - public OSSHdfsFileSystem(OSSHdfsProperties properties) { - super(properties, StorageBackend.StorageType.HDFS); - } +/** + * Abstraction for looking up FileSystem instances from a cache, + * used by SwitchingFileSystem to avoid direct dependency on ExternalMetaCacheMgr. + */ +public interface FileSystemLookup { + /** + * Looks up and returns a cached FileSystem for the given key. + * Creates a new instance if not cached. + */ + FileSystem lookup(FileSystemCache.FileSystemCacheKey key); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemProviderImpl.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemProviderImpl.java index 7a2e417a711..c9c2010dd1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemProviderImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemProviderImpl.java @@ -37,6 +37,8 @@ public class FileSystemProviderImpl implements FileSystemProvider { @Override public FileSystem get(SessionContext ctx) { - return new SwitchingFileSystem(extMetaCacheMgr, storagePropertiesMap); + return new SwitchingFileSystem( + key -> extMetaCacheMgr.getFsCache().getRemoteFileSystem(key), + storagePropertiesMap); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FsStorageTypeAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FsStorageTypeAdapter.java new file mode 100644 index 00000000000..34aae6a6d04 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FsStorageTypeAdapter.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fs; + +import org.apache.doris.analysis.StorageBackend; +import org.apache.doris.foundation.fs.FsStorageType; + +/** + * Bidirectional converter between FsStorageType (fe-foundation) and + * Thrift-generated StorageBackend.StorageType (fe-thrift). + * Lives in fe-core which can see both types. + */ +public final class FsStorageTypeAdapter { + private FsStorageTypeAdapter() {} + + public static FsStorageType fromThrift(StorageBackend.StorageType thriftType) { + switch (thriftType) { + case S3: return FsStorageType.S3; + case HDFS: return FsStorageType.HDFS; + case BROKER: return FsStorageType.BROKER; + case AZURE: return FsStorageType.AZURE; + case OFS: return FsStorageType.OFS; + case JFS: return FsStorageType.JFS; + case LOCAL: return FsStorageType.LOCAL; + default: + throw new IllegalArgumentException("Unknown Thrift StorageType: " + thriftType); + } + } + + public static StorageBackend.StorageType toThrift(FsStorageType fsType) { + switch (fsType) { + case S3: return StorageBackend.StorageType.S3; + case HDFS: return StorageBackend.StorageType.HDFS; + case OSS_HDFS: return StorageBackend.StorageType.HDFS; + case BROKER: return StorageBackend.StorageType.BROKER; + case AZURE: return StorageBackend.StorageType.AZURE; + case OFS: return StorageBackend.StorageType.OFS; + case JFS: return StorageBackend.StorageType.JFS; + case LOCAL: return StorageBackend.StorageType.LOCAL; + default: + throw new IllegalArgumentException("Unknown FsStorageType: " + fsType); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java index 08d2f251fb3..86981b559b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java @@ -19,6 +19,7 @@ package org.apache.doris.fs; import org.apache.doris.analysis.StorageBackend; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.foundation.fs.FsStorageType; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -34,11 +35,11 @@ public abstract class PersistentFileSystem implements FileSystem { public Map<String, String> properties = Maps.newHashMap(); @SerializedName("n") public String name; - public StorageBackend.StorageType type; + public FsStorageType type; public abstract StorageProperties getStorageProperties(); - public PersistentFileSystem(String name, StorageBackend.StorageType type) { + public PersistentFileSystem(String name, FsStorageType type) { this.name = name; this.type = type; } @@ -51,7 +52,12 @@ public abstract class PersistentFileSystem implements FileSystem { return properties; } - public StorageBackend.StorageType getStorageType() { + public FsStorageType getStorageType() { return type; } + + @Deprecated + public StorageBackend.StorageType getThriftStorageType() { + return FsStorageTypeAdapter.toThrift(type); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/ObjStorage.java index ee9f9dc1e71..1589d2fccd1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/ObjStorage.java @@ -25,6 +25,7 @@ import org.apache.commons.lang3.tuple.Triple; import org.jetbrains.annotations.Nullable; import java.io.File; +import java.io.IOException; import java.io.InputStream; /** @@ -37,6 +38,8 @@ public interface ObjStorage<C> extends AutoCloseable { // CHUNK_SIZE for multi part upload int CHUNK_SIZE = 5 * 1024 * 1024; + // ==================== Legacy Status-based API (keep for backward compat) ==================== + C getClient() throws UserException; Triple<String, String, String> getStsToken() throws DdlException; @@ -55,6 +58,51 @@ public interface ObjStorage<C> extends AutoCloseable { RemoteObjects listObjects(String remotePath, String continuationToken) throws DdlException; + // ==================== IOException-based API (bridge to legacy methods above) ==================== + + /** Checks that the remote object exists; throws {@link java.io.FileNotFoundException} if not. */ + default void checkObjectExists(String remotePath) throws IOException { + ObjStorageStatusAdapter.throwIfFailed(headObject(remotePath), "headObject", remotePath); + } + + /** Downloads a remote object to a local file; throws {@link IOException} on failure. */ + default void getObjectChecked(String remoteFilePath, File localFile) throws IOException { + ObjStorageStatusAdapter.throwIfFailed(getObject(remoteFilePath, localFile), "getObject", remoteFilePath); + } + + /** Uploads content to a remote path; throws {@link IOException} on failure. */ + default void putObjectChecked(String remotePath, @Nullable InputStream content, + long contentLength) throws IOException { + ObjStorageStatusAdapter.throwIfFailed(putObject(remotePath, content, contentLength), "putObject", remotePath); + } + + /** Deletes a remote object; throws {@link IOException} on failure. */ + default void deleteObjectChecked(String remotePath) throws IOException { + ObjStorageStatusAdapter.throwIfFailed(deleteObject(remotePath), "deleteObject", remotePath); + } + + /** Deletes remote objects by prefix; throws {@link IOException} on failure. */ + default void deleteObjectsChecked(String remotePath) throws IOException { + ObjStorageStatusAdapter.throwIfFailed(deleteObjects(remotePath), "deleteObjects", remotePath); + } + + /** Copies a remote object; throws {@link IOException} on failure. */ + default void copyObjectChecked(String origFilePath, String destFilePath) throws IOException { + ObjStorageStatusAdapter.throwIfFailed( + copyObject(origFilePath, destFilePath), "copyObject", origFilePath + " -> " + destFilePath); + } + + /** Lists objects at a remote path; throws {@link IOException} instead of {@link DdlException}. */ + default RemoteObjects listObjectsChecked(String remotePath, String continuationToken) throws IOException { + try { + return listObjects(remotePath, continuationToken); + } catch (DdlException e) { + throw new IOException("listObjects failed for path: " + remotePath, e); + } + } + + // ==================== Utility default methods ==================== + default String normalizePrefix(String prefix) { return prefix.isEmpty() ? "" : (prefix.endsWith("/") ? prefix : String.format("%s/", prefix)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/ObjStorageStatusAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/ObjStorageStatusAdapter.java new file mode 100644 index 00000000000..f6473d0d489 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/ObjStorageStatusAdapter.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fs.obj; + +import org.apache.doris.backup.Status; + +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * Adapter that converts {@link Status}-based results from {@link ObjStorage} + * to standard {@link IOException}. + * + * <p>This is a transitional bridge: it allows new code to work with IOExceptions + * while the underlying ObjStorage implementation still returns Status objects. + * Once all implementations are migrated to throw IOException directly, + * this class can be removed. + */ +public final class ObjStorageStatusAdapter { + + private ObjStorageStatusAdapter() {} + + /** + * Throws IOException if the status indicates failure. + * Maps {@link Status.ErrCode#NOT_FOUND} to {@link FileNotFoundException}; + * all other errors become plain {@link IOException}. + * + * @param status the status to check + * @param operation the operation name for error message context + * @param path the remote path involved in the operation + * @throws FileNotFoundException if status code is NOT_FOUND + * @throws IOException for all other error statuses + */ + public static void throwIfFailed(Status status, String operation, String path) + throws IOException { + if (status.ok()) { + return; + } + throw toIOException(status, operation, path); + } + + /** + * Converts a failed Status to an appropriate IOException subtype. + * Convenience for use in lambda/stream contexts. + */ + public static IOException toIOException(Status status, String operation, String path) { + String msg = operation + " failed for path [" + path + "]: " + status.getErrMsg(); + if (Status.ErrCode.NOT_FOUND.equals(status.getErrCode())) { + return new FileNotFoundException(msg); + } + return new IOException(msg); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/AzureFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/AzureFileSystem.java index c568b6b6f9c..a6ccac5687e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/AzureFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/AzureFileSystem.java @@ -17,10 +17,10 @@ package org.apache.doris.fs.remote; -import org.apache.doris.analysis.StorageBackend.StorageType; import org.apache.doris.backup.Status; import org.apache.doris.datasource.property.storage.AzureProperties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.foundation.fs.FsStorageType; import org.apache.doris.fs.obj.AzureObjStorage; import org.apache.logging.log4j.LogManager; @@ -31,12 +31,12 @@ import java.util.List; import java.util.Map; import java.util.Set; -public class AzureFileSystem extends ObjFileSystem { +public class AzureFileSystem extends ObjFileSystem implements MultipartUploadCapable { private static final Logger LOG = LogManager.getLogger(AzureFileSystem.class); private final AzureProperties azureProperties; public AzureFileSystem(AzureProperties azureProperties) { - super(StorageType.AZURE.name(), StorageType.AZURE, new AzureObjStorage(azureProperties)); + super(FsStorageType.AZURE.name(), FsStorageType.AZURE, new AzureObjStorage(azureProperties)); this.azureProperties = azureProperties; this.properties.putAll(azureProperties.getOrigProps()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java index 224381f10d1..ea5cc86292a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java @@ -17,7 +17,6 @@ package org.apache.doris.fs.remote; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.backup.Status; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FsBroker; @@ -28,6 +27,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.datasource.property.storage.BrokerProperties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.foundation.fs.FsStorageType; import org.apache.doris.fs.operations.BrokerFileOperations; import org.apache.doris.fs.operations.OpParams; import org.apache.doris.service.FrontendOptions; @@ -79,7 +79,7 @@ public class BrokerFileSystem extends RemoteFileSystem { //todo The method parameter should use the interface type StorageProperties instead of a specific implementation. public BrokerFileSystem(BrokerProperties brokerProperties) { - super(brokerProperties.getBrokerName(), StorageBackend.StorageType.BROKER); + super(brokerProperties.getBrokerName(), FsStorageType.BROKER); this.brokerProperties = brokerProperties; this.operations = new BrokerFileOperations(name, brokerProperties.getBrokerParams()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/MultipartUploadCapable.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/MultipartUploadCapable.java new file mode 100644 index 00000000000..5302b7730a9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/MultipartUploadCapable.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fs.remote; + +import java.util.Map; + +/** + * Optional capability interface for file systems that support S3-style multipart upload completion. + * + * <p>In object storage systems (S3, Azure Blob Storage), large files can be uploaded in multiple + * parts. After all parts are uploaded, the parts must be explicitly merged via + * {@link #completeMultipartUpload}. + * + * <p>Not all {@link ObjFileSystem} implementations support this protocol; callers must check + * {@code instanceof MultipartUploadCapable} before use. + * + * <p>Example usage: + * <pre>{@code + * FileSystem fs = switchingFs.fileSystem(path); + * if (!(fs instanceof MultipartUploadCapable)) { + * throw new IllegalStateException("FileSystem does not support multipart upload: " + path); + * } + * ((MultipartUploadCapable) fs).completeMultipartUpload(bucket, key, uploadId, parts); + * }</pre> + */ +public interface MultipartUploadCapable { + + /** + * Completes a multipart upload by merging all uploaded parts into a single object. + * + * @param bucket the name of the target bucket + * @param key the full object key (path) within the bucket + * @param uploadId the unique identifier of the multipart upload session + * @param parts mapping of part numbers (1-based) to their ETag values, + * used to assemble parts in the correct order + */ + void completeMultipartUpload(String bucket, String key, String uploadId, Map<Integer, String> parts); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/ObjFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/ObjFileSystem.java index f04fb6fa01b..3d55e1e9f68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/ObjFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/ObjFileSystem.java @@ -17,8 +17,8 @@ package org.apache.doris.fs.remote; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.backup.Status; +import org.apache.doris.foundation.fs.FsStorageType; import org.apache.doris.fs.obj.ObjStorage; import org.apache.logging.log4j.LogManager; @@ -42,7 +42,7 @@ public abstract class ObjFileSystem extends RemoteFileSystem { protected final ObjStorage<?> objStorage; - public ObjFileSystem(String name, StorageBackend.StorageType type, ObjStorage<?> objStorage) { + public ObjFileSystem(String name, FsStorageType type, ObjStorage<?> objStorage) { super(name, type); this.objStorage = objStorage; } @@ -165,23 +165,4 @@ public abstract class ObjFileSystem extends RemoteFileSystem { return objStorage.deleteObjects(absolutePath); } - - /** - * Completes a multipart upload operation. - * - * <p>In object storage systems, large files are often uploaded in multiple parts. - * Once all parts have been successfully uploaded, this method is called to merge - * them into a single finalized object. - * - * <p>The main purpose of this method is to notify the underlying storage service - * to perform the final merge and make the object available for normal access. - * - * @param bucket The name of the target bucket. - * @param key The full object key (path) within the bucket. - * @param uploadId The unique identifier of the multipart upload session. - * @param parts A mapping of part numbers to their corresponding ETag values, - * used to assemble the parts in the correct order. - */ - public abstract void completeMultipartUpload(String bucket, String key, - String uploadId, Map<Integer, String> parts); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java index 36cc77ac94b..8903b9c9318 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java @@ -17,9 +17,9 @@ package org.apache.doris.fs.remote; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.backup.Status; import org.apache.doris.common.UserException; +import org.apache.doris.foundation.fs.FsStorageType; import org.apache.doris.fs.PersistentFileSystem; import org.apache.hadoop.fs.FileStatus; @@ -35,7 +35,7 @@ public abstract class RemoteFileSystem extends PersistentFileSystem implements C protected AtomicBoolean closed = new AtomicBoolean(false); - public RemoteFileSystem(String name, StorageBackend.StorageType type) { + public RemoteFileSystem(String name, FsStorageType type) { super(name, type); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java index 366cfaadc25..828e46873a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java @@ -17,12 +17,12 @@ package org.apache.doris.fs.remote; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.backup.Status; import org.apache.doris.common.UserException; import org.apache.doris.common.util.S3URI; import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.foundation.fs.FsStorageType; import org.apache.doris.fs.GlobListResult; import org.apache.doris.fs.obj.S3ObjStorage; @@ -40,13 +40,13 @@ import java.util.List; import java.util.Map; import java.util.Set; -public class S3FileSystem extends ObjFileSystem { +public class S3FileSystem extends ObjFileSystem implements MultipartUploadCapable { private static final Logger LOG = LogManager.getLogger(S3FileSystem.class); private final AbstractS3CompatibleProperties s3Properties; public S3FileSystem(AbstractS3CompatibleProperties s3Properties) { - super(StorageBackend.StorageType.S3.name(), StorageBackend.StorageType.S3, + super(FsStorageType.S3.name(), FsStorageType.S3, new S3ObjStorage(s3Properties)); this.s3Properties = s3Properties; initFsProperties(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java index 65905476946..93dbfae6b72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/SwitchingFileSystem.java @@ -19,10 +19,10 @@ package org.apache.doris.fs.remote; import org.apache.doris.backup.Status; import org.apache.doris.common.util.LocationPath; -import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.FileSystem; import org.apache.doris.fs.FileSystemCache; +import org.apache.doris.fs.FileSystemLookup; import java.util.List; import java.util.Map; @@ -30,13 +30,13 @@ import java.util.Set; public class SwitchingFileSystem implements FileSystem { - private final ExternalMetaCacheMgr extMetaCacheMgr; + private final FileSystemLookup lookup; private final Map<StorageProperties.Type, StorageProperties> storagePropertiesMap; - public SwitchingFileSystem(ExternalMetaCacheMgr extMetaCacheMgr, + public SwitchingFileSystem(FileSystemLookup lookup, Map<StorageProperties.Type, StorageProperties> storagePropertiesMap) { - this.extMetaCacheMgr = extMetaCacheMgr; + this.lookup = lookup; this.storagePropertiesMap = storagePropertiesMap; } @@ -126,6 +126,6 @@ public class SwitchingFileSystem implements FileSystem { FileSystemCache.FileSystemCacheKey fileSystemCacheKey = new FileSystemCache.FileSystemCacheKey( path.getFsIdentifier(), path.getStorageProperties() ); - return extMetaCacheMgr.getFsCache().getRemoteFileSystem(fileSystemCacheKey); + return lookup.lookup(fileSystemCacheKey); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java index 2ee3c156931..5d58edf343d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java @@ -17,7 +17,6 @@ package org.apache.doris.fs.remote.dfs; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.backup.Status; import org.apache.doris.common.UserException; import org.apache.doris.common.security.authentication.HadoopAuthenticator; @@ -25,6 +24,7 @@ import org.apache.doris.common.util.S3Util; import org.apache.doris.common.util.URI; import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.foundation.fs.FsStorageType; import org.apache.doris.fs.io.DorisInputFile; import org.apache.doris.fs.io.DorisOutputFile; import org.apache.doris.fs.io.ParsedPath; @@ -74,7 +74,7 @@ public class DFSFileSystem extends RemoteFileSystem { protected volatile org.apache.hadoop.fs.FileSystem dfsFileSystem = null; public DFSFileSystem(HdfsCompatibleProperties hdfsProperties) { - super(StorageBackend.StorageType.HDFS.name(), StorageBackend.StorageType.HDFS); + super(FsStorageType.HDFS.name(), FsStorageType.HDFS); this.properties.putAll(hdfsProperties.getOrigProps()); this.hdfsProperties = hdfsProperties; } @@ -123,7 +123,7 @@ public class DFSFileSystem extends RemoteFileSystem { return Status.OK; } - public DFSFileSystem(HdfsCompatibleProperties hdfsProperties, StorageBackend.StorageType storageType) { + public DFSFileSystem(HdfsCompatibleProperties hdfsProperties, FsStorageType storageType) { super(storageType.name(), storageType); this.properties.putAll(hdfsProperties.getOrigProps()); this.hdfsProperties = hdfsProperties; diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/JFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/JFSFileSystem.java index 200a7f3908e..00cd11f7ee7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/JFSFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/JFSFileSystem.java @@ -17,11 +17,11 @@ package org.apache.doris.fs.remote.dfs; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties; +import org.apache.doris.foundation.fs.FsStorageType; public class JFSFileSystem extends DFSFileSystem { public JFSFileSystem(HdfsCompatibleProperties hdfsProperties) { - super(hdfsProperties, StorageBackend.StorageType.JFS); + super(hdfsProperties, FsStorageType.JFS); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OFSFileSystem.java index a061fdda730..54d91bf9e5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OFSFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OFSFileSystem.java @@ -17,11 +17,11 @@ package org.apache.doris.fs.remote.dfs; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties; +import org.apache.doris.foundation.fs.FsStorageType; public class OFSFileSystem extends DFSFileSystem { public OFSFileSystem(HdfsCompatibleProperties properties) { - super(properties, StorageBackend.StorageType.OFS); + super(properties, FsStorageType.OFS); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java index 636cb6ecb84..4d8f25de0bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java @@ -17,11 +17,11 @@ package org.apache.doris.fs.remote.dfs; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.datasource.property.storage.OSSHdfsProperties; +import org.apache.doris.foundation.fs.FsStorageType; public class OSSHdfsFileSystem extends DFSFileSystem { public OSSHdfsFileSystem(OSSHdfsProperties properties) { - super(properties, StorageBackend.StorageType.HDFS); + super(properties, FsStorageType.HDFS); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/spi/FileSystemSpiProvider.java b/fe/fe-core/src/main/java/org/apache/doris/fs/spi/FileSystemSpiProvider.java new file mode 100644 index 00000000000..ffacffd8f33 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/spi/FileSystemSpiProvider.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fs.spi; + +import org.apache.doris.fs.FileSystem; + +import java.io.IOException; +import java.util.Map; + +/** + * Service Provider Interface (SPI) for filesystem implementations. + * + * This interface has zero dependencies outside JDK and FileSystem (which will + * move to fe-filesystem-spi). Storage implementation modules register their + * implementations via Java ServiceLoader. + * + * In Phase 3, this interface (along with FileSystem) will be moved to the + * fe-filesystem-spi Maven module. Each implementation module will provide: + * META-INF/services/org.apache.doris.filesystem.spi.FileSystemSpiProvider + * + * Phase 0: Only skeleton definition. Not yet wired into FileSystemFactory. + */ +public interface FileSystemSpiProvider { + /** + * Returns the human-readable name of this storage implementation (for logging). + */ + String storageName(); + + /** + * Returns true if this provider can create a FileSystem from the given properties. + * Must be fast (no I/O, no network calls). + * + * @param properties storage configuration key-value map, never null + */ + boolean supports(Map<String, String> properties); + + /** + * Creates and returns a new FileSystem instance for the given properties. + * + * @param properties storage configuration key-value map + * @throws IOException if the filesystem cannot be created + */ + FileSystem create(Map<String, String> properties) throws IOException; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index 67c0a1fcb53..93af1af7463 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -179,13 +179,6 @@ import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalTable; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.fs.PersistentFileSystem; -import org.apache.doris.fs.remote.AzureFileSystem; -import org.apache.doris.fs.remote.BrokerFileSystem; -import org.apache.doris.fs.remote.ObjFileSystem; -import org.apache.doris.fs.remote.S3FileSystem; -import org.apache.doris.fs.remote.dfs.DFSFileSystem; -import org.apache.doris.fs.remote.dfs.JFSFileSystem; -import org.apache.doris.fs.remote.dfs.OFSFileSystem; import org.apache.doris.job.extensions.insert.InsertJob; import org.apache.doris.job.extensions.insert.streaming.StreamingInsertJob; import org.apache.doris.job.extensions.insert.streaming.StreamingTaskTxnCommitAttachment; @@ -223,6 +216,8 @@ import org.apache.doris.system.HeartbeatResponse; import org.apache.doris.transaction.TxnCommitAttachment; import com.google.common.base.Preconditions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.HashBasedTable; import com.google.common.collect.HashMultimap; @@ -289,6 +284,8 @@ import java.util.zip.GZIPOutputStream; * See the following "GuavaTableAdapter" and "GuavaMultimapAdapter" for example. */ public class GsonUtils { + private static final Logger LOG = LogManager.getLogger(GsonUtils.class); + // runtime adapter for class "Type" private static RuntimeTypeAdapterFactory<org.apache.doris.catalog.Type> columnTypeAdapterFactory = RuntimeTypeAdapterFactory @@ -578,14 +575,38 @@ public class GsonUtils { .registerSubtype(KafkaRoutineLoadJob.class, KafkaRoutineLoadJob.class.getSimpleName()); private static RuntimeTypeAdapterFactory<PersistentFileSystem> remoteFileSystemTypeAdapterFactory - = RuntimeTypeAdapterFactory.of(PersistentFileSystem.class, "clazz") - .registerSubtype(BrokerFileSystem.class, BrokerFileSystem.class.getSimpleName()) - .registerSubtype(DFSFileSystem.class, DFSFileSystem.class.getSimpleName()) - .registerSubtype(JFSFileSystem.class, JFSFileSystem.class.getSimpleName()) - .registerSubtype(OFSFileSystem.class, OFSFileSystem.class.getSimpleName()) - .registerSubtype(ObjFileSystem.class, ObjFileSystem.class.getSimpleName()) - .registerSubtype(S3FileSystem.class, S3FileSystem.class.getSimpleName()) - .registerSubtype(AzureFileSystem.class, AzureFileSystem.class.getSimpleName()); + = buildLegacyFileSystemAdapterFactory(); + + private static RuntimeTypeAdapterFactory<PersistentFileSystem> buildLegacyFileSystemAdapterFactory() { + RuntimeTypeAdapterFactory<PersistentFileSystem> factory = + RuntimeTypeAdapterFactory.of(PersistentFileSystem.class, "clazz"); + // Register via reflection to avoid compile-time dependency on concrete classes. + // These registrations exist only for backward-compatible deserialization of old metadata. + // New metadata uses FileSystemDescriptor, which does not require these registrations. + String[][] subtypes = { + {"BrokerFileSystem", "org.apache.doris.fs.remote.BrokerFileSystem"}, + {"DFSFileSystem", "org.apache.doris.fs.remote.dfs.DFSFileSystem"}, + {"JFSFileSystem", "org.apache.doris.fs.remote.dfs.JFSFileSystem"}, + {"OFSFileSystem", "org.apache.doris.fs.remote.dfs.OFSFileSystem"}, + {"ObjFileSystem", "org.apache.doris.fs.remote.ObjFileSystem"}, + {"S3FileSystem", "org.apache.doris.fs.remote.S3FileSystem"}, + {"AzureFileSystem", "org.apache.doris.fs.remote.AzureFileSystem"}, + }; + for (String[] entry : subtypes) { + String simpleName = entry[0]; + String fqcn = entry[1]; + try { + Class<?> clazz = Class.forName(fqcn); + if (PersistentFileSystem.class.isAssignableFrom(clazz)) { + factory.registerSubtype( + clazz.asSubclass(PersistentFileSystem.class), simpleName); + } + } catch (ClassNotFoundException e) { + LOG.warn("Legacy FileSystem class '{}' not found, skipping GSON registration.", fqcn); + } + } + return factory; + } private static RuntimeTypeAdapterFactory<org.apache.doris.backup.AbstractJob> jobBackupTypeAdapterFactory diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java index 5269b707b06..c25d59b43c7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java @@ -17,9 +17,9 @@ package org.apache.doris.fs.remote; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.backup.Status; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.foundation.fs.FsStorageType; import org.apache.hadoop.fs.FileSystem; import org.junit.jupiter.api.Assertions; @@ -36,7 +36,7 @@ public class RemoteFileSystemTest { private static class DummyRemoteFileSystem extends RemoteFileSystem { public DummyRemoteFileSystem() { - super("dummy", StorageBackend.StorageType.HDFS); + super("dummy", FsStorageType.HDFS); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java b/fe/fe-foundation/src/main/java/org/apache/doris/foundation/fs/FsStorageType.java similarity index 66% copy from fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java copy to fe/fe-foundation/src/main/java/org/apache/doris/foundation/fs/FsStorageType.java index 636cb6ecb84..3fea2b230da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/OSSHdfsFileSystem.java +++ b/fe/fe-foundation/src/main/java/org/apache/doris/foundation/fs/FsStorageType.java @@ -15,13 +15,24 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.fs.remote.dfs; +package org.apache.doris.foundation.fs; -import org.apache.doris.analysis.StorageBackend; -import org.apache.doris.datasource.property.storage.OSSHdfsProperties; +/** + * Storage type enum for persistent file system identification. + * Intentionally in fe-foundation (zero-dependency module) so that + * fe-filesystem-spi can reference it without depending on fe-thrift. + */ +public enum FsStorageType { + S3, + HDFS, + BROKER, + AZURE, + OFS, + JFS, + OSS_HDFS, + LOCAL; -public class OSSHdfsFileSystem extends DFSFileSystem { - public OSSHdfsFileSystem(OSSHdfsProperties properties) { - super(properties, StorageBackend.StorageType.HDFS); + public String typeName() { + return name(); } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
