This is an automated email from the ASF dual-hosted git repository.
koushiro pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 70ff36c5f refactor(services/hdfs): split hdfs into separate crate
(#7040)
70ff36c5f is described below
commit 70ff36c5f6066b3a5c7972198b769d9b651e4b3e
Author: Jintao Zhang <[email protected]>
AuthorDate: Fri Dec 19 21:27:44 2025 +0800
refactor(services/hdfs): split hdfs into separate crate (#7040)
* refactor(services/hdfs): split hdfs into separate crate
This PR splits the hdfs service into its own crate `opendal-service-hdfs`.
- Create new crate at `core/services/hdfs`
- Update feature flag to use the new crate
- Remove hdfs from `opendal-core`
- Re-export from facade crate
Closes #6899
* fix: format code
* fix: address reviewer comment - use opendal_core::OperatorUri
* fix: remove incorrectly added ipmfs and lakefs module declarations
* fix: update hdfs doctest to use the new separate crate import
---
core/Cargo.lock | 16 ++++++++-
core/Cargo.toml | 3 +-
core/core/Cargo.toml | 6 ++--
core/core/src/services/mod.rs | 5 ---
core/services/hdfs/Cargo.toml | 42 ++++++++++++++++++++++
.../services/hdfs => services/hdfs/src}/backend.rs | 4 +--
.../services/hdfs => services/hdfs/src}/config.rs | 8 ++---
.../services/hdfs => services/hdfs/src}/core.rs | 4 +--
.../services/hdfs => services/hdfs/src}/deleter.rs | 4 +--
.../services/hdfs => services/hdfs/src}/docs.md | 7 ++--
.../hdfs/mod.rs => services/hdfs/src/lib.rs} | 2 +-
.../services/hdfs => services/hdfs/src}/lister.rs | 8 ++---
.../services/hdfs => services/hdfs/src}/reader.rs | 4 +--
.../services/hdfs => services/hdfs/src}/writer.rs | 4 +--
core/src/lib.rs | 2 ++
15 files changed, 85 insertions(+), 34 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index 0b1a64ac4..e5f0ef985 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -5576,6 +5576,7 @@ dependencies = [
"opendal-service-gdrive",
"opendal-service-ghac",
"opendal-service-github",
+ "opendal-service-hdfs",
"opendal-service-hdfs-native",
"opendal-service-http",
"opendal-service-huggingface",
@@ -5660,7 +5661,6 @@ dependencies = [
"flume",
"futures",
"getrandom 0.2.16",
- "hdrs",
"http 1.4.0",
"http-body 1.0.1",
"jiff",
@@ -6264,6 +6264,20 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "opendal-service-hdfs"
+version = "0.55.0"
+dependencies = [
+ "bytes",
+ "ctor",
+ "futures",
+ "hdrs",
+ "log",
+ "opendal-core",
+ "serde",
+ "tokio",
+]
+
[[package]]
name = "opendal-service-hdfs-native"
version = "0.55.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 56573c212..d1db38515 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -134,7 +134,7 @@ services-gdrive = ["dep:opendal-service-gdrive"]
services-ghac = ["dep:opendal-service-ghac"]
services-github = ["dep:opendal-service-github"]
services-gridfs = ["opendal-core/services-gridfs"]
-services-hdfs = ["opendal-core/services-hdfs"]
+services-hdfs = ["dep:opendal-service-hdfs"]
services-hdfs-native = ["dep:opendal-service-hdfs-native"]
services-http = ["dep:opendal-service-http"]
services-huggingface = ["dep:opendal-service-huggingface"]
@@ -238,6 +238,7 @@ opendal-service-gcs = { path = "services/gcs", version =
"0.55.0", optional = tr
opendal-service-gdrive = { path = "services/gdrive", version = "0.55.0",
optional = true, default-features = false }
opendal-service-ghac = { path = "services/ghac", version = "0.55.0", optional
= true, default-features = false }
opendal-service-github = { path = "services/github", version = "0.55.0",
optional = true, default-features = false }
+opendal-service-hdfs = { path = "services/hdfs", version = "0.55.0", optional
= true, default-features = false }
opendal-service-hdfs-native = { path = "services/hdfs-native", version =
"0.55.0", optional = true, default-features = false }
opendal-service-http = { path = "services/http", version = "0.55.0", optional
= true, default-features = false }
opendal-service-huggingface = { path = "services/huggingface", version =
"0.55.0", optional = true, default-features = false }
diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml
index a82cf240e..d2037a313 100644
--- a/core/core/Cargo.toml
+++ b/core/core/Cargo.toml
@@ -58,7 +58,8 @@ executors-tokio = ["tokio/rt"]
layers-dtrace = ["dep:probe"]
services-gridfs = ["dep:mongodb", "dep:mongodb-internal-macros"]
-services-hdfs = ["dep:hdrs"]
+services-ipmfs = []
+services-lakefs = []
services-memory = []
services-mongodb = ["dep:mongodb", "dep:mongodb-internal-macros"]
services-monoiofs = ["dep:monoio", "dep:flume"]
@@ -106,8 +107,7 @@ sha2 = { workspace = true, optional = true }
# For http based services.
reqsign = { workspace = true, features = ["reqwest_request"], optional = true }
-# for services-hdfs
-hdrs = { version = "0.3.2", optional = true, features = ["async_file"] }
+
# for services-moka
moka = { version = "0.12", optional = true, features = ["future", "sync"] }
# for services-mongodb
diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs
index 5341f438b..186f7f0ff 100644
--- a/core/core/src/services/mod.rs
+++ b/core/core/src/services/mod.rs
@@ -24,11 +24,6 @@ mod gridfs;
#[cfg(feature = "services-gridfs")]
pub use gridfs::*;
-#[cfg(feature = "services-hdfs")]
-mod hdfs;
-#[cfg(feature = "services-hdfs")]
-pub use self::hdfs::*;
-
#[cfg(feature = "services-memory")]
mod memory;
#[cfg(feature = "services-memory")]
diff --git a/core/services/hdfs/Cargo.toml b/core/services/hdfs/Cargo.toml
new file mode 100644
index 000000000..5e615429f
--- /dev/null
+++ b/core/services/hdfs/Cargo.toml
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL hdfs service implementation"
+name = "opendal-service-hdfs"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+
+bytes = { workspace = true }
+ctor = { workspace = true }
+futures = { workspace = true }
+hdrs = { version = "0.3.2", features = ["async_file"] }
+log = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+tokio = { workspace = true }
diff --git a/core/core/src/services/hdfs/backend.rs
b/core/services/hdfs/src/backend.rs
similarity index 99%
rename from core/core/src/services/hdfs/backend.rs
rename to core/services/hdfs/src/backend.rs
index ae1a5b887..5b00b59f8 100644
--- a/core/core/src/services/hdfs/backend.rs
+++ b/core/services/hdfs/src/backend.rs
@@ -27,8 +27,8 @@ use super::deleter::HdfsDeleter;
use super::lister::HdfsLister;
use super::reader::HdfsReader;
use super::writer::HdfsWriter;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
#[doc = include_str!("docs.md")]
#[derive(Debug, Default)]
diff --git a/core/core/src/services/hdfs/config.rs
b/core/services/hdfs/src/config.rs
similarity index 94%
rename from core/core/src/services/hdfs/config.rs
rename to core/services/hdfs/src/config.rs
index ddfcc1c95..2e086c82f 100644
--- a/core/core/src/services/hdfs/config.rs
+++ b/core/services/hdfs/src/config.rs
@@ -59,10 +59,10 @@ impl Debug for HdfsConfig {
}
}
-impl crate::Configurator for HdfsConfig {
+impl opendal_core::Configurator for HdfsConfig {
type Builder = HdfsBuilder;
- fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> {
+ fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self>
{
let mut map = uri.options().clone();
if let Some(authority) = uri.authority() {
map.insert("name_node".to_string(), format!("hdfs://{authority}"));
@@ -85,8 +85,8 @@ impl crate::Configurator for HdfsConfig {
#[cfg(test)]
mod tests {
use super::*;
- use crate::Configurator;
- use crate::types::OperatorUri;
+ use opendal_core::Configurator;
+ use opendal_core::OperatorUri;
#[test]
fn from_uri_sets_name_node_and_root() {
diff --git a/core/core/src/services/hdfs/core.rs
b/core/services/hdfs/src/core.rs
similarity index 99%
rename from core/core/src/services/hdfs/core.rs
rename to core/services/hdfs/src/core.rs
index 75025fb3e..997d62af5 100644
--- a/core/core/src/services/hdfs/core.rs
+++ b/core/services/hdfs/src/core.rs
@@ -20,8 +20,8 @@ use std::io;
use std::io::SeekFrom;
use std::sync::Arc;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
/// HdfsCore contains code that directly interacts with HDFS.
#[derive(Clone)]
diff --git a/core/core/src/services/hdfs/deleter.rs
b/core/services/hdfs/src/deleter.rs
similarity index 97%
rename from core/core/src/services/hdfs/deleter.rs
rename to core/services/hdfs/src/deleter.rs
index 11471d794..eb0eeb8a8 100644
--- a/core/core/src/services/hdfs/deleter.rs
+++ b/core/services/hdfs/src/deleter.rs
@@ -19,8 +19,8 @@ use std::io;
use std::sync::Arc;
use super::core::HdfsCore;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct HdfsDeleter {
core: Arc<HdfsCore>,
diff --git a/core/core/src/services/hdfs/docs.md
b/core/services/hdfs/src/docs.md
similarity index 97%
rename from core/core/src/services/hdfs/docs.md
rename to core/services/hdfs/src/docs.md
index fb6429036..a410b9a5b 100644
--- a/core/core/src/services/hdfs/docs.md
+++ b/core/services/hdfs/src/docs.md
@@ -107,14 +107,11 @@ Enabling the vendored feature ensures that hdrs includes
the necessary libhdfs.s
### Via Builder
```rust,no_run
-use std::sync::Arc;
-
-use anyhow::Result;
-use opendal_core::services::Hdfs;
use opendal_core::Operator;
+use opendal_service_hdfs::Hdfs;
#[tokio::main]
-async fn main() -> Result<()> {
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Create fs backend builder.
let mut builder = Hdfs::default()
// Set the name node for hdfs.
diff --git a/core/core/src/services/hdfs/mod.rs b/core/services/hdfs/src/lib.rs
similarity index 96%
rename from core/core/src/services/hdfs/mod.rs
rename to core/services/hdfs/src/lib.rs
index de2c0e349..24d852a91 100644
--- a/core/core/src/services/hdfs/mod.rs
+++ b/core/services/hdfs/src/lib.rs
@@ -18,7 +18,7 @@
/// Default scheme for hdfs service.
pub const HDFS_SCHEME: &str = "hdfs";
-use crate::types::DEFAULT_OPERATOR_REGISTRY;
+use opendal_core::DEFAULT_OPERATOR_REGISTRY;
mod backend;
mod config;
diff --git a/core/core/src/services/hdfs/lister.rs
b/core/services/hdfs/src/lister.rs
similarity index 95%
rename from core/core/src/services/hdfs/lister.rs
rename to core/services/hdfs/src/lister.rs
index 84bea9340..e97dd0eae 100644
--- a/core/core/src/services/hdfs/lister.rs
+++ b/core/services/hdfs/src/lister.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use crate::EntryMode;
-use crate::Metadata;
-use crate::Result;
-use crate::raw::*;
+use opendal_core::EntryMode;
+use opendal_core::Metadata;
+use opendal_core::Result;
+use opendal_core::raw::*;
pub struct HdfsLister {
root: String,
diff --git a/core/core/src/services/hdfs/reader.rs
b/core/services/hdfs/src/reader.rs
similarity index 97%
rename from core/core/src/services/hdfs/reader.rs
rename to core/services/hdfs/src/reader.rs
index 2cbddcd41..670fad5e5 100644
--- a/core/core/src/services/hdfs/reader.rs
+++ b/core/services/hdfs/src/reader.rs
@@ -20,8 +20,8 @@ use futures::AsyncReadExt;
use hdrs::AsyncFile;
use tokio::io::ReadBuf;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct HdfsReader<F> {
f: F,
diff --git a/core/core/src/services/hdfs/writer.rs
b/core/services/hdfs/src/writer.rs
similarity index 98%
rename from core/core/src/services/hdfs/writer.rs
rename to core/services/hdfs/src/writer.rs
index 4c44cac85..0c5aec3b2 100644
--- a/core/core/src/services/hdfs/writer.rs
+++ b/core/services/hdfs/src/writer.rs
@@ -20,8 +20,8 @@ use std::sync::Arc;
use bytes::Buf;
use futures::AsyncWriteExt;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct HdfsWriter<F> {
target_path: String,
diff --git a/core/src/lib.rs b/core/src/lib.rs
index bc8d788d3..dd823fb56 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -74,6 +74,8 @@ pub mod services {
pub use opendal_service_ghac::*;
#[cfg(feature = "services-github")]
pub use opendal_service_github::*;
+ #[cfg(feature = "services-hdfs")]
+ pub use opendal_service_hdfs::*;
#[cfg(feature = "services-hdfs-native")]
pub use opendal_service_hdfs_native::*;
#[cfg(feature = "services-http")]
