This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new b7e5daca1 refactor: Split service hdfs-native to new crate (#6953)
b7e5daca1 is described below
commit b7e5daca11d32104701c637b26391f688779a5a8
Author: Chitral Verma <[email protected]>
AuthorDate: Mon Dec 8 19:03:52 2025 +0530
refactor: Split service hdfs-native to new crate (#6953)
* crate for hdfs-native service
* update docs.md
---
core/Cargo.lock | 14 ++++++++++++++
core/Cargo.toml | 7 ++++---
core/core/Cargo.toml | 3 +--
core/core/src/services/mod.rs | 5 -----
core/edge/file_write_on_full_disk/Cargo.toml | 2 +-
core/examples/basic/Cargo.toml | 2 +-
core/examples/concurrent-upload/Cargo.toml | 2 +-
core/examples/multipart-upload/Cargo.toml | 2 +-
core/layers/await-tree/Cargo.toml | 2 +-
.../{layers/await-tree => services/hdfs-native}/Cargo.toml | 12 ++++++++----
.../hdfs_native => services/hdfs-native/src}/backend.rs | 4 ++--
.../hdfs_native => services/hdfs-native/src}/config.rs | 8 ++++----
.../hdfs_native => services/hdfs-native/src}/core.rs | 4 ++--
.../hdfs_native => services/hdfs-native/src}/deleter.rs | 4 ++--
.../hdfs_native => services/hdfs-native/src}/docs.md | 12 ------------
.../hdfs_native => services/hdfs-native/src}/error.rs | 2 +-
.../hdfs_native/mod.rs => services/hdfs-native/src/lib.rs} | 4 ++--
.../hdfs_native => services/hdfs-native/src}/lister.rs | 13 ++++++-------
.../hdfs_native => services/hdfs-native/src}/reader.rs | 6 +++---
.../hdfs_native => services/hdfs-native/src}/writer.rs | 6 +++---
core/src/lib.rs | 2 ++
21 files changed, 59 insertions(+), 57 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index baa1b2579..501ccb4bb 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -5483,6 +5483,7 @@ dependencies = [
"opendal-service-azdls",
"opendal-service-azfile",
"opendal-service-ghac",
+ "opendal-service-hdfs-native",
"opendal-service-moka",
"opendal-service-s3",
"opentelemetry",
@@ -5769,6 +5770,19 @@ dependencies = [
"sha2",
]
+[[package]]
+name = "opendal-service-hdfs-native"
+version = "0.55.0"
+dependencies = [
+ "bytes",
+ "ctor",
+ "futures",
+ "hdfs-native",
+ "log",
+ "opendal-core",
+ "serde",
+]
+
[[package]]
name = "opendal-service-moka"
version = "0.55.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index d087b0414..390abab99 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -24,7 +24,6 @@ members = [
"fuzz",
"edge/*",
"benches/vs_*",
- "services/aliyun-drive",
"services/*",
"layers/*",
"layers/await-tree",
@@ -46,6 +45,7 @@ version = "0.55.0"
base64 = "0.22"
bytes = "1.10"
ctor = "0.6"
+futures = { version = "0.3", default-features = false }
http = "1.1"
log = "0.4"
quick-xml = { version = "0.38", default-features = false }
@@ -129,7 +129,7 @@ services-ghac = ["dep:opendal-service-ghac"]
services-github = ["opendal-core/services-github"]
services-gridfs = ["opendal-core/services-gridfs"]
services-hdfs = ["opendal-core/services-hdfs"]
-services-hdfs-native = ["opendal-core/services-hdfs-native"]
+services-hdfs-native = ["dep:opendal-service-hdfs-native"]
services-http = ["opendal-core/services-http"]
services-huggingface = ["opendal-core/services-huggingface"]
services-ipfs = ["opendal-core/services-ipfs"]
@@ -198,6 +198,7 @@ opendal-service-azblob = { path = "services/azblob",
version = "0.55.0", optiona
opendal-service-azdls = { path = "services/azdls", version = "0.55.0",
optional = true, default-features = false }
opendal-service-azfile = { path = "services/azfile", version = "0.55.0",
optional = true, default-features = false }
opendal-service-ghac = { path = "services/ghac", version = "0.55.0", optional
= true, default-features = false }
+opendal-service-hdfs-native = { path = "services/hdfs-native", version =
"0.55.0", optional = true, default-features = false }
opendal-service-moka = { path = "services/moka", version = "0.55.0", optional
= true, default-features = false }
opendal-service-s3 = { path = "services/s3", version = "0.55.0", optional =
true, default-features = false }
@@ -209,7 +210,7 @@ divan = { version = "0.1" }
dotenvy = "0.15"
fastrace = { version = "0.7", features = ["enable"] }
fastrace-jaeger = "0.7"
-futures = "0.3"
+futures = { workspace = true, default-features = true }
http = { workspace = true }
libtest-mimic = "0.8"
log = { workspace = true }
diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml
index a268c0478..3846596f1 100644
--- a/core/core/Cargo.toml
+++ b/core/core/Cargo.toml
@@ -127,7 +127,6 @@ services-gdrive = ["internal-path-cache"]
services-github = []
services-gridfs = ["dep:mongodb", "dep:mongodb-internal-macros"]
services-hdfs = ["dep:hdrs"]
-services-hdfs-native = ["hdfs-native"]
services-http = []
services-huggingface = []
services-ipfs = ["dep:prost"]
@@ -189,7 +188,7 @@ backon = { version = "1.6", features = ["tokio-sleep"] }
base64 = { workspace = true }
bytes = { workspace = true }
ctor = { workspace = true }
-futures = { version = "0.3", default-features = false, features = [
+futures = { workspace = true, default-features = false, features = [
"std",
"async-await",
] }
diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs
index d0dfb83a4..2033e882a 100644
--- a/core/core/src/services/mod.rs
+++ b/core/core/src/services/mod.rs
@@ -114,11 +114,6 @@ mod hdfs;
#[cfg(feature = "services-hdfs")]
pub use self::hdfs::*;
-#[cfg(feature = "services-hdfs-native")]
-mod hdfs_native;
-#[cfg(feature = "services-hdfs-native")]
-pub use hdfs_native::*;
-
#[cfg(feature = "services-http")]
mod http;
#[cfg(feature = "services-http")]
diff --git a/core/edge/file_write_on_full_disk/Cargo.toml
b/core/edge/file_write_on_full_disk/Cargo.toml
index df157804c..8f7e0c75f 100644
--- a/core/edge/file_write_on_full_disk/Cargo.toml
+++ b/core/edge/file_write_on_full_disk/Cargo.toml
@@ -25,7 +25,7 @@ rust-version.workspace = true
version.workspace = true
[dependencies]
-futures = "0.3"
+futures = { workspace = true, default-features = true }
opendal = { path = "../..", features = ["services-memory", "services-fs"] }
rand = { workspace = true }
tokio = { workspace = true, features = ["full"] }
diff --git a/core/examples/basic/Cargo.toml b/core/examples/basic/Cargo.toml
index d9020fe31..b200bebc8 100644
--- a/core/examples/basic/Cargo.toml
+++ b/core/examples/basic/Cargo.toml
@@ -24,6 +24,6 @@ rust-version.workspace = true
version.workspace = true
[dependencies]
-futures = "0.3"
+futures = { workspace = true, default-features = true }
opendal = { path = "../..", features = ["tests"] }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/core/examples/concurrent-upload/Cargo.toml
b/core/examples/concurrent-upload/Cargo.toml
index ffd8bde0f..b27c7752c 100644
--- a/core/examples/concurrent-upload/Cargo.toml
+++ b/core/examples/concurrent-upload/Cargo.toml
@@ -24,6 +24,6 @@ rust-version.workspace = true
version.workspace = true
[dependencies]
-futures = "0.3"
+futures = { workspace = true, default-features = true }
opendal = { path = "../..", features = ["tests"] }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/core/examples/multipart-upload/Cargo.toml
b/core/examples/multipart-upload/Cargo.toml
index 6a33a22e2..3ba85904e 100644
--- a/core/examples/multipart-upload/Cargo.toml
+++ b/core/examples/multipart-upload/Cargo.toml
@@ -24,6 +24,6 @@ rust-version.workspace = true
version.workspace = true
[dependencies]
-futures = "0.3"
+futures = { workspace = true, default-features = true }
opendal = { path = "../..", features = ["tests"] }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/core/layers/await-tree/Cargo.toml
b/core/layers/await-tree/Cargo.toml
index 61c293f60..d062d7ebe 100644
--- a/core/layers/await-tree/Cargo.toml
+++ b/core/layers/await-tree/Cargo.toml
@@ -34,4 +34,4 @@ all-features = true
opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
await-tree = "0.3"
-futures = "0.3"
+futures = { workspace = true, default-features = true }
diff --git a/core/layers/await-tree/Cargo.toml
b/core/services/hdfs-native/Cargo.toml
similarity index 79%
copy from core/layers/await-tree/Cargo.toml
copy to core/services/hdfs-native/Cargo.toml
index 61c293f60..b6186be67 100644
--- a/core/layers/await-tree/Cargo.toml
+++ b/core/services/hdfs-native/Cargo.toml
@@ -16,8 +16,8 @@
# under the License.
[package]
-description = "Apache OpenDAL await-tree layer"
-name = "opendal-layer-await-tree"
+description = "Apache OpenDAL HDFS Native service implementation"
+name = "opendal-service-hdfs-native"
authors = { workspace = true }
edition = { workspace = true }
@@ -33,5 +33,9 @@ all-features = true
[dependencies]
opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
-await-tree = "0.3"
-futures = "0.3"
+bytes = { workspace = true }
+ctor = { workspace = true }
+futures = { workspace = true }
+hdfs-native = { version = "0.13" }
+log = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
diff --git a/core/core/src/services/hdfs_native/backend.rs
b/core/services/hdfs-native/src/backend.rs
similarity index 99%
rename from core/core/src/services/hdfs_native/backend.rs
rename to core/services/hdfs-native/src/backend.rs
index a0af8a3a3..27a44e026 100644
--- a/core/core/src/services/hdfs_native/backend.rs
+++ b/core/services/hdfs-native/src/backend.rs
@@ -27,8 +27,8 @@ use super::error::parse_hdfs_error;
use super::lister::HdfsNativeLister;
use super::reader::HdfsNativeReader;
use super::writer::HdfsNativeWriter;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
/// [Hadoop Distributed File System (HDFS™)](https://hadoop.apache.org/)
support.
/// Using [Native Rust HDFS client](https://github.com/Kimahriman/hdfs-native).
diff --git a/core/core/src/services/hdfs_native/config.rs
b/core/services/hdfs-native/src/config.rs
similarity index 93%
rename from core/core/src/services/hdfs_native/config.rs
rename to core/services/hdfs-native/src/config.rs
index eeb14e177..2861ba720 100644
--- a/core/core/src/services/hdfs_native/config.rs
+++ b/core/services/hdfs-native/src/config.rs
@@ -45,10 +45,10 @@ impl Debug for HdfsNativeConfig {
}
}
-impl crate::Configurator for HdfsNativeConfig {
+impl opendal_core::Configurator for HdfsNativeConfig {
type Builder = HdfsNativeBuilder;
- fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> {
+ fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self>
{
let mut map = uri.options().clone();
if let Some(authority) = uri.authority() {
map.insert("name_node".to_string(), format!("hdfs://{authority}"));
@@ -71,8 +71,8 @@ impl crate::Configurator for HdfsNativeConfig {
#[cfg(test)]
mod tests {
use super::*;
- use crate::Configurator;
- use crate::types::OperatorUri;
+ use opendal_core::Configurator;
+ use opendal_core::OperatorUri;
#[test]
fn from_uri_sets_name_node_and_root() {
diff --git a/core/core/src/services/hdfs_native/core.rs
b/core/services/hdfs-native/src/core.rs
similarity index 99%
rename from core/core/src/services/hdfs_native/core.rs
rename to core/services/hdfs-native/src/core.rs
index ab38e0699..da2808c0f 100644
--- a/core/core/src/services/hdfs_native/core.rs
+++ b/core/services/hdfs-native/src/core.rs
@@ -22,8 +22,8 @@ use hdfs_native::HdfsError;
use hdfs_native::WriteOptions;
use super::error::parse_hdfs_error;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
/// HdfsNativeCore contains code that directly interacts with HDFS Native
client.
#[derive(Clone)]
diff --git a/core/core/src/services/hdfs_native/deleter.rs
b/core/services/hdfs-native/src/deleter.rs
similarity index 96%
rename from core/core/src/services/hdfs_native/deleter.rs
rename to core/services/hdfs-native/src/deleter.rs
index 648f15346..454ef80b4 100644
--- a/core/core/src/services/hdfs_native/deleter.rs
+++ b/core/services/hdfs-native/src/deleter.rs
@@ -18,8 +18,8 @@
use std::sync::Arc;
use super::core::HdfsNativeCore;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct HdfsNativeDeleter {
core: Arc<HdfsNativeCore>,
diff --git a/core/core/src/services/hdfs_native/docs.md
b/core/services/hdfs-native/src/docs.md
similarity index 62%
rename from core/core/src/services/hdfs_native/docs.md
rename to core/services/hdfs-native/src/docs.md
index 1df626f66..9d5230e82 100644
--- a/core/core/src/services/hdfs_native/docs.md
+++ b/core/services/hdfs-native/src/docs.md
@@ -15,18 +15,6 @@ This service can be used to:
- [x] rename
- [ ] ~~presign~~
-## Differences with webhdfs
-
-[Webhdfs][crate::services::Webhdfs] is powered by hdfs's RESTful HTTP API.
-
-## Differences with hdfs
-
-[hdfs][crate::services::Hdfs] is powered by libhdfs and require the Java
dependencies
-
-## Features
-
-HDFS-native support needs to enable feature `services-hdfs-native`.
-
## Configuration
- `root`: Set the work dir for backend.
diff --git a/core/core/src/services/hdfs_native/error.rs
b/core/services/hdfs-native/src/error.rs
similarity index 99%
rename from core/core/src/services/hdfs_native/error.rs
rename to core/services/hdfs-native/src/error.rs
index f3d637540..5d1716bf8 100644
--- a/core/core/src/services/hdfs_native/error.rs
+++ b/core/services/hdfs-native/src/error.rs
@@ -17,7 +17,7 @@
use hdfs_native::HdfsError;
-use crate::*;
+use opendal_core::*;
/// Parse hdfs-native error into opendal::Error.
pub fn parse_hdfs_error(hdfs_error: HdfsError) -> Error {
diff --git a/core/core/src/services/hdfs_native/mod.rs
b/core/services/hdfs-native/src/lib.rs
similarity index 93%
rename from core/core/src/services/hdfs_native/mod.rs
rename to core/services/hdfs-native/src/lib.rs
index 6ff307f5a..1f8b77e36 100644
--- a/core/core/src/services/hdfs_native/mod.rs
+++ b/core/services/hdfs-native/src/lib.rs
@@ -18,7 +18,7 @@
/// Default scheme for hdfs-native service.
pub const HDFS_NATIVE_SCHEME: &str = "hdfs-native";
-use crate::types::DEFAULT_OPERATOR_REGISTRY;
+use opendal_core::DEFAULT_OPERATOR_REGISTRY;
mod backend;
mod config;
@@ -33,6 +33,6 @@ pub use backend::HdfsNativeBuilder as HdfsNative;
pub use config::HdfsNativeConfig;
#[ctor::ctor]
-fn register_hdfsnative_service() {
+fn register_hdfs_native_service() {
DEFAULT_OPERATOR_REGISTRY.register::<HdfsNative>(HDFS_NATIVE_SCHEME);
}
diff --git a/core/core/src/services/hdfs_native/lister.rs
b/core/services/hdfs-native/src/lister.rs
similarity index 92%
rename from core/core/src/services/hdfs_native/lister.rs
rename to core/services/hdfs-native/src/lister.rs
index b7583b024..d1d2f2af4 100644
--- a/core/core/src/services/hdfs_native/lister.rs
+++ b/core/services/hdfs-native/src/lister.rs
@@ -17,13 +17,12 @@
use hdfs_native::client::ListStatusIterator;
-use crate::EntryMode;
-use crate::Metadata;
-use crate::Result;
-use crate::raw::Timestamp;
-use crate::raw::build_rel_path;
-use crate::raw::oio;
-use crate::services::hdfs_native::error::parse_hdfs_error;
+use opendal_core::EntryMode;
+use opendal_core::Metadata;
+use opendal_core::Result;
+use opendal_core::raw::*;
+
+use crate::error::parse_hdfs_error;
pub struct HdfsNativeLister {
root: String,
diff --git a/core/core/src/services/hdfs_native/reader.rs
b/core/services/hdfs-native/src/reader.rs
similarity index 95%
rename from core/core/src/services/hdfs_native/reader.rs
rename to core/services/hdfs-native/src/reader.rs
index 011391819..267bb1e32 100644
--- a/core/core/src/services/hdfs_native/reader.rs
+++ b/core/services/hdfs-native/src/reader.rs
@@ -20,9 +20,9 @@ use futures::StreamExt;
use hdfs_native::HdfsError;
use hdfs_native::file::FileReader;
-use crate::raw::*;
-use crate::services::hdfs_native::error::parse_hdfs_error;
-use crate::*;
+use crate::error::parse_hdfs_error;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct HdfsNativeReader {
read: usize,
diff --git a/core/core/src/services/hdfs_native/writer.rs
b/core/services/hdfs-native/src/writer.rs
similarity index 94%
rename from core/core/src/services/hdfs_native/writer.rs
rename to core/services/hdfs-native/src/writer.rs
index 5033a83fe..9b719b679 100644
--- a/core/core/src/services/hdfs_native/writer.rs
+++ b/core/services/hdfs-native/src/writer.rs
@@ -17,9 +17,9 @@
use hdfs_native::file::FileWriter;
-use crate::raw::*;
-use crate::services::hdfs_native::error::parse_hdfs_error;
-use crate::*;
+use crate::error::parse_hdfs_error;
+use opendal_core::raw::*;
+use opendal_core::*;
pub struct HdfsNativeWriter {
f: FileWriter,
size: u64,
diff --git a/core/src/lib.rs b/core/src/lib.rs
index c6872d9c7..3f3da0356 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -37,6 +37,8 @@ pub mod services {
pub use opendal_service_azfile::*;
#[cfg(feature = "services-ghac")]
pub use opendal_service_ghac::*;
+ #[cfg(feature = "services-hdfs-native")]
+ pub use opendal_service_hdfs_native::*;
#[cfg(feature = "services-moka")]
pub use opendal_service_moka::*;
#[cfg(feature = "services-s3")]
