This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 1928b61a9 refactor: Split lakefs service to new crate (#7046)
1928b61a9 is described below
commit 1928b61a9209323dca1a66743224e332ec3e29e0
Author: Qinxuan Chen <[email protected]>
AuthorDate: Fri Dec 19 00:10:56 2025 +0800
refactor: Split lakefs service to new crate (#7046)
* refactor: Split lakefs service to new crate
* remove useless docs
* fix license check
---
core/Cargo.lock | 15 ++++++++
core/Cargo.toml | 3 +-
core/core/Cargo.toml | 1 -
core/core/src/services/mod.rs | 5 ---
core/services/lakefs/Cargo.toml | 43 ++++++++++++++++++++++
.../lakefs => services/lakefs/src}/backend.rs | 4 +-
.../lakefs => services/lakefs/src}/config.rs | 40 +++++++++-----------
.../lakefs => services/lakefs/src}/core.rs | 5 +--
.../lakefs => services/lakefs/src}/deleter.rs | 4 +-
.../lakefs => services/lakefs/src}/docs.md | 4 +-
.../lakefs => services/lakefs/src}/error.rs | 7 +---
.../lakefs/mod.rs => services/lakefs/src/lib.rs} | 11 ++++--
.../lakefs => services/lakefs/src}/lister.rs | 7 ++--
.../lakefs => services/lakefs/src}/writer.rs | 4 +-
core/src/lib.rs | 2 +
15 files changed, 103 insertions(+), 52 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index c0e70503f..6ceb01aca 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -5577,6 +5577,7 @@ dependencies = [
"opendal-service-huggingface",
"opendal-service-ipfs",
"opendal-service-koofr",
+ "opendal-service-lakefs",
"opendal-service-memcached",
"opendal-service-mini-moka",
"opendal-service-moka",
@@ -6277,6 +6278,20 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "opendal-service-lakefs"
+version = "0.55.0"
+dependencies = [
+ "bytes",
+ "ctor",
+ "http 1.4.0",
+ "log",
+ "opendal-core",
+ "serde",
+ "serde_json",
+ "tokio",
+]
+
[[package]]
name = "opendal-service-memcached"
version = "0.55.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index baedf191c..11621f814 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -131,7 +131,7 @@ services-huggingface = ["dep:opendal-service-huggingface"]
services-ipfs = ["dep:opendal-service-ipfs"]
services-ipmfs = ["opendal-core/services-ipmfs"]
services-koofr = ["dep:opendal-service-koofr"]
-services-lakefs = ["opendal-core/services-lakefs"]
+services-lakefs = ["dep:opendal-service-lakefs"]
services-memcached = ["dep:opendal-service-memcached"]
services-memory = ["opendal-core/services-memory"]
services-mini-moka = ["dep:opendal-service-mini-moka"]
@@ -229,6 +229,7 @@ opendal-service-http = { path = "services/http", version =
"0.55.0", optional =
opendal-service-huggingface = { path = "services/huggingface", version =
"0.55.0", optional = true, default-features = false }
opendal-service-ipfs = { path = "services/ipfs", version = "0.55.0", optional
= true, default-features = false }
opendal-service-koofr = { path = "services/koofr", version = "0.55.0",
optional = true, default-features = false }
+opendal-service-lakefs = { path = "services/lakefs", version = "0.55.0",
optional = true, default-features = false }
opendal-service-memcached = { path = "services/memcached", version = "0.55.0",
optional = true, default-features = false }
opendal-service-mini-moka = { path = "services/mini_moka", version = "0.55.0",
optional = true, default-features = false }
opendal-service-moka = { path = "services/moka", version = "0.55.0", optional
= true, default-features = false }
diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml
index 31e694217..03ab50ab9 100644
--- a/core/core/Cargo.toml
+++ b/core/core/Cargo.toml
@@ -61,7 +61,6 @@ services-gdrive = ["internal-path-cache"]
services-gridfs = ["dep:mongodb", "dep:mongodb-internal-macros"]
services-hdfs = ["dep:hdrs"]
services-ipmfs = []
-services-lakefs = []
services-memory = []
services-mongodb = ["dep:mongodb", "dep:mongodb-internal-macros"]
services-monoiofs = ["dep:monoio", "dep:flume"]
diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs
index 946996206..5b89dd0aa 100644
--- a/core/core/src/services/mod.rs
+++ b/core/core/src/services/mod.rs
@@ -39,11 +39,6 @@ mod ipmfs;
#[cfg(feature = "services-ipmfs")]
pub use ipmfs::*;
-#[cfg(feature = "services-lakefs")]
-mod lakefs;
-#[cfg(feature = "services-lakefs")]
-pub use lakefs::*;
-
#[cfg(feature = "services-memory")]
mod memory;
#[cfg(feature = "services-memory")]
diff --git a/core/services/lakefs/Cargo.toml b/core/services/lakefs/Cargo.toml
new file mode 100644
index 000000000..ce63dce93
--- /dev/null
+++ b/core/services/lakefs/Cargo.toml
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL lakeFS service implementation"
+name = "opendal-service-lakefs"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+bytes = { workspace = true }
+ctor = { workspace = true }
+http = { workspace = true }
+log = { workspace = true }
+opendal-core = { path = "../../core", version = "0.55.0", default-features =
false }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/core/core/src/services/lakefs/backend.rs
b/core/services/lakefs/src/backend.rs
similarity index 99%
rename from core/core/src/services/lakefs/backend.rs
rename to core/services/lakefs/src/backend.rs
index c4ffc6b6e..e789cc33c 100644
--- a/core/core/src/services/lakefs/backend.rs
+++ b/core/services/lakefs/src/backend.rs
@@ -21,6 +21,8 @@ use bytes::Buf;
use http::Response;
use http::StatusCode;
use log::debug;
+use opendal_core::raw::*;
+use opendal_core::*;
use super::LAKEFS_SCHEME;
use super::config::LakefsConfig;
@@ -30,8 +32,6 @@ use super::deleter::LakefsDeleter;
use super::error::parse_error;
use super::lister::LakefsLister;
use super::writer::LakefsWriter;
-use crate::raw::*;
-use crate::*;
/// [Lakefs](https://docs.lakefs.io/reference/api.html#/)'s API support.
#[doc = include_str!("docs.md")]
diff --git a/core/core/src/services/lakefs/config.rs
b/core/services/lakefs/src/config.rs
similarity index 83%
rename from core/core/src/services/lakefs/config.rs
rename to core/services/lakefs/src/config.rs
index dff70ec74..04a338808 100644
--- a/core/core/src/services/lakefs/config.rs
+++ b/core/services/lakefs/src/config.rs
@@ -17,6 +17,7 @@
use std::fmt::Debug;
+use opendal_core::*;
use serde::Deserialize;
use serde::Serialize;
@@ -66,16 +67,13 @@ impl Debug for LakefsConfig {
}
}
-impl crate::Configurator for LakefsConfig {
+impl Configurator for LakefsConfig {
type Builder = LakefsBuilder;
- fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> {
+ fn from_uri(uri: &OperatorUri) -> Result<Self> {
let raw_path = uri.root().ok_or_else(|| {
- crate::Error::new(
- crate::ErrorKind::ConfigInvalid,
- "uri path must contain repository",
- )
- .with_context("service", LAKEFS_SCHEME)
+ Error::new(ErrorKind::ConfigInvalid, "uri path must contain
repository")
+ .with_context("service", LAKEFS_SCHEME)
})?;
let (repository, remainder) = match raw_path.split_once('/') {
@@ -89,8 +87,8 @@ impl crate::Configurator for LakefsConfig {
Some(repository)
}
.ok_or_else(|| {
- crate::Error::new(
- crate::ErrorKind::ConfigInvalid,
+ Error::new(
+ ErrorKind::ConfigInvalid,
"repository is required in uri path",
)
.with_context("service", LAKEFS_SCHEME)
@@ -136,42 +134,40 @@ impl crate::Configurator for LakefsConfig {
#[cfg(test)]
mod tests {
use super::*;
- use crate::Configurator;
- use crate::types::OperatorUri;
#[test]
- fn from_uri_sets_endpoint_repository_branch_and_root() {
+ fn from_uri_sets_endpoint_repository_branch_and_root() -> Result<()> {
let uri = OperatorUri::new(
"lakefs://api.example.com/sample/main/data/dir",
Vec::<(String, String)>::new(),
- )
- .unwrap();
+ )?;
- let cfg = LakefsConfig::from_uri(&uri).unwrap();
+ let cfg = LakefsConfig::from_uri(&uri)?;
assert_eq!(cfg.endpoint.as_deref(), Some("https://api.example.com";));
assert_eq!(cfg.repository.as_deref(), Some("sample"));
assert_eq!(cfg.branch.as_deref(), Some("main"));
assert_eq!(cfg.root.as_deref(), Some("data/dir"));
+ Ok(())
}
#[test]
- fn from_uri_requires_repository() {
- let uri =
- OperatorUri::new("lakefs://api.example.com", Vec::<(String,
String)>::new()).unwrap();
+ fn from_uri_requires_repository() -> Result<()> {
+ let uri = OperatorUri::new("lakefs://api.example.com", Vec::<(String,
String)>::new())?;
assert!(LakefsConfig::from_uri(&uri).is_err());
+ Ok(())
}
#[test]
- fn from_uri_respects_branch_override_and_sets_root() {
+ fn from_uri_respects_branch_override_and_sets_root() -> Result<()> {
let uri = OperatorUri::new(
"lakefs://api.example.com/sample/content",
vec![("branch".to_string(), "develop".to_string())],
- )
- .unwrap();
+ )?;
- let cfg = LakefsConfig::from_uri(&uri).unwrap();
+ let cfg = LakefsConfig::from_uri(&uri)?;
assert_eq!(cfg.branch.as_deref(), Some("develop"));
assert_eq!(cfg.root.as_deref(), Some("content"));
+ Ok(())
}
}
diff --git a/core/core/src/services/lakefs/core.rs
b/core/services/lakefs/src/core.rs
similarity index 99%
rename from core/core/src/services/lakefs/core.rs
rename to core/services/lakefs/src/core.rs
index 86051acd4..29e7ec64f 100644
--- a/core/core/src/services/lakefs/core.rs
+++ b/core/services/lakefs/src/core.rs
@@ -22,11 +22,10 @@ use std::sync::Arc;
use http::Request;
use http::Response;
use http::header;
+use opendal_core::raw::*;
+use opendal_core::*;
use serde::Deserialize;
-use crate::raw::*;
-use crate::*;
-
pub struct LakefsCore {
pub info: Arc<AccessorInfo>,
pub endpoint: String,
diff --git a/core/core/src/services/lakefs/deleter.rs
b/core/services/lakefs/src/deleter.rs
similarity index 97%
rename from core/core/src/services/lakefs/deleter.rs
rename to core/services/lakefs/src/deleter.rs
index a75820919..fb8ee462e 100644
--- a/core/core/src/services/lakefs/deleter.rs
+++ b/core/services/lakefs/src/deleter.rs
@@ -18,11 +18,11 @@
use std::sync::Arc;
use http::StatusCode;
+use opendal_core::raw::*;
+use opendal_core::*;
use super::core::*;
use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
pub struct LakefsDeleter {
core: Arc<LakefsCore>,
diff --git a/core/core/src/services/lakefs/docs.md
b/core/services/lakefs/src/docs.md
similarity index 96%
rename from core/core/src/services/lakefs/docs.md
rename to core/services/lakefs/src/docs.md
index 20348c7fd..cb23d4f85 100644
--- a/core/core/src/services/lakefs/docs.md
+++ b/core/services/lakefs/src/docs.md
@@ -34,8 +34,8 @@ Refer to [`LakefsBuilder`]'s public API docs for more
information.
```rust,no_run
use opendal_core::Operator;
-use opendal_core::services::Lakefs;
-use anyhow::Result;
+use opendal_core::Result;
+use opendal_service_lakefs::Lakefs;
#[tokio::main]
async fn main() -> Result<()> {
diff --git a/core/core/src/services/lakefs/error.rs
b/core/services/lakefs/src/error.rs
similarity index 96%
rename from core/core/src/services/lakefs/error.rs
rename to core/services/lakefs/src/error.rs
index 3e64b0eea..41d3a7501 100644
--- a/core/core/src/services/lakefs/error.rs
+++ b/core/services/lakefs/src/error.rs
@@ -19,11 +19,10 @@ use std::fmt::Debug;
use http::Response;
use http::StatusCode;
+use opendal_core::raw::*;
+use opendal_core::*;
use serde::Deserialize;
-use crate::raw::*;
-use crate::*;
-
/// LakefsError is the error returned by Lakefs File System.
#[derive(Default, Deserialize)]
struct LakefsError {
@@ -72,8 +71,6 @@ pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
#[cfg(test)]
mod test {
use super::*;
- use crate::raw::new_json_deserialize_error;
- use crate::types::Result;
#[test]
fn test_parse_error() -> Result<()> {
diff --git a/core/core/src/services/lakefs/mod.rs
b/core/services/lakefs/src/lib.rs
similarity index 84%
rename from core/core/src/services/lakefs/mod.rs
rename to core/services/lakefs/src/lib.rs
index cc55ed939..b9e0076f8 100644
--- a/core/core/src/services/lakefs/mod.rs
+++ b/core/services/lakefs/src/lib.rs
@@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-/// Default scheme for lakefs service.
-pub const LAKEFS_SCHEME: &str = "lakefs";
+//! lakeFS service implementation for Apache OpenDAL.
-use crate::types::DEFAULT_OPERATOR_REGISTRY;
+#![cfg_attr(docsrs, feature(doc_cfg))]
+#![deny(missing_docs)]
mod backend;
mod config;
@@ -31,7 +31,10 @@ mod writer;
pub use backend::LakefsBuilder as Lakefs;
pub use config::LakefsConfig;
+/// Default scheme for lakefs service.
+pub const LAKEFS_SCHEME: &str = "lakefs";
+
#[ctor::ctor]
fn register_lakefs_service() {
- DEFAULT_OPERATOR_REGISTRY.register::<Lakefs>(LAKEFS_SCHEME);
+ opendal_core::DEFAULT_OPERATOR_REGISTRY.register::<Lakefs>(LAKEFS_SCHEME);
}
diff --git a/core/core/src/services/lakefs/lister.rs
b/core/services/lakefs/src/lister.rs
similarity index 98%
rename from core/core/src/services/lakefs/lister.rs
rename to core/services/lakefs/src/lister.rs
index fe49f1529..bac970098 100644
--- a/core/core/src/services/lakefs/lister.rs
+++ b/core/services/lakefs/src/lister.rs
@@ -17,12 +17,13 @@
use std::sync::Arc;
+use bytes::Buf;
+use opendal_core::raw::*;
+use opendal_core::*;
+
use super::core::LakefsCore;
use super::core::LakefsListResponse;
use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-use bytes::Buf;
pub struct LakefsLister {
core: Arc<LakefsCore>,
diff --git a/core/core/src/services/lakefs/writer.rs
b/core/services/lakefs/src/writer.rs
similarity index 98%
rename from core/core/src/services/lakefs/writer.rs
rename to core/services/lakefs/src/writer.rs
index 8699cc07e..8ddaa9411 100644
--- a/core/core/src/services/lakefs/writer.rs
+++ b/core/services/lakefs/src/writer.rs
@@ -19,12 +19,12 @@ use std::sync::Arc;
use bytes::Buf;
use http::StatusCode;
+use opendal_core::raw::*;
+use opendal_core::*;
use super::core::LakefsCore;
use super::core::LakefsStatus;
use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
pub struct LakefsWriter {
core: Arc<LakefsCore>,
diff --git a/core/src/lib.rs b/core/src/lib.rs
index ad19c2a2a..b6dc54044 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -82,6 +82,8 @@ pub mod services {
pub use opendal_service_ipfs::*;
#[cfg(feature = "services-koofr")]
pub use opendal_service_koofr::*;
+ #[cfg(feature = "services-lakefs")]
+ pub use opendal_service_lakefs::*;
#[cfg(feature = "services-memcached")]
pub use opendal_service_memcached::*;
#[cfg(feature = "services-mini-moka")]
