This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 74d92a1b refactor: Polish the behavior of scan (#1926)
74d92a1b is described below
commit 74d92a1b87af1736704ec4a12d209cf33b8e28ae
Author: Xuanwo <git...@xuanwo.io>
AuthorDate: Wed Apr 12 20:34:11 2023 +0800
refactor: Polish the behavior of scan (#1926)
* refactor: Polish the behavior of scan
Signed-off-by: Xuanwo <git...@xuanwo.io>
* FIx test for sled
Signed-off-by: Xuanwo <git...@xuanwo.io>
* Fix sled
Signed-off-by: Xuanwo <git...@xuanwo.io>
* Fix CI
Signed-off-by: Xuanwo <git...@xuanwo.io>
* Make clippy happy
Signed-off-by: Xuanwo <git...@xuanwo.io>
* FIx kv adapter
Signed-off-by: Xuanwo <git...@xuanwo.io>
---------
Signed-off-by: Xuanwo <git...@xuanwo.io>
---
core/src/raw/adapters/kv/backend.rs | 14 ++------------
core/src/raw/oio/to_flat_pager.rs | 12 ++++++++++--
core/src/services/dashmap/backend.rs | 23 +++++++++++++++++++----
core/src/services/memory/backend.rs | 22 ++++++++++++++++++----
core/src/services/sled/backend.rs | 12 +++++++++++-
core/src/types/operator/blocking_operator.rs | 6 +++++-
core/src/types/operator/operator.rs | 9 ++++++++-
core/tests/behavior/list.rs | 18 ++++++++++++++++++
8 files changed, 91 insertions(+), 25 deletions(-)
diff --git a/core/src/raw/adapters/kv/backend.rs
b/core/src/raw/adapters/kv/backend.rs
index b35e3b4e..7e6ea81a 100644
--- a/core/src/raw/adapters/kv/backend.rs
+++ b/core/src/raw/adapters/kv/backend.rs
@@ -209,14 +209,8 @@ pub struct KvPager {
impl KvPager {
fn new(root: &str, inner: Vec<String>) -> Self {
- let root = if root == "/" {
- "".to_string()
- } else {
- root.to_string()
- };
-
Self {
- root,
+ root: root.to_string(),
inner: Some(inner),
}
}
@@ -233,11 +227,7 @@ impl KvPager {
EntryMode::FILE
};
- oio::Entry::new(
- v.strip_prefix(&self.root)
- .expect("key must start with root"),
- Metadata::new(mode),
- )
+ oio::Entry::new(&build_rel_path(&self.root, &v),
Metadata::new(mode))
})
.collect();
diff --git a/core/src/raw/oio/to_flat_pager.rs
b/core/src/raw/oio/to_flat_pager.rs
index 8f20093c..6cbce7b3 100644
--- a/core/src/raw/oio/to_flat_pager.rs
+++ b/core/src/raw/oio/to_flat_pager.rs
@@ -42,6 +42,7 @@ pub fn to_flat_pager<A: Accessor, P>(acc: A, path: &str,
size: usize) -> ToFlatP
ToFlatPager {
acc,
size,
+ root: path.to_string(),
dirs: VecDeque::from([oio::Entry::new(path,
Metadata::new(EntryMode::DIR))]),
pagers: vec![],
res: Vec::with_capacity(size),
@@ -87,6 +88,7 @@ pub fn to_flat_pager<A: Accessor, P>(acc: A, path: &str,
size: usize) -> ToFlatP
pub struct ToFlatPager<A: Accessor, P> {
acc: A,
size: usize,
+ root: String,
dirs: VecDeque<oio::Entry>,
pagers: Vec<(P, oio::Entry, Vec<oio::Entry>)>,
res: Vec<oio::Entry>,
@@ -121,7 +123,10 @@ where
buf = v;
}
None => {
- self.res.push(de);
+ // Only push entry if it's not root dir
+ if de.path() != self.root {
+ self.res.push(de);
+ }
continue;
}
}
@@ -178,7 +183,10 @@ where
buf = v;
}
None => {
- self.res.push(de);
+ // Only push entry if it's not root dir
+ if de.path() != self.root {
+ self.res.push(de);
+ }
continue;
}
}
diff --git a/core/src/services/dashmap/backend.rs
b/core/src/services/dashmap/backend.rs
index c2847f1d..74e8ab58 100644
--- a/core/src/services/dashmap/backend.rs
+++ b/core/src/services/dashmap/backend.rs
@@ -38,20 +38,35 @@ use crate::*;
/// - [ ] ~~presign~~
/// - [x] blocking
#[derive(Default)]
-pub struct DashmapBuilder {}
+pub struct DashmapBuilder {
+ root: Option<String>,
+}
+
+impl DashmapBuilder {
+ /// Set the root for dashmap.
+ pub fn root(&mut self, path: &str) -> &mut Self {
+ self.root = Some(path.into());
+ self
+ }
+}
impl Builder for DashmapBuilder {
const SCHEME: Scheme = Scheme::Dashmap;
type Accessor = DashmapBackend;
- fn from_map(_: HashMap<String, String>) -> Self {
- Self::default()
+ fn from_map(map: HashMap<String, String>) -> Self {
+ let mut builder = Self::default();
+
+ map.get("root").map(|v| builder.root(v));
+
+ builder
}
fn build(&mut self) -> Result<Self::Accessor> {
Ok(DashmapBackend::new(Adapter {
inner: DashMap::default(),
- }))
+ })
+ .with_root(self.root.as_deref().unwrap_or_default()))
}
}
diff --git a/core/src/services/memory/backend.rs
b/core/src/services/memory/backend.rs
index 838917be..66589c6b 100644
--- a/core/src/services/memory/backend.rs
+++ b/core/src/services/memory/backend.rs
@@ -39,14 +39,28 @@ use crate::*;
/// - [ ] ~~presign~~
/// - [x] blocking
#[derive(Default)]
-pub struct MemoryBuilder {}
+pub struct MemoryBuilder {
+ root: Option<String>,
+}
+
+impl MemoryBuilder {
+ /// Set the root for dashmap.
+ pub fn root(&mut self, path: &str) -> &mut Self {
+ self.root = Some(path.into());
+ self
+ }
+}
impl Builder for MemoryBuilder {
const SCHEME: Scheme = Scheme::Memory;
type Accessor = MemoryBackend;
- fn from_map(_: HashMap<String, String>) -> Self {
- Self::default()
+ fn from_map(map: HashMap<String, String>) -> Self {
+ let mut builder = Self::default();
+
+ map.get("root").map(|v| builder.root(v));
+
+ builder
}
fn build(&mut self) -> Result<Self::Accessor> {
@@ -54,7 +68,7 @@ impl Builder for MemoryBuilder {
inner: Arc::new(Mutex::new(BTreeMap::default())),
};
- Ok(MemoryBackend::new(adapter))
+
Ok(MemoryBackend::new(adapter).with_root(self.root.as_deref().unwrap_or_default()))
}
}
diff --git a/core/src/services/sled/backend.rs
b/core/src/services/sled/backend.rs
index aea5a738..cbd43885 100644
--- a/core/src/services/sled/backend.rs
+++ b/core/src/services/sled/backend.rs
@@ -70,6 +70,7 @@ use crate::*;
pub struct SledBuilder {
/// That path to the sled data directory.
datadir: Option<String>,
+ root: Option<String>,
}
impl SledBuilder {
@@ -78,6 +79,12 @@ impl SledBuilder {
self.datadir = Some(path.into());
self
}
+
+ /// Set the root for sled.
+ pub fn root(&mut self, path: &str) -> &mut Self {
+ self.root = Some(path.into());
+ self
+ }
}
impl Builder for SledBuilder {
@@ -88,6 +95,7 @@ impl Builder for SledBuilder {
let mut builder = SledBuilder::default();
map.get("datadir").map(|v| builder.datadir(v));
+ map.get("root").map(|v| builder.root(v));
builder
}
@@ -108,7 +116,8 @@ impl Builder for SledBuilder {
Ok(SledBackend::new(Adapter {
datadir: datadir_path,
db,
- }))
+ })
+ .with_root(self.root.as_deref().unwrap_or_default()))
}
}
@@ -174,6 +183,7 @@ impl kv::Adapter for Adapter {
for i in it {
let bs = i.map_err(parse_error)?.to_vec();
+
res.push(String::from_utf8(bs).map_err(|err| {
Error::new(ErrorKind::Unexpected, "store key is not valid
utf-8 string")
.set_source(err)
diff --git a/core/src/types/operator/blocking_operator.rs
b/core/src/types/operator/blocking_operator.rs
index fe52d3b6..a2503ed6 100644
--- a/core/src/types/operator/blocking_operator.rs
+++ b/core/src/types/operator/blocking_operator.rs
@@ -718,10 +718,14 @@ impl BlockingOperator {
/// List dir in flat way.
///
- /// This function will create a new handle to list entries.
+ /// Also, this function can be used to list a prefix.
///
/// An error will be returned if given path doesn't end with `/`.
///
+ /// # Notes
+ ///
+ /// - `scan` will not return the prefix itself.
+ ///
/// # Examples
///
/// ```no_run
diff --git a/core/src/types/operator/operator.rs
b/core/src/types/operator/operator.rs
index b33b0638..a85b963e 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -944,6 +944,9 @@ impl Operator {
.await?;
}
+ // Remove the directory itself.
+ self.delete(path).await?;
+
Ok(())
}
@@ -1000,10 +1003,14 @@ impl Operator {
/// List dir in flat way.
///
- /// This function will create a new handle to list entries.
+ /// Also, this function can be used to list a prefix.
///
/// An error will be returned if given path doesn't end with `/`.
///
+ /// # Notes
+ ///
+ /// - `scan` will not return the prefix itself.
+ ///
/// # Examples
///
/// ```no_run
diff --git a/core/tests/behavior/list.rs b/core/tests/behavior/list.rs
index 85e023f8..3c8d5347 100644
--- a/core/tests/behavior/list.rs
+++ b/core/tests/behavior/list.rs
@@ -82,6 +82,7 @@ macro_rules! behavior_list_tests {
test_list_nested_dir,
test_list_dir_with_file_path,
test_scan,
+ test_scan_root,
test_remove_all,
);
)*
@@ -279,6 +280,23 @@ pub async fn test_list_dir_with_file_path(op: Operator) ->
Result<()> {
Ok(())
}
+pub async fn test_scan_root(op: Operator) -> Result<()> {
+ let w = op.scan("").await?;
+ let actual = w
+ .try_collect::<Vec<_>>()
+ .await?
+ .into_iter()
+ .map(|v| v.path().to_string())
+ .collect::<HashSet<_>>();
+
+ assert!(
+ actual.is_empty(),
+ "empty root should return empty, but got {:?}",
+ actual
+ );
+ Ok(())
+}
+
// Walk top down should output as expected
pub async fn test_scan(op: Operator) -> Result<()> {
let expected = vec