corwinjoy commented on code in PR #7019:
URL: https://github.com/apache/arrow-rs/pull/7019#discussion_r1945671511
##########
object_store/src/local.rs:
##########
@@ -678,6 +622,93 @@ impl ObjectStore for LocalFileSystem {
}
}
+impl LocalFileSystem {
+ fn list_with_maybe_offset(
+ &self,
+ prefix: Option<&Path>,
+ maybe_offset: Option<&Path>,
+ ) -> BoxStream<'static, Result<ObjectMeta>> {
+ let config = Arc::clone(&self.config);
+
+ let root_path = match prefix {
+ Some(prefix) => match config.prefix_to_filesystem(prefix) {
+ Ok(path) => path,
+ Err(e) => return
futures::future::ready(Err(e)).into_stream().boxed(),
+ },
+ None => config.root.to_file_path().unwrap(),
+ };
+
+ let walkdir = WalkDir::new(root_path)
+ // Don't include the root directory itself
+ .min_depth(1)
+ .follow_links(true);
+
+ let maybe_offset = maybe_offset.cloned();
+
+ let s = walkdir.into_iter().flat_map(move |result_dir_entry| {
+ // Apply offset filter before proceeding, to reduce statx file
system calls
+ // This matters for NFS mounts
+ if let (Some(offset), Ok(entry)) = (maybe_offset.as_ref(),
result_dir_entry.as_ref()) {
+ let location = config.filesystem_to_path(entry.path());
+ match location {
+ Ok(path) if path <= *offset => return None,
+ Err(e) => return Some(Err(e)),
+ _ => {}
+ }
+ }
+
+ let entry = match
convert_walkdir_result(result_dir_entry).transpose()? {
Review Comment:
Yes, and we try to prefilter the list so that we can avoid these calls.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
