D8251: rust-status: traverse working directory in parallel
Closed by commit rHGfe7d2cf0b429: rust-status: traverse working directory in
parallel (authored by Alphare).
This revision was automatically updated to reflect the committed changes.
This revision was not accepted when it landed; it landed in state "Needs
Review".
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D8251?vs=20708=20732
CHANGES SINCE LAST ACTION
https://phab.mercurial-scm.org/D8251/new/
REVISION DETAIL
https://phab.mercurial-scm.org/D8251
AFFECTED FILES
rust/Cargo.lock
rust/hg-core/Cargo.toml
rust/hg-core/src/dirstate/status.rs
CHANGE DETAILS
diff --git a/rust/hg-core/src/dirstate/status.rs
b/rust/hg-core/src/dirstate/status.rs
--- a/rust/hg-core/src/dirstate/status.rs
+++ b/rust/hg-core/src/dirstate/status.rs
@@ -300,61 +300,55 @@
}
/// Dispatch a single entry (file, folder, symlink...) found during `traverse`.
-/// If the entry is a folder that needs to be traversed, it will be pushed into
-/// `work`.
+/// If the entry is a folder that needs to be traversed, it will be handled
+/// in a separate thread.
+
fn handle_traversed_entry<'a>(
-dir_entry: ,
-matcher: &(impl Matcher + Sync),
-root_dir: impl AsRef,
-dmap: ,
-filename: impl AsRef,
-old_results: , Dispatch>,
-ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
-dir_ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
+scope: ::Scope<'a>,
+files_sender: &'a crossbeam::Sender>,
+matcher: &'a (impl Matcher + Sync),
+root_dir: impl AsRef + Sync + Send + Copy + 'a,
+dmap: &'a DirstateMap,
+old_results: &'a FastHashMap, Dispatch>,
+ignore_fn: &'a (impl for<'r> Fn(&'r HgPath) -> bool + Sync),
+dir_ignore_fn: &'a (impl for<'r> Fn(&'r HgPath) -> bool + Sync),
options: StatusOptions,
-) -> IoResult, Dispatch)>> {
+filename: HgPathBuf,
+dir_entry: DirEntry,
+) -> IoResult<()> {
let file_type = dir_entry.file_type()?;
-let filename = filename.as_ref();
-let entry_option = dmap.get(filename);
+let entry_option = dmap.get();
if file_type.is_dir() {
-// Do we need to traverse it?
-if !ignore_fn() || options.list_ignored {
-return traverse_dir(
-matcher,
-root_dir,
-dmap,
-filename.to_owned(),
-_results,
-ignore_fn,
-dir_ignore_fn,
-options,
-);
-}
-// Nested `if` until `rust-lang/rust#53668` is stable
-if let Some(entry) = entry_option {
-// Used to be a file, is now a folder
-if matcher.matches_everything() || matcher.matches() {
-return Ok(vec![(
-Cow::Owned(filename.to_owned()),
-dispatch_missing(entry.state),
-)]);
-}
-}
+handle_traversed_dir(
+scope,
+files_sender,
+matcher,
+root_dir,
+dmap,
+old_results,
+ignore_fn,
+dir_ignore_fn,
+options,
+entry_option,
+filename,
+);
} else if file_type.is_file() || file_type.is_symlink() {
if let Some(entry) = entry_option {
if matcher.matches_everything() || matcher.matches() {
let metadata = dir_entry.metadata()?;
-return Ok(vec![(
-Cow::Owned(filename.to_owned()),
-dispatch_found(
-,
-*entry,
-HgMetadata::from_metadata(metadata),
-_map,
-options,
-),
-)]);
+files_sender
+.send(Ok((
+filename.to_owned(),
+dispatch_found(
+,
+*entry,
+HgMetadata::from_metadata(metadata),
+_map,
+options,
+),
+)))
+.unwrap();
}
} else if (matcher.matches_everything() || matcher.matches())
&& !ignore_fn()
@@ -363,53 +357,96 @@
&& dir_ignore_fn()
{
if options.list_ignored {
-return Ok(vec![(
-Cow::Owned(filename.to_owned()),
-Dispatch::Ignored,
-)]);
+files_sender
+.send(Ok((filename.to_owned(), Dispatch::Ignored)))
+.unwrap();
}
} else {
-return Ok(vec![(
-Cow::Owned(filename.to_owned()),
-Dispatch::Unknown,
-)]);
+
D8251: rust-status: traverse working directory in parallel
Alphare updated this revision to Diff 20708.
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D8251?vs=20565=20708
BRANCH
default
CHANGES SINCE LAST ACTION
https://phab.mercurial-scm.org/D8251/new/
REVISION DETAIL
https://phab.mercurial-scm.org/D8251
AFFECTED FILES
rust/Cargo.lock
rust/hg-core/Cargo.toml
rust/hg-core/src/dirstate/status.rs
CHANGE DETAILS
diff --git a/rust/hg-core/src/dirstate/status.rs
b/rust/hg-core/src/dirstate/status.rs
--- a/rust/hg-core/src/dirstate/status.rs
+++ b/rust/hg-core/src/dirstate/status.rs
@@ -300,61 +300,55 @@
}
/// Dispatch a single entry (file, folder, symlink...) found during `traverse`.
-/// If the entry is a folder that needs to be traversed, it will be pushed into
-/// `work`.
+/// If the entry is a folder that needs to be traversed, it will be handled
+/// in a separate thread.
+
fn handle_traversed_entry<'a>(
-dir_entry: ,
-matcher: &(impl Matcher + Sync),
-root_dir: impl AsRef,
-dmap: ,
-filename: impl AsRef,
-old_results: , Dispatch>,
-ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
-dir_ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
+scope: ::Scope<'a>,
+files_sender: &'a crossbeam::Sender>,
+matcher: &'a (impl Matcher + Sync),
+root_dir: impl AsRef + Sync + Send + Copy + 'a,
+dmap: &'a DirstateMap,
+old_results: &'a FastHashMap, Dispatch>,
+ignore_fn: &'a (impl for<'r> Fn(&'r HgPath) -> bool + Sync),
+dir_ignore_fn: &'a (impl for<'r> Fn(&'r HgPath) -> bool + Sync),
options: StatusOptions,
-) -> IoResult, Dispatch)>> {
+filename: HgPathBuf,
+dir_entry: DirEntry,
+) -> IoResult<()> {
let file_type = dir_entry.file_type()?;
-let filename = filename.as_ref();
-let entry_option = dmap.get(filename);
+let entry_option = dmap.get();
if file_type.is_dir() {
-// Do we need to traverse it?
-if !ignore_fn() || options.list_ignored {
-return traverse_dir(
-matcher,
-root_dir,
-dmap,
-filename.to_owned(),
-_results,
-ignore_fn,
-dir_ignore_fn,
-options,
-);
-}
-// Nested `if` until `rust-lang/rust#53668` is stable
-if let Some(entry) = entry_option {
-// Used to be a file, is now a folder
-if matcher.matches_everything() || matcher.matches() {
-return Ok(vec![(
-Cow::Owned(filename.to_owned()),
-dispatch_missing(entry.state),
-)]);
-}
-}
+handle_traversed_dir(
+scope,
+files_sender,
+matcher,
+root_dir,
+dmap,
+old_results,
+ignore_fn,
+dir_ignore_fn,
+options,
+entry_option,
+filename,
+);
} else if file_type.is_file() || file_type.is_symlink() {
if let Some(entry) = entry_option {
if matcher.matches_everything() || matcher.matches() {
let metadata = dir_entry.metadata()?;
-return Ok(vec![(
-Cow::Owned(filename.to_owned()),
-dispatch_found(
-,
-*entry,
-HgMetadata::from_metadata(metadata),
-_map,
-options,
-),
-)]);
+files_sender
+.send(Ok((
+filename.to_owned(),
+dispatch_found(
+,
+*entry,
+HgMetadata::from_metadata(metadata),
+_map,
+options,
+),
+)))
+.unwrap();
}
} else if (matcher.matches_everything() || matcher.matches())
&& !ignore_fn()
@@ -363,53 +357,96 @@
&& dir_ignore_fn()
{
if options.list_ignored {
-return Ok(vec![(
-Cow::Owned(filename.to_owned()),
-Dispatch::Ignored,
-)]);
+files_sender
+.send(Ok((filename.to_owned(), Dispatch::Ignored)))
+.unwrap();
}
} else {
-return Ok(vec![(
-Cow::Owned(filename.to_owned()),
-Dispatch::Unknown,
-)]);
+files_sender
+.send(Ok((filename.to_owned(), Dispatch::Unknown)))
+.unwrap();
}
} else if ignore_fn() && options.list_ignored {
-return
D8251: rust-status: traverse working directory in parallel
Alphare created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.
REVISION SUMMARY
Using `rayon` for this task ensures that we are using the same work-stealing
threadpool for everything.
This change introduces `crossbeam` as an explicit dependency, although it is
already a dependency of `rayon`. It provides better structures for
multi-threaded tasks than the stdlib.
REPOSITORY
rHG Mercurial
BRANCH
default
REVISION DETAIL
https://phab.mercurial-scm.org/D8251
AFFECTED FILES
rust/Cargo.lock
rust/hg-core/Cargo.toml
rust/hg-core/src/dirstate/status.rs
CHANGE DETAILS
diff --git a/rust/hg-core/src/dirstate/status.rs
b/rust/hg-core/src/dirstate/status.rs
--- a/rust/hg-core/src/dirstate/status.rs
+++ b/rust/hg-core/src/dirstate/status.rs
@@ -300,61 +300,55 @@
}
/// Dispatch a single entry (file, folder, symlink...) found during `traverse`.
-/// If the entry is a folder that needs to be traversed, it will be pushed into
-/// `work`.
+/// If the entry is a folder that needs to be traversed, it will be handled
+/// in a separate thread.
+
fn handle_traversed_entry<'a>(
-dir_entry: ,
-matcher: &(impl Matcher + Sync),
-root_dir: impl AsRef,
-dmap: ,
-filename: impl AsRef,
-old_results: , Dispatch>,
-ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
-dir_ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
+scope: ::Scope<'a>,
+files_sender: &'a crossbeam::Sender>,
+matcher: &'a (impl Matcher + Sync),
+root_dir: impl AsRef + Sync + Send + Copy + 'a,
+dmap: &'a DirstateMap,
+old_results: &'a FastHashMap, Dispatch>,
+ignore_fn: &'a (impl for<'r> Fn(&'r HgPath) -> bool + Sync),
+dir_ignore_fn: &'a (impl for<'r> Fn(&'r HgPath) -> bool + Sync),
options: StatusOptions,
-) -> IoResult, Dispatch)>> {
+filename: HgPathBuf,
+dir_entry: DirEntry,
+) -> IoResult<()> {
let file_type = dir_entry.file_type()?;
-let filename = filename.as_ref();
-let entry_option = dmap.get(filename);
+let entry_option = dmap.get();
if file_type.is_dir() {
-// Do we need to traverse it?
-if !ignore_fn() || options.list_ignored {
-return traverse_dir(
-matcher,
-root_dir,
-dmap,
-filename.to_owned(),
-_results,
-ignore_fn,
-dir_ignore_fn,
-options,
-);
-}
-// Nested `if` until `rust-lang/rust#53668` is stable
-if let Some(entry) = entry_option {
-// Used to be a file, is now a folder
-if matcher.matches_everything() || matcher.matches() {
-return Ok(vec![(
-Cow::Owned(filename.to_owned()),
-dispatch_missing(entry.state),
-)]);
-}
-}
+handle_traversed_dir(
+scope,
+files_sender,
+matcher,
+root_dir,
+dmap,
+old_results,
+ignore_fn,
+dir_ignore_fn,
+options,
+entry_option,
+filename,
+);
} else if file_type.is_file() || file_type.is_symlink() {
if let Some(entry) = entry_option {
if matcher.matches_everything() || matcher.matches() {
let metadata = dir_entry.metadata()?;
-return Ok(vec![(
-Cow::Owned(filename.to_owned()),
-dispatch_found(
-,
-*entry,
-HgMetadata::from_metadata(metadata),
-_map,
-options,
-),
-)]);
+files_sender
+.send(Ok((
+filename.to_owned(),
+dispatch_found(
+,
+*entry,
+HgMetadata::from_metadata(metadata),
+_map,
+options,
+),
+)))
+.unwrap();
}
} else if (matcher.matches_everything() || matcher.matches())
&& !ignore_fn()
@@ -363,53 +357,96 @@
&& dir_ignore_fn()
{
if options.list_ignored {
-return Ok(vec![(
-Cow::Owned(filename.to_owned()),
-Dispatch::Ignored,
-)]);
+files_sender
+.send(Ok((filename.to_owned(), Dispatch::Ignored)))
+.unwrap();
}
} else {
-return Ok(vec