feat!: add support for explicit non-parallel iteration.

That way we can allow the implementation to choose whether they
need greatest speed at some cost or not.

This also allows us to create a new thread-pool on each iteration
as those who expect high cost or many files will likely chose to do
that instead of single-threaded iteration, which nicely contains the
threads needed and avoids keeping them alive as part of some global pool.

Author: Byron

Cargo.lock

@@ -95,7 +95,7 @@ check: ## Build all code in suitable configurations
 			   && cargo check
 	cd git-features && cargo check --all-features \
 			   && cargo check --features parallel \
-			   && cargo check --features parallel,fs-walkdir-parallel \
+			   && cargo check --features fs-walkdir-parallel \
 			   && cargo check --features rustsha1 \
 			   && cargo check --features fast-sha1 \
 			   && cargo check --features progress \

Makefile

@@ -19,8 +19,7 @@ default = []
 progress = ["prodash"]
 
 ## If set, walkdir iterators will be multi-threaded.
-## This feature has [certain side-effects](https://github.com/starship/starship/issues/4251) of rayon threadpool configuration with `jwalk`.
-fs-walkdir-parallel = ["parallel", "jwalk" ]
+fs-walkdir-parallel = [ "num_cpus", "jwalk" ]
 
 ## Use scoped threads and channels to parallelize common workloads on multiple objects. If enabled, it is used everywhere
 ## where it makes sense.
@@ -107,7 +106,7 @@ crossbeam-channel = { version = "0.5.0", optional = true }
 num_cpus = { version = "1.13.0", optional = true }
 parking_lot = { version = "0.12.0", default-features = false, optional = true }
 
-jwalk = { version = "0.6.0", optional = true }
+jwalk = { version = "0.8.1", optional = true }
 ## Makes facilities of the `walkdir` crate partially available.
 ## In conjunction with the **parallel** feature, directory walking will be parallel instead behind a compatible interface.
 walkdir = { version = "2.3.2", optional = true } # used when parallel is off

git-features/Cargo.toml

@@ -6,29 +6,62 @@
 //! For information on how to use the [`WalkDir`] type, have a look at
 //! * [`jwalk::WalkDir`](https://docs.rs/jwalk/0.5.1/jwalk/type.WalkDir.html) if `parallel` feature is enabled
 //! * [walkdir::WalkDir](https://docs.rs/walkdir/2.3.1/walkdir/struct.WalkDir.html) otherwise
-#[cfg(feature = "fs-walkdir-parallel")]
+
+#[cfg(any(feature = "walkdir", feature = "fs-walkdir-parallel"))]
+mod shared {
+    /// The desired level of parallelism.
+    pub enum Parallelism {
+        /// Do not parallelize at all by making a serial traversal on the current thread.
+        Serial,
+        /// Create a new thread pool for each traversal with up to 16 threads or the amount of logical cores of the machine.
+        ThreadPoolPerTraversal {
+            /// The base name of the threads we create as part of the thread-pool.
+            thread_name: &'static str,
+        },
+    }
+}
+
 ///
+#[cfg(feature = "fs-walkdir-parallel")]
 pub mod walkdir {
-    use std::path::Path;
-
+    pub use super::shared::Parallelism;
     pub use jwalk::{DirEntry as DirEntryGeneric, DirEntryIter as DirEntryIterGeneric, Error, WalkDir};
+    use std::path::Path;
 
     /// An alias for an uncustomized directory entry to match the one of the non-parallel version offered by `walkdir`.
     pub type DirEntry = DirEntryGeneric<((), ())>;
 
-    /// Instantiate a new directory iterator which will not skip hidden files.
-    pub fn walkdir_new(root: impl AsRef<Path>) -> WalkDir {
-        WalkDir::new(root)
-            .skip_hidden(false)
-            .parallelism(jwalk::Parallelism::Serial)
+    impl From<Parallelism> for jwalk::Parallelism {
+        fn from(v: Parallelism) -> Self {
+            match v {
+                Parallelism::Serial => jwalk::Parallelism::Serial,
+                Parallelism::ThreadPoolPerTraversal { thread_name } => {
+                    let pool = jwalk::rayon::ThreadPoolBuilder::new()
+                        .num_threads(num_cpus::get().min(16))
+                        .stack_size(128 * 1024)
+                        .thread_name(move |idx| format!("{thread_name} {idx}"))
+                        .build()
+                        .expect("we only set options that can't cause a build failure");
+                    jwalk::Parallelism::RayonExistingPool {
+                        pool: pool.into(),
+                        busy_timeout: None,
+                    }
+                }
+            }
+        }
+    }
+
+    /// Instantiate a new directory iterator which will not skip hidden files, with the given level of `parallelism`.
+    pub fn walkdir_new(root: impl AsRef<Path>, parallelism: Parallelism) -> WalkDir {
+        WalkDir::new(root).skip_hidden(false).parallelism(parallelism.into())
     }
 
     /// Instantiate a new directory iterator which will not skip hidden files and is sorted
-    pub fn walkdir_sorted_new(root: impl AsRef<Path>) -> WalkDir {
+    pub fn walkdir_sorted_new(root: impl AsRef<Path>, parallelism: Parallelism) -> WalkDir {
         WalkDir::new(root)
             .skip_hidden(false)
             .sort(true)
-            .parallelism(jwalk::Parallelism::Serial)
+            .parallelism(parallelism.into())
     }
 
     /// The Iterator yielding directory items
@@ -38,17 +71,18 @@ pub mod walkdir {
 #[cfg(all(feature = "walkdir", not(feature = "fs-walkdir-parallel")))]
 ///
 pub mod walkdir {
+    pub use super::shared::Parallelism;
     use std::path::Path;
 
     pub use walkdir::{DirEntry, Error, WalkDir};
 
-    /// Instantiate a new directory iterator which will not skip hidden files.
-    pub fn walkdir_new(root: impl AsRef<Path>) -> WalkDir {
+    /// Instantiate a new directory iterator which will not skip hidden files, with the given level of `parallelism`.
+    pub fn walkdir_new(root: impl AsRef<Path>, _: Parallelism) -> WalkDir {
         WalkDir::new(root)
     }
 
-    /// Instantiate a new directory iterator which will not skip hidden files and is sorted
-    pub fn walkdir_sorted_new(root: impl AsRef<Path>) -> WalkDir {
+    /// Instantiate a new directory iterator which will not skip hidden files and is sorted, with the given level of `parallelism`.
+    pub fn walkdir_sorted_new(root: impl AsRef<Path>, _: Parallelism) -> WalkDir {
         WalkDir::new(root).sort_by_file_name()
     }