From 3e6ed53e5ceee051116c575711b0bb9cc820c7c5 Mon Sep 17 00:00:00 2001 From: Integral Date: Sat, 23 Nov 2024 20:58:24 +0800 Subject: [PATCH] perf: use rayon to parallelize directory operations --- Cargo.lock | 12 +++++ yazi-fs/src/fns.rs | 66 +++++++++++++++------------ yazi-plugin/src/fs/fs.rs | 2 +- yazi-scheduler/src/file/file.rs | 2 +- yazi-scheduler/src/prework/prework.rs | 2 +- yazi-scheduler/src/scheduler.rs | 2 +- yazi-shared/Cargo.toml | 1 + 7 files changed, 54 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a22c8c928..9206b15a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2107,6 +2107,16 @@ dependencies = [ "rgb", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + [[package]] name = "rayon-core" version = "1.12.1" @@ -3625,6 +3635,8 @@ dependencies = [ "parking_lot", "percent-encoding", "ratatui", + "rayon", + "regex", "serde", "shell-words", "tokio", diff --git a/yazi-fs/src/fns.rs b/yazi-fs/src/fns.rs index 70872ca26..27eea4789 100644 --- a/yazi-fs/src/fns.rs +++ b/yazi-fs/src/fns.rs @@ -1,7 +1,8 @@ -use std::{borrow::Cow, collections::{HashMap, HashSet, VecDeque}, ffi::{OsStr, OsString}, path::{Path, PathBuf}}; +use std::{borrow::Cow, collections::{HashMap, HashSet}, ffi::{OsStr, OsString}, path::{Path, PathBuf}}; use anyhow::{Result, bail}; use tokio::{fs, io, select, sync::{mpsc, oneshot}, time}; +use rayon::prelude::*; use super::Cha; @@ -155,28 +156,28 @@ pub async fn realname_unchecked<'a>( } } -pub async fn calculate_size(path: &Path) -> u64 { - let mut total = 0; - let mut stack = VecDeque::from([path.to_path_buf()]); - while let Some(path) = stack.pop_front() { - let Ok(meta) = fs::symlink_metadata(&path).await else { continue }; - if !meta.is_dir() { - total += meta.len(); - continue; - } +pub async fn calculate_size(path: &Path) -> io::Result { + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || _calculate_size(&path)).await? +} - let Ok(mut it) = fs::read_dir(path).await else { continue }; - while let Ok(Some(entry)) = it.next_entry().await { - let Ok(meta) = entry.metadata().await else { continue }; +fn _calculate_size(path: &Path) -> io::Result { + let entries: Vec<_> = std::fs::read_dir(&path)?.collect(); - if meta.is_dir() { - stack.push_back(entry.path()); - } else { - total += meta.len(); - } + let total = entries.par_iter().filter_map(|entry| { + match entry { + Ok(entry) => { + match entry.metadata() { + Ok(meta) if meta.is_file() => Some(meta.len()), + Ok(meta) if meta.is_dir() => _calculate_size(&entry.path()).ok(), + _ => None, + } + }, + _ => None, } - } - total + }).sum(); + + Ok(total) } pub fn copy_with_progress( @@ -284,18 +285,25 @@ async fn _copy_with_progress(from: PathBuf, to: PathBuf, cha: Cha) -> io::Result } } -pub async fn remove_dir_clean(dir: &Path) { - let Ok(mut it) = fs::read_dir(dir).await else { return }; +pub async fn remove_dir_clean(dir: &Path) -> io::Result<()> { + let dir = dir.to_path_buf(); + tokio::task::spawn_blocking(move || _remove_dir_clean(&dir)).await? +} + +fn _remove_dir_clean(dir: &Path) -> io::Result<()> { + let entries: Vec<_> = std::fs::read_dir(&dir)?.collect(); - while let Ok(Some(entry)) = it.next_entry().await { - if entry.file_type().await.is_ok_and(|t| t.is_dir()) { - let path = entry.path(); - Box::pin(remove_dir_clean(&path)).await; - fs::remove_dir(path).await.ok(); + entries.par_iter().for_each(|entry| { + match entry { + Ok(entry) if entry.file_type().is_ok_and(|t| t.is_dir()) => { + let _ = _remove_dir_clean(&entry.path()); + let _ = std::fs::remove_dir(&entry.path()); + } + _ => (), } - } + }); - fs::remove_dir(dir).await.ok(); + std::fs::remove_dir(&dir) } // Convert a file mode to a string representation diff --git a/yazi-plugin/src/fs/fs.rs b/yazi-plugin/src/fs/fs.rs index daab60d2f..fb7500744 100644 --- a/yazi-plugin/src/fs/fs.rs +++ b/yazi-plugin/src/fs/fs.rs @@ -81,7 +81,7 @@ fn remove(lua: &Lua) -> mlua::Result { b"file" => fs::remove_file(&*url).await, b"dir" => fs::remove_dir(&*url).await, b"dir_all" => fs::remove_dir_all(&*url).await, - b"dir_clean" => Ok(remove_dir_clean(&url).await), + b"dir_clean" => Ok(remove_dir_clean(&url).await?), _ => Err("Removal type must be 'file', 'dir', 'dir_all', or 'dir_clean'".into_lua_err())?, }; diff --git a/yazi-scheduler/src/file/file.rs b/yazi-scheduler/src/file/file.rs index 16b7ed0ff..dd6a9d89b 100644 --- a/yazi-scheduler/src/file/file.rs +++ b/yazi-scheduler/src/file/file.rs @@ -318,7 +318,7 @@ impl File { pub async fn trash(&self, mut task: FileOpTrash) -> Result<()> { let id = task.id; - task.length = calculate_size(&task.target).await; + task.length = calculate_size(&task.target).await?; self.prog.send(TaskProg::New(id, task.length))?; self.queue(FileOp::Trash(task), LOW).await?; diff --git a/yazi-scheduler/src/prework/prework.rs b/yazi-scheduler/src/prework/prework.rs index 084b8944d..c5a28354f 100644 --- a/yazi-scheduler/src/prework/prework.rs +++ b/yazi-scheduler/src/prework/prework.rs @@ -83,7 +83,7 @@ impl Prework { self.prog.send(TaskProg::Adv(task.id, 1, 0))?; } PreworkOp::Size(task) => { - let length = calculate_size(&task.target).await; + let length = calculate_size(&task.target).await?; task.throttle.done((task.target, length), |buf| { { let mut loading = self.size_loading.write(); diff --git a/yazi-scheduler/src/scheduler.rs b/yazi-scheduler/src/scheduler.rs index c20ea0b63..07f9bffb8 100644 --- a/yazi-scheduler/src/scheduler.rs +++ b/yazi-scheduler/src/scheduler.rs @@ -86,7 +86,7 @@ impl Scheduler { Box::new(move |canceled: bool| { async move { if !canceled { - remove_dir_clean(&from).await; + let _ = remove_dir_clean(&from).await; Pump::push_move(from, to); } ongoing.lock().try_remove(id, TaskStage::Hooked); diff --git a/yazi-shared/Cargo.toml b/yazi-shared/Cargo.toml index 2e4281af6..93b75cd81 100644 --- a/yazi-shared/Cargo.toml +++ b/yazi-shared/Cargo.toml @@ -22,6 +22,7 @@ ratatui = { workspace = true } serde = { workspace = true } shell-words = { workspace = true } tokio = { workspace = true } +rayon = "1.10.0" [target."cfg(unix)".dependencies] libc = { workspace = true }