preprocessor - cache file tokenization for common includes (#901)

Cargo.toml

@@ -43,6 +43,7 @@ byteorder = "1.5.0"
 chumsky = "0.9.3"
 clap = "4.5.30"
 codespan-reporting = { version = "0.11.1", features = ["serialization"] }
+dashmap = "6.1.0"
 dirs = "6.0.0"
 git2 = "0.20.0"
 indexmap = "2.7.1"

hls/Cargo.toml

@@ -6,14 +6,14 @@ edition = "2024"
 [dependencies]
 hemtt-common = { path = "../libs/common" }
 hemtt-config = { path = "../libs/config" }
-hemtt-preprocessor = { path = "../libs/preprocessor" }
+hemtt-preprocessor = { path = "../libs/preprocessor", features = ["lsp"] }
 hemtt-sqf = { path = "../libs/sqf" }
 hemtt-workspace = { path = "../libs/workspace" }
 
 arma3-wiki = { workspace = true }
 chumsky = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
-dashmap = "6.1.0"
+dashmap = { workspace = true }
 regex = { workspace = true }
 ropey = "1.6.1"
 serde = { workspace = true }

libs/config/Cargo.toml

@@ -13,8 +13,8 @@ bench = false
 
 [dependencies]
 hemtt-common = { path = "../common", version = "1.0.0" }
-hemtt-preprocessor = { path = "../preprocessor", version = "1.0.0", features = ["lsp"] }
-hemtt-workspace = { path = "../workspace", version = "1.0.0", features = ["lsp"] }
+hemtt-preprocessor = { path = "../preprocessor", version = "1.0.0" }
+hemtt-workspace = { path = "../workspace", version = "1.0.0" }
 
 automod = { workspace = true }
 byteorder = { workspace = true }

libs/preprocessor/Cargo.toml

@@ -16,6 +16,7 @@ hemtt-common = { path = "../common", version = "1.0.0" }
 hemtt-workspace = { path = "../workspace", version = "1.0.0" }
 
 automod = { workspace = true }
+dashmap = { workspace = true }
 lsp-types = { workspace = true, optional = true }
 peekmore = { workspace = true }
 pest = { workspace = true }

libs/preprocessor/src/parse/mod.rs

@@ -1,5 +1,6 @@
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 
+use dashmap::DashMap;
 use hemtt_workspace::{
     WorkspacePath,
     position::{LineCol, Position},
@@ -16,6 +17,20 @@ use crate::{Error, codes::pe24_parsing_failed::ParsingFailed};
 /// Parser for the preprocessor, generated from `config.pest`
 pub struct PreprocessorParser;
 
+#[derive(Debug, Clone)]
+struct Cache {
+    tokens: Arc<DashMap<WorkspacePath, Vec<Arc<Token>>>>,
+}
+
+impl Cache {
+    pub fn get() -> Self {
+        static SINGLETON: LazyLock<Cache> = LazyLock::new(|| Cache {
+            tokens: Arc::new(DashMap::new()),
+        });
+        (*SINGLETON).clone()
+    }
+}
+
 /// Parse a file into tokens
 ///
 /// # Errors
@@ -24,14 +39,31 @@ pub struct PreprocessorParser;
 /// # Panics
 /// If the file is invalid
 pub fn file(path: &WorkspacePath) -> Result<Vec<Arc<Token>>, Error> {
+    let cache = Cache::get();
+    if cache.tokens.contains_key(path) {
+        return Ok(cache.tokens.get(path).expect("exists").value().clone());
+    }
     let source = path.read_to_string()?;
-    str(&source, path)
+    let res = str(&source, path)?;
+    // The LSP manages its own caches, having this enabled would cause the LSP to never see any changes
+    // This will make the LSP slower, in the future it could have a way to invalidate the cache (when a file is saved)
+    #[cfg(not(feature = "lsp"))]
+    {
+        let path_str = path.as_str();
+        if ["macros", "common", "script", "component"]
+            .iter()
+            .any(|&x| path_str.contains(x))
+        {
+            cache.tokens.insert(path.clone(), res.clone());
+        }
+    }
+    Ok(res)
 }
 
 /// Parse a string into tokens
 ///
 /// # Errors
-/// If the string is invalid
+/// If the string is invalid1
 ///
 /// # Panics
 /// If the string is invalid