make it fast (#25)

Author: bend-n

.gitignore

@@ -15,3 +15,6 @@ Cargo.lock
 
 # Addons folder
 addons/
+
+# Performance data
+perf.data*

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "godot-package-manager"
-version = "1.1.2"
+version = "1.2.0"
 edition = "2021"
 authors = ["bendn <[email protected]>"]
 description = "A package manager for godot"
@@ -15,7 +15,6 @@ deser-hjson = "1.0.2"
 flate2 = "1.0.25"
 lazy_static = "1.4.0"
 regex = "1.7.0"
-ureq = "2.5.0"
 serde = { version = "1.0.150", features = ["derive"] }
 serde_json = "1.0.89"
 serde_yaml = "0.9.14"
@@ -26,9 +25,16 @@ console = "0.15.4"
 indicatif = "0.17.2"
 anyhow = "1.0.68"
 dialoguer = { version = "0.10.3", features = [] }
-rayon = "1.6"
+reqwest = { version = "0.11", features = [] }
+tokio = { version = "1", features = ["full"] }
+async-recursion = "1.0.2"
+futures = "0.3"
 
 [dev-dependencies]
 glob = "0.3.0"
 sha2 = "0.10.6"
 tempdir = "0.3.7"
+
+[profile.release]
+lto = true
+strip = true

src/config_file.rs

@@ -1,6 +1,7 @@
 use crate::package::Package;
 use anyhow::Result;
 use console::style;
+use futures::stream::{self, StreamExt};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
@@ -17,7 +18,7 @@ pub struct ConfigFile {
 /// A wrapper to [ConfigFile]. This _is_ necessary.
 /// Any alternatives will end up being more ugly than this. (trust me i tried)
 /// There is no way to automatically deserialize the map into a vec.
-struct ConfigWrapper {
+struct ParsedConfig {
     // support NPM package.json files (also allows gpm -c package.json -u)
     #[serde(alias = "dependencies")]
     packages: HashMap<String, String>,
@@ -36,33 +37,40 @@ impl std::fmt::Display for ConfigType {
     }
 }
 
-impl From<ConfigWrapper> for ConfigFile {
-    fn from(from: ConfigWrapper) -> Self {
+impl From<ConfigFile> for ParsedConfig {
+    fn from(from: ConfigFile) -> Self {
         Self {
             packages: from
                 .packages
                 .into_iter()
-                .map(|(name, version)| Package::new(name, version).unwrap())
+                .map(|p| (p.name, p.version))
                 .collect(),
         }
     }
 }
 
-impl From<ConfigFile> for ConfigWrapper {
-    fn from(from: ConfigFile) -> Self {
-        Self {
-            packages: from
-                .packages
-                .into_iter()
-                .map(|p| (p.name, p.version))
-                .collect(),
-        }
+impl ParsedConfig {
+    pub fn parse(txt: &str, t: ConfigType) -> Result<Self> {
+        Ok(match t {
+            ConfigType::TOML => toml::from_str::<ParsedConfig>(txt)?,
+            ConfigType::JSON => deser_hjson::from_str::<ParsedConfig>(txt)?,
+            ConfigType::YAML => serde_yaml::from_str::<ParsedConfig>(txt)?,
+        })
+    }
+
+    pub async fn into_configfile(self) -> ConfigFile {
+        let packages = stream::iter(self.packages.into_iter())
+            .map(|(name, version)| async { Package::new(name, version).await.unwrap() })
+            .buffer_unordered(crate::PARALLEL)
+            .collect::<Vec<Package>>()
+            .await;
+        ConfigFile { packages }
     }
 }
 
 impl ConfigFile {
     pub fn print(self, t: ConfigType) -> String {
-        let w = ConfigWrapper::from(self);
+        let w = ParsedConfig::from(self);
         match t {
             ConfigType::JSON => serde_json::to_string_pretty(&w).unwrap(),
             ConfigType::YAML => serde_yaml::to_string(&w).unwrap(),
@@ -72,20 +80,24 @@ impl ConfigFile {
 
     /// Creates a new [ConfigFile] from the given text
     /// Panics if the file cant be parsed as toml, hjson or yaml.
-    pub fn new(contents: &String) -> Self {
-        if contents.len() == 0 {
+    pub async fn new(contents: &String) -> Self {
+        if contents.is_empty() {
             panic!("Empty CFG");
         }
 
         // definetly not going to backfire
         let mut cfg = if contents.as_bytes()[0] == b'{' {
             // json gets brute forced first so this isnt really needed
-            Self::parse(contents, ConfigType::JSON).expect("Parsing CFG from JSON should work")
+            Self::parse(contents, ConfigType::JSON)
+                .await
+                .expect("Parsing CFG from JSON should work")
         } else if contents.len() > 3 && contents[..3] == *"---" {
-            Self::parse(contents, ConfigType::YAML).expect("Parsing CFG from YAML should work")
+            Self::parse(contents, ConfigType::YAML)
+                .await
+                .expect("Parsing CFG from YAML should work")
         } else {
             for i in [ConfigType::JSON, ConfigType::YAML, ConfigType::TOML].into_iter() {
-                let res = Self::parse(contents, i.clone());
+                let res = Self::parse(contents, i).await;
 
                 // im sure theres some kind of idiomatic rust way to do this that i dont know of
                 if res.is_ok() {
@@ -106,30 +118,19 @@ impl ConfigFile {
         cfg
     }
 
-    pub fn parse(txt: &String, t: ConfigType) -> Result<Self> {
-        type W = ConfigWrapper;
-        Ok(match t {
-            ConfigType::TOML => toml::from_str::<W>(txt)?.into(),
-            ConfigType::JSON => deser_hjson::from_str::<W>(txt)?.into(),
-            ConfigType::YAML => serde_yaml::from_str::<W>(txt)?.into(),
-        })
+    pub async fn parse(txt: &str, t: ConfigType) -> Result<Self> {
+        Ok(ParsedConfig::parse(txt, t)?.into_configfile().await)
     }
 
     /// Creates a lockfile for this config file.
     /// note: Lockfiles are currently unused.
-    pub fn lock(&mut self) -> String {
-        let mut pkgs = vec![] as Vec<Package>;
-        self.collect()
-            .into_iter()
-            .filter(|p| p.is_installed())
-            .for_each(|mut p| {
-                if p.integrity.is_empty() {
-                    p.integrity = p
-                        .get_integrity()
-                        .expect("Should be able to get package integrity");
-                }
-                pkgs.push(p);
-            });
+    pub async fn lock(&mut self) -> String {
+        let mut pkgs = vec![];
+        for mut p in self.collect() {
+            if p.is_installed() && p.get_manifest().await.is_ok() {
+                pkgs.push(p)
+            };
+        }
         serde_json::to_string_pretty(&pkgs).unwrap()
     }
 
@@ -164,13 +165,13 @@ impl ConfigFile {
 mod tests {
     use crate::config_file::*;
 
-    #[test]
-    fn parse() {
+    #[tokio::test]
+    async fn parse() {
         let _t = crate::test_utils::mktemp();
         let cfgs: [&mut ConfigFile; 3] = [
-            &mut ConfigFile::new(&r#"dependencies: { "@bendn/test": 2.0.10 }"#.into()), // quoteless fails as a result of https://github.com/Canop/deser-hjson/issues/9
-            &mut ConfigFile::new(&"dependencies:\n  \"@bendn/test\": 2.0.10".into()),
-            &mut ConfigFile::new(&"[dependencies]\n\"@bendn/test\" = \"2.0.10\"".into()),
+            &mut ConfigFile::new(&r#"dependencies: { "@bendn/test": 2.0.10 }"#.into()).await, // quoteless fails as a result of https://github.com/Canop/deser-hjson/issues/9
+            &mut ConfigFile::new(&"dependencies:\n  \"@bendn/test\": 2.0.10".into()).await,
+            &mut ConfigFile::new(&"[dependencies]\n\"@bendn/test\" = \"2.0.10\"".into()).await,
         ];
         #[derive(Debug, Deserialize, Clone, Eq, PartialEq)]
         struct LockFileEntry {
@@ -189,9 +190,11 @@ mod tests {
                 cfg.packages[0].dependencies[0].to_string(),
                 "@bendn/[email protected]"
             );
-            cfg.for_each(|p| p.download());
+            for mut p in cfg.collect() {
+                p.download().await
+            }
             assert_eq!(
-                serde_json::from_str::<Vec<LockFileEntry>>(cfg.lock().as_str()).unwrap(),
+                serde_json::from_str::<Vec<LockFileEntry>>(cfg.lock().await.as_str()).unwrap(),
                 wanted_lockfile
             );
         }