Actually download gems from a server

Author: adamchalmers

Cargo.lock

@@ -1,5 +1,5 @@
-mod datatypes;
-pub mod parser;
+pub mod datatypes;
+mod parser;
 #[cfg(test)]
 mod tests;
 

crates/rv-lockfile/src/lib.rs

@@ -4,7 +4,7 @@ PATH
     pathgem (0.1.0)
 
 GEM
-  remote: https://example.com
+  remote: https://gem.coop
   specs:
     actioncable (7.2.2.1)
 

crates/rv-lockfile/tests/inputs/Gemfile.lock.withpath

@@ -52,6 +52,8 @@ once_cell = { workspace = true }
 rayon-tracing = { workspace = true }
 clap_complete = "4.5.57"
 clap_complete_nushell = "4.5.8"
+url = "2.5.7"
+bytes = "1.10.1"
 
 [dev-dependencies]
 insta = { workspace = true }

crates/rv/Cargo.toml

@@ -1,18 +1,96 @@
+use bytes::Bytes;
+use camino::Utf8PathBuf;
+use futures_util::StreamExt;
+use futures_util::TryStreamExt;
+use reqwest::Client;
+use rv_lockfile::datatypes::GemfileDotLock;
+use url::Url;
+
 use crate::config::Config;
 use std::io;
 
+const CONCURRENT_REQUESTS_PER_SOURCE: usize = 10;
+
 #[derive(Debug, thiserror::Error, miette::Diagnostic)]
 pub enum Error {
     #[error(transparent)]
     Parse(#[from] rv_lockfile::ParseErrors),
     #[error(transparent)]
     Io(#[from] io::Error),
+    #[error(transparent)]
+    Reqwest(#[from] reqwest::Error),
+    #[error("Invalid remote URL")]
+    BadRemote {
+        remote: String,
+        err: url::ParseError,
+    },
+    #[error(transparent)]
+    UrlError(#[from] url::ParseError),
 }
 
 type Result<T> = std::result::Result<T, Error>;
 
-pub async fn ci(_config: &Config, gemfile: camino::Utf8PathBuf) -> Result<()> {
+pub async fn ci(_config: &Config, gemfile: Utf8PathBuf) -> Result<()> {
+    ci_inner(gemfile).await
+}
+
+async fn ci_inner(gemfile: Utf8PathBuf) -> Result<()> {
     let gemfile_contents = std::fs::read_to_string(gemfile)?;
     let lockfile = rv_lockfile::parse(&gemfile_contents)?;
+    download_gems(lockfile).await?;
+    Ok(())
+}
+
+async fn download_gems<'i>(lockfile: GemfileDotLock<'i>) -> Result<()> {
+    let client = reqwest::Client::builder()
+        .user_agent(format!("rv-{}", env!("CARGO_PKG_VERSION")))
+        .build()?;
+    for gem_source in lockfile.gem {
+        // Get all URLs for downloading all gems from this source.
+        let urls = gem_source
+            .specs
+            .iter()
+            .map(|gem| {
+                let remote = gem_source.remote;
+                let gem_name = gem.gem_version.name;
+                let gem_version = gem.gem_version.version;
+                let path = format!("gems/{gem_name}-{gem_version}.gem");
+                let url = url::Url::parse(remote)
+                    .map_err(|err| Error::BadRemote {
+                        remote: remote.to_owned(),
+                        err,
+                    })?
+                    .join(&path)?;
+                Ok(url)
+            })
+            .collect::<Result<Vec<Url>>>()?;
+        let url_stream = futures_util::stream::iter(urls);
+        let _downloaded_gems: Vec<(Bytes, Url)> = url_stream
+            .map(|url| download_gem(url, &client))
+            .buffered(CONCURRENT_REQUESTS_PER_SOURCE)
+            .try_collect()
+            .await?;
+        // TODO: Extract the gems and put them somewhere on the filesystem
+    }
     Ok(())
 }
+
+async fn download_gem(url: Url, client: &Client) -> Result<(Bytes, Url)> {
+    eprintln!("Downloading from {url}");
+    // TODO: Validate the checksum from the Lockfile if present.
+    let contents = client.get(url.clone()).send().await?.bytes().await?;
+    Ok((contents, url))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_download_gems() -> Result<()> {
+        eprintln!("{:?}", std::env::current_dir());
+        let file = "../rv-lockfile/tests/inputs/Gemfile.lock.test0".into();
+        ci_inner(file).await?;
+        Ok(())
+    }
+}