Web crate (#714)

* issue-620 user agent is required

* showing that crate_id is the crate in the params

* Introduce rust files as tested modules

* web crate uses modules

* removed extra file

* broken module checks links with threads with easy interface

* test the cookbook

* scraping complete

Author: AndyGauge

crates/web/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "web"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = "1.0.94"
+mime = "0.3.17"
+regex = "1.11.1"
+reqwest = { version = "0.12.9", features = ["blocking", "json", "stream"] }
+select = "0.6.0"
+serde = { version = "1.0.215", features = ["derive"] }
+tempfile = "3.14.0"
+thiserror = "2.0.5"
+tokio = { version = "1.42.0", features = ["full"] }
+url = "2.5.4"

crates/web/README.md

@@ -0,0 +1,11 @@
+To see the test output run
+
+```bash
+cargo test -- --nocapture
+```
+
+To test the wiki module run
+
+```bash
+cargo test wiki -- --nocapture
+```

crates/web/src/broken.rs

@@ -0,0 +1,76 @@
+use thiserror::Error;
+use reqwest::StatusCode;
+use select::document::Document;
+use select::predicate::Name;
+use std::collections::HashSet;
+use url::{Position, Url};
+
+#[derive(Error, Debug)]
+pub enum BrokenError {
+    #[error("Reqwest error: {0}")]
+    ReqError(#[from] reqwest::Error),
+    #[error("IO error: {0}")]
+    IoError(#[from] std::io::Error),
+    #[error("URL parse error: {0}")]
+    UrlParseError(#[from] url::ParseError),
+    #[error("Join error: {0}")]
+    JoinError(#[from] tokio::task::JoinError),
+}
+
+pub struct CategorizedUrls {
+    pub ok: Vec<String>,
+    pub broken: Vec<String>,
+}
+
+enum Link {
+    GoodLink(Url),
+    BadLink(Url),
+}
+
+async fn get_base_url(url: &Url, doc: &Document) -> Result<Url, BrokenError> {
+  let base_tag_href = doc.find(Name("base")).filter_map(|n| n.attr("href")).nth(0);
+  let base_url =
+    base_tag_href.map_or_else(|| Url::parse(&url[..Position::BeforePath]), Url::parse)?;
+  Ok(base_url)
+}
+
+async fn check_link(url: &Url) -> Result<bool, BrokenError> {
+  let res = reqwest::get(url.as_ref()).await?;
+  Ok(res.status() != StatusCode::NOT_FOUND)
+}
+
+pub async fn check(site: &str) -> Result<CategorizedUrls, BrokenError> {
+  let url = Url::parse(site)?;
+  let res = reqwest::get(url.as_ref()).await?.text().await?;
+  let document = Document::from(res.as_str());
+  let base_url = get_base_url(&url, &document).await?;
+  let base_parser = Url::options().base_url(Some(&base_url));
+  let links: HashSet<Url> = document
+    .find(Name("a"))
+    .filter_map(|n| n.attr("href"))
+    .filter_map(|link| base_parser.parse(link).ok())
+    .collect();
+    let mut tasks = vec![];
+    let mut ok = vec![];
+    let mut broken = vec![];
+
+    for link in links {
+        tasks.push(tokio::spawn(async move {
+            if check_link(&link).await.unwrap_or(false) {
+                Link::GoodLink(link) 
+            } else {
+                Link::BadLink(link)
+            }
+        }));
+    }
+
+    for task in tasks {
+        match task.await? {
+            Link::GoodLink(link) => ok.push(link.to_string()),
+            Link::BadLink(link) => broken.push(link.to_string()),
+        }
+    }
+
+  Ok(CategorizedUrls { ok, broken })
+}
+

crates/web/src/lib.rs

@@ -0,0 +1,49 @@
+mod broken;
+mod paginated;
+mod links;
+mod wiki;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_reverse_dependencies() -> reqwest::Result<()> {
+        for dep in paginated::ReverseDependencies::of("serde")?.take(5) {
+            let dependency = dep?;
+            println!("{} depends on {}", dependency.id, dependency.crate_id);
+        }
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_links() -> Result<(), links::LinkError> {
+        let page_links = links::get_links("https://rust-lang-nursery.github.io/rust-cookbook/").await?;
+        for link in page_links {
+            println!("{}", link);
+        }
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_broken() -> Result<(), broken::BrokenError> {
+        let categorized = broken::check("https://rust-lang-nursery.github.io/rust-cookbook/web/scraping.html").await?;
+        println!("OK: {:?}", categorized.ok);
+        println!("Broken: {:?}", categorized.broken);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_wiki() -> anyhow::Result<()> {
+      let content = reqwest::get(
+        "https://en.wikipedia.org/w/index.php?title=Rust_(programming_language)&action=raw",
+      )
+      .await?
+      .text()
+      .await?;
+    
+      println!("{:#?}", wiki::extract_links(content.as_str()));
+    
+      Ok(())
+    }
+}

crates/web/src/links.rs

@@ -0,0 +1,28 @@
+use thiserror::Error;
+use select::document::Document;
+use select::predicate::Name;
+
+#[derive(Error, Debug)]
+pub enum LinkError {
+    #[error("Reqwest error: {0}")]
+    ReqError(#[from] reqwest::Error),
+    #[error("IO error: {0}")]
+    IoError(#[from] std::io::Error),
+}
+
+pub async fn get_links(page: &str) -> Result<Vec<Box<str>>, LinkError> {
+  let res = reqwest::get(page)
+    .await?
+    .text()
+    .await?;
+
+  let links = Document::from(res.as_str())
+    .find(Name("a"))
+    .filter_map(|node| node.attr("href"))
+    .into_iter()
+    .map(|link| Box::<str>::from(link.to_string()))
+    .collect();
+
+  Ok(links)
+}
+

crates/web/src/paginated.rs

@@ -0,0 +1,79 @@
+use reqwest::Result;
+use reqwest::header::USER_AGENT;
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+struct ApiResponse {
+    dependencies: Vec<Dependency>,
+    meta: Meta,
+}
+
+#[derive(Deserialize)]
+pub struct Dependency {
+    pub crate_id: String,
+    pub id: u32,
+}
+
+#[derive(Deserialize)]
+struct Meta {
+    total: u32,
+}
+
+pub struct ReverseDependencies {
+    crate_id: String,
+    dependencies: <Vec<Dependency> as IntoIterator>::IntoIter,
+    client: reqwest::blocking::Client,
+    page: u32,
+    per_page: u32,
+    total: u32,
+}
+
+impl ReverseDependencies {
+    pub fn of(crate_id: &str) -> Result<Self> {
+        Ok(ReverseDependencies {
+               crate_id: crate_id.to_owned(),
+               dependencies: vec![].into_iter(),
+               client: reqwest::blocking::Client::new(),
+               page: 0,
+               per_page: 100,
+               total: 0,
+           })
+    }
+
+    fn try_next(&mut self) -> Result<Option<Dependency>> {
+        if let Some(dep) = self.dependencies.next() {
+            return Ok(Some(dep));
+        }
+
+        if self.page > 0 && self.page * self.per_page >= self.total {
+            return Ok(None);
+        }
+
+        self.page += 1;
+        let url = format!("https://crates.io/api/v1/crates/{}/reverse_dependencies?page={}&per_page={}",
+                          self.crate_id,
+                          self.page,
+                          self.per_page);
+        println!("{}", url);
+
+        let response = self.client.get(&url).header(
+                   USER_AGENT,
+                   "cookbook agent",
+               ).send()?.json::<ApiResponse>()?;
+        self.dependencies = response.dependencies.into_iter();
+        self.total = response.meta.total;
+        Ok(self.dependencies.next())
+    }
+}
+
+impl Iterator for ReverseDependencies {
+    type Item = Result<Dependency>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.try_next() {
+            Ok(Some(dep)) => Some(Ok(dep)),
+            Ok(None) => None,
+            Err(err) => Some(Err(err)),
+        }
+    }
+}

crates/web/src/wiki.rs

@@ -0,0 +1,27 @@
+use regex::Regex;
+use std::borrow::Cow;
+use std::collections::HashSet;
+use std::sync::LazyLock;
+
+pub fn extract_links(content: &str) -> HashSet<Cow<str>> {
+  static WIKI_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(
+      r"(?x)
+                \[\[(?P<internal>[^\[\]|]*)[^\[\]]*\]\]    # internal links
+                |
+                (url=|URL\||\[)(?P<external>http.*?)[ \|}] # external links
+            "
+    )
+    .unwrap()
+  );
+
+  let links: HashSet<_> = WIKI_REGEX
+    .captures_iter(content)
+    .map(|c| match (c.name("internal"), c.name("external")) {
+        (Some(val), None) => Cow::from(val.as_str()),
+        (None, Some(val)) => Cow::from(val.as_str()),
+        _ => unreachable!(),
+    })
+    .collect::<HashSet<_>>();
+
+  links
+}

src/web/clients/api/paginated.md

@@ -3,87 +3,17 @@
 [![reqwest-badge]][reqwest] [![serde-badge]][serde] [![cat-net-badge]][cat-net] [![cat-encoding-badge]][cat-encoding]
 
 Wraps a paginated web API in a convenient Rust iterator. The iterator lazily
-fetches the next page of results from the remote server as it arrives at the end
-of each page.
+fetches the next page of results from the remote server as it arrives at the end of each page.
 
-```rust,edition2018,no_run
-use reqwest::Result;
-use serde::Deserialize;
-
-#[derive(Deserialize)]
-struct ApiResponse {
-    dependencies: Vec<Dependency>,
-    meta: Meta,
-}
-
-#[derive(Deserialize)]
-struct Dependency {
-    crate_id: String,
-}
-
-#[derive(Deserialize)]
-struct Meta {
-    total: u32,
-}
-
-struct ReverseDependencies {
-    crate_id: String,
-    dependencies: <Vec<Dependency> as IntoIterator>::IntoIter,
-    client: reqwest::blocking::Client,
-    page: u32,
-    per_page: u32,
-    total: u32,
-}
-
-impl ReverseDependencies {
-    fn of(crate_id: &str) -> Result<Self> {
-        Ok(ReverseDependencies {
-               crate_id: crate_id.to_owned(),
-               dependencies: vec![].into_iter(),
-               client: reqwest::blocking::Client::new(),
-               page: 0,
-               per_page: 100,
-               total: 0,
-           })
-    }
-
-    fn try_next(&mut self) -> Result<Option<Dependency>> {
-        if let Some(dep) = self.dependencies.next() {
-            return Ok(Some(dep));
-        }
-
-        if self.page > 0 && self.page * self.per_page >= self.total {
-            return Ok(None);
-        }
-
-        self.page += 1;
-        let url = format!("https://crates.io/api/v1/crates/{}/reverse_dependencies?page={}&per_page={}",
-                          self.crate_id,
-                          self.page,
-                          self.per_page);
-
-        let response = self.client.get(&url).send()?.json::<ApiResponse>()?;
-        self.dependencies = response.dependencies.into_iter();
-        self.total = response.meta.total;
-        Ok(self.dependencies.next())
-    }
-}
-
-impl Iterator for ReverseDependencies {
-    type Item = Result<Dependency>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self.try_next() {
-            Ok(Some(dep)) => Some(Ok(dep)),
-            Ok(None) => None,
-            Err(err) => Some(Err(err)),
-        }
-    }
+```rust,edition2024,no_run
+mod paginated {
+  {{#include ../../../../crates/web/src/paginated.rs}}
 }
 
 fn main() -> Result<()> {
-    for dep in ReverseDependencies::of("serde")? {
-        println!("reverse dependency: {}", dep?.crate_id);
+    for dep in paginated::ReverseDependencies::of("serde")? {
+        let dependency = dep?;
+        println!("{} depends on {}", dependency.id, dependency.crate_id);
     }
     Ok(())
 }