Some refactoring in CLI (#159)

Signed-off-by: Sergio Castaño Arteaga <[email protected]>

src/cache.rs → src/build/cache.rs

@@ -1,3 +1,5 @@
+//! This module defines the cache used to cache files across builds.
+
 use anyhow::{format_err, Result};
 use std::{fs, io::Write, path::PathBuf};
 use tracing::instrument;
@@ -8,7 +10,7 @@ const CACHE_PATH: &str = "landscape";
 /// Cache used to store data collected from external services.
 #[derive(Debug, Clone, Default)]
 pub(crate) struct Cache {
-    pub cache_dir: PathBuf,
+    cache_dir: PathBuf,
 }
 
 impl Cache {

src/crunchbase.rs → src/build/crunchbase.rs

@@ -2,7 +2,7 @@
 //! from Crunchbase for each of the landscape items (when applicable), as well
 //! as the functionality used to collect that information.
 
-use crate::{cache::Cache, data::LandscapeData};
+use super::{cache::Cache, LandscapeData};
 use anyhow::{format_err, Result};
 use async_trait::async_trait;
 use chrono::{DateTime, Utc};
@@ -117,11 +117,13 @@ pub(crate) async fn collect_crunchbase_data(
 pub(crate) type CrunchbaseData = HashMap<CrunchbaseUrl, Organization>;
 
 /// Type alias to represent a crunchbase url.
-pub(crate) type CrunchbaseUrl = String;
+type CrunchbaseUrl = String;
 
 /// Organization information collected from Crunchbase.
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
 pub(crate) struct Organization {
+    pub generated_at: DateTime<Utc>,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub city: Option<String>,
 
@@ -137,9 +139,6 @@ pub(crate) struct Organization {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub funding: Option<i64>,
 
-    /// Represents the moment at which this instance was generated
-    pub generated_at: DateTime<Utc>,
-
     #[serde(skip_serializing_if = "Option::is_none")]
     pub homepage_url: Option<String>,
 
@@ -177,8 +176,8 @@ pub(crate) struct Organization {
 impl Organization {
     /// Create a new Organization instance from information obtained from the
     /// Crunchbase API.
-    pub(crate) async fn new(cb: DynCB, cb_url: &str) -> Result<Self> {
-        // Collec some information from Crunchbase
+    async fn new(cb: DynCB, cb_url: &str) -> Result<Self> {
+        // Collect some information from Crunchbase
         let permalink = get_permalink(cb_url)?;
         let cb_org = cb.get_organization(&permalink).await?;
 
@@ -224,24 +223,24 @@ impl Organization {
 const CRUNCHBASE_API_URL: &str = "https://api.crunchbase.com/api/v4";
 
 /// Type alias to represent a CB trait object.
-pub(crate) type DynCB = Arc<dyn CB + Send + Sync>;
+type DynCB = Arc<dyn CB + Send + Sync>;
 
 /// Trait that defines some operations a CB implementation must support.
 #[async_trait]
 #[cfg_attr(test, automock)]
-pub(crate) trait CB {
+trait CB {
     /// Get organization information.
     async fn get_organization(&self, permalink: &str) -> Result<CBOrganizationEntity>;
 }
 
 /// CB implementation backed by the Crunchbase API.
-pub struct CBApi {
+struct CBApi {
     http_client: reqwest::Client,
 }
 
 impl CBApi {
     /// Create a new CBApi instance.
-    pub fn new(key: &str) -> Result<Self> {
+    fn new(key: &str) -> Result<Self> {
         // Setup HTTP client ready to make requests to the Crunchbase API
         let user_agent = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
         let mut headers = header::HeaderMap::new();
@@ -286,13 +285,13 @@ impl CB for CBApi {
 }
 
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-pub(crate) struct CBOrganizationEntity {
+struct CBOrganizationEntity {
     properties: CBOrganization,
     cards: CBCards,
 }
 
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-pub(crate) struct CBOrganization {
+struct CBOrganization {
     categories: Option<Vec<CBEntityIdentifier>>,
     company_type: Option<String>,
     funding_total: Option<CBFundingTotal>,
@@ -307,32 +306,32 @@ pub(crate) struct CBOrganization {
 }
 
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-pub(crate) struct CBEntityIdentifier {
+struct CBEntityIdentifier {
     value: Option<String>,
 }
 
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-pub(crate) struct CBFundingTotal {
+struct CBFundingTotal {
     value_usd: Option<i64>,
 }
 
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-pub(crate) struct CBValue {
+struct CBValue {
     value: Option<String>,
 }
 
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-pub(crate) struct CBCards {
+struct CBCards {
     headquarters_address: Option<Vec<CBAddress>>,
 }
 
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-pub(crate) struct CBAddress {
+struct CBAddress {
     location_identifiers: Option<Vec<CBLocationIdentifier>>,
 }
 
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-pub(crate) struct CBLocationIdentifier {
+struct CBLocationIdentifier {
     location_type: Option<String>,
     value: Option<String>,
 }

src/data.rs → src/build/data.rs

@@ -7,12 +7,12 @@
 //! backwards compatibility, this module provides a `legacy` submodule that
 //! allows parsing the legacy format and convert it to the new one.
 
-use crate::{
+use super::{
     crunchbase::{CrunchbaseData, Organization},
     github::{self, GithubData},
     settings::LandscapeSettings,
-    DataSource,
 };
+use crate::DataSource;
 use anyhow::{format_err, Result};
 use chrono::NaiveDate;
 use reqwest::StatusCode;
@@ -75,7 +75,7 @@ impl LandscapeData {
         Ok(LandscapeData::from(legacy_data))
     }
 
-    /// Add Crunchbase data to the landscape data.
+    /// Add items Crunchbase data.
     #[instrument(skip_all, err)]
     pub(crate) fn add_crunchbase_data(&mut self, crunchbase_data: CrunchbaseData) -> Result<()> {
         for item in &mut self.items {
@@ -133,7 +133,7 @@ impl LandscapeData {
         Ok(())
     }
 
-    /// Add GitHub data to the landscape data.
+    /// Add items repositories GitHub data.
     #[instrument(skip_all, err)]
     pub(crate) fn add_github_data(&mut self, github_data: GithubData) -> Result<()> {
         for item in &mut self.items {
@@ -199,14 +199,15 @@ impl From<legacy::LandscapeData> for LandscapeData {
                     let mut item = Item {
                         name: legacy_item.name,
                         category: legacy_category.name.clone(),
-                        subcategory: legacy_subcategory.name.clone(),
                         crunchbase_url: legacy_item.crunchbase,
+                        description: legacy_item.description.clone(),
                         enduser: legacy_item.enduser,
-                        homepage_url: legacy_item.homepage_url,
                         joined_at: legacy_item.joined,
+                        homepage_url: legacy_item.homepage_url,
                         logo: legacy_item.logo,
                         maturity: legacy_item.project,
                         openssf_best_practices_url: legacy_item.url_for_bestpractices,
+                        subcategory: legacy_subcategory.name.clone(),
                         twitter_url: legacy_item.twitter,
                         unnamed_organization: legacy_item.unnamed_organization,
                         ..Default::default()
@@ -240,7 +241,6 @@ impl From<legacy::LandscapeData> for LandscapeData {
                     // Additional information in extra field
                     if let Some(extra) = legacy_item.extra {
                         item.accepted_at = extra.accepted;
-                        item.latest_annual_review_at = extra.annual_review_date;
                         item.archived_at = extra.archived;
                         item.artwork_url = extra.artwork_url;
                         item.audits = extra.audits;
@@ -252,8 +252,9 @@ impl From<legacy::LandscapeData> for LandscapeData {
                         item.github_discussions_url = extra.github_discussions_url;
                         item.graduated_at = extra.graduated;
                         item.incubating_at = extra.incubating;
-                        item.mailing_list_url = extra.mailing_list_url;
+                        item.latest_annual_review_at = extra.annual_review_date;
                         item.latest_annual_review_url = extra.annual_review_url;
+                        item.mailing_list_url = extra.mailing_list_url;
                         item.slack_url = extra.slack_url;
                         item.specification = extra.specification;
                         item.stack_overflow_url = extra.stack_overflow_url;
@@ -347,6 +348,9 @@ pub(crate) struct Item {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub crunchbase_url: Option<String>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     pub devstats_url: Option<String>,
 
@@ -418,6 +422,34 @@ pub(crate) struct Item {
 }
 
 impl Item {
+    /// Get item's description.
+    #[allow(dead_code)]
+    pub(crate) fn description(&self) -> Option<&String> {
+        // Use item's description if available
+        let mut description = self.description.as_ref();
+
+        // Otherwise, use primary repository description if available
+        if description.is_none() || description.unwrap().is_empty() {
+            description =
+                self.primary_repository().and_then(|r| r.github_data.as_ref().map(|gh| &gh.description));
+        }
+
+        // Otherwise, use Crunchbase data description
+        if description.is_none() || description.unwrap().is_empty() {
+            description = self.crunchbase_data.as_ref().and_then(|cb| cb.description.as_ref());
+        }
+
+        description
+    }
+
+    /// Get primary repository if available.
+    #[allow(dead_code)]
+    pub(crate) fn primary_repository(&self) -> Option<&Repository> {
+        self.repositories
+            .as_ref()
+            .and_then(|repos| repos.iter().find(|r| r.primary.unwrap_or_default()))
+    }
+
     /// Generate and set the item's id.
     fn set_id(&mut self) {
         let key = format!("{}##{}##{}", &self.category, &self.subcategory, &self.name);
@@ -476,7 +508,7 @@ pub(crate) struct ItemSummary {
 /// Project maturity level.
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "kebab-case")]
-pub(super) enum Maturity {
+pub(crate) enum Maturity {
     Archived,
     Graduated,
     Incubating,
@@ -528,21 +560,21 @@ mod legacy {
     //! legacy format and convert it to the new one.
 
     use super::{ItemAudit, Maturity};
-    use crate::{crunchbase::CRUNCHBASE_URL, github::GITHUB_REPO_URL};
+    use crate::build::{crunchbase::CRUNCHBASE_URL, github::GITHUB_REPO_URL};
     use anyhow::{format_err, Context, Result};
     use chrono::NaiveDate;
     use serde::{Deserialize, Serialize};
     use url::Url;
 
     /// Landscape data (legacy format).
     #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-    pub(super) struct LandscapeData {
+    pub(crate) struct LandscapeData {
         pub landscape: Vec<Category>,
     }
 
     impl LandscapeData {
         /// Validate landscape data.
-        pub(super) fn validate(&self) -> Result<()> {
+        pub(crate) fn validate(&self) -> Result<()> {
             let mut items_seen = Vec::new();
 
             for (category_index, category) in self.landscape.iter().enumerate() {
@@ -603,27 +635,28 @@ mod legacy {
 
     /// Landscape category.
     #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-    pub(super) struct Category {
+    pub(crate) struct Category {
         pub name: String,
         pub subcategories: Vec<SubCategory>,
     }
 
     /// Landscape subcategory.
     #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-    pub(super) struct SubCategory {
+    pub(crate) struct SubCategory {
         pub name: String,
         pub items: Vec<Item>,
     }
 
     /// Landscape item (project, product, member, etc).
     #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-    pub(super) struct Item {
+    pub(crate) struct Item {
         pub name: String,
         pub homepage_url: String,
         pub logo: String,
         pub additional_repos: Option<Vec<Repository>>,
         pub branch: Option<String>,
         pub crunchbase: Option<String>,
+        pub description: Option<String>,
         pub enduser: Option<bool>,
         pub extra: Option<ItemExtra>,
         pub joined: Option<NaiveDate>,
@@ -636,14 +669,14 @@ mod legacy {
 
     /// Landscape item repository.
     #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-    pub(super) struct Repository {
+    pub(crate) struct Repository {
         pub repo_url: String,
         pub branch: Option<String>,
     }
 
     /// Extra information for a landscape item.
     #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
-    pub(super) struct ItemExtra {
+    pub(crate) struct ItemExtra {
         pub accepted: Option<NaiveDate>,
         pub archived: Option<NaiveDate>,
         pub audits: Option<Vec<ItemAudit>>,

src/datasets.rs → src/build/datasets.rs

@@ -6,7 +6,7 @@
 //! that they can be fetched when needed.
 
 use self::{base::Base, full::Full};
-use crate::{data::LandscapeData, settings::LandscapeSettings};
+use super::{settings::LandscapeSettings, LandscapeData};
 use anyhow::{Ok, Result};
 
 /// Datasets collection.
@@ -40,13 +40,14 @@ impl Datasets {
 /// This dataset contains the minimal data the web application needs to render
 /// the initial page and power the features available on it.
 mod base {
-    use crate::{
+    use crate::build::{
         data::{Category, CategoryName, ItemFeatured, LandscapeData, Maturity},
         settings::{Colors, GridItemsSize, Group, Images, LandscapeSettings, SocialNetworks},
     };
     use serde::{Deserialize, Serialize};
     use uuid::Uuid;
 
+    /// Base dataset information.
     #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
     pub(crate) struct Base {
         pub foundation: String,
@@ -75,6 +76,7 @@ mod base {
         pub social_networks: Option<SocialNetworks>,
     }
 
+    /// Base dataset item information.
     #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
     pub(crate) struct Item {
         pub category: String,
@@ -145,9 +147,10 @@ mod base {
 /// information is used by the web application to power features that require
 /// some extra data not available in the base dataset.
 mod full {
-    use crate::data::{Item, LandscapeData};
+    use crate::build::data::{Item, LandscapeData};
     use serde::{Deserialize, Serialize};
 
+    /// Full dataset information.
     #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
     pub(crate) struct Full {
         #[serde(skip_serializing_if = "Vec::is_empty")]