Implement guide section

Signed-off-by: Sergio Castaño Arteaga <[email protected]>

Cargo.toml

@@ -21,6 +21,7 @@ futures = "0.3.28"
 hex = "0.4.3"
 lazy_static = "1.4.0"
 leaky-bucket = "1.0.1"
+markdown = "1.0.0-alpha.12"
 mime_guess = "2.0.4"
 num_cpus = "1.16.0"
 octorust = "0.3.2"

docs/config/guide.yml

@@ -0,0 +1,141 @@
+# Landscape2 guide
+#
+# This file allows defining the content of the landscape guide.
+
+# The landscape guide is organized into categories and subcategories. Each of
+# these entities requires a name and some content. The content can be provided
+# in markdown format. Categories and subcategories names are not required to
+# match the ones defined in the landscape data file but, when they do, those
+# categories/subcategories will be enriched with some extra information. So
+# whenever possible, it's highly recommended that they do.
+
+# We recommend using headings of level 4-6 within the content blocks as levels
+# 1-3 are reserved to illustrate the hierarchy of categories and subcategories.
+
+# The following example contains a subset of the CNCF landscape guide content:
+
+categories:
+  - category: "Introduction"
+    keywords: []
+    content: >
+      If you've researched cloud native applications and technologies, you've probably come
+      across the [CNCF cloud native landscape](https://landscape.cncf.io). Unsurprisingly,
+      the sheer scale of it can be overwhelming. So many categories and so many technologies.
+      How do you make sense of it?
+
+      As with anything else, if you break it down and analyze it one piece at a time, you'll
+      find it's not that complex and makes a lot of sense. In fact, the map is neatly organized
+      by functionality and, once you understand what each category represents, navigating it
+      becomes a lot easier.
+
+      In this guide, we'll break this mammoth landscape down and provide a high-level overview
+      of its layers, columns, and categories.
+
+    subcategories:
+      - subcategory: "What is the cloud native landscape?"
+        content: >
+          The goal of the cloud native landscape is to compile and organize all cloud native open
+          source projects and proprietary products into categories, providing an overview of the
+          current ecosystem. Organizations that have a cloud native project or product can
+          [submit a PR](https://github.com/cncf/landscape/) to request it to be added to the
+          landscape.
+
+      - subcategory: "How to use this guide"
+        content: >
+          In this guide, you'll find one chapter per layer and column which discusses each category
+          within it. Categories are broken down into: what it is, the problem it addresses, how it
+          helps, and technical 101. While the first three sections assume no technical background,
+          the technical 101 is targeted to engineers just getting started with cloud native. We
+          also included a section for associated buzzwords and lists CNCF projects.
+
+          > ##### INFOBOX
+          >
+          > When looking at the landscape, you'll note a few distinctions:
+          >  * *Projects in large boxes* are CNCF-hosted open source projects. Some are still in
+          >    the incubation phase (light blue/purple frame), while others are graduated
+          >    projects (dark blue frame).
+          >  * *Projects in small white boxes* are open source projects.
+          >  * *Products in gray boxes* are proprietary products.
+          >
+          > Please note that new projects are continuously becoming part of the CNCF so
+          > always refer to the actual landscape - things are moving fast!
+
+      - subcategory: "Contribute to the CNCF Landscape"
+        content: >
+          Are you searching for an exciting project to contribute to within the CNCF ecosystem?
+          Look no further! The CNCF hosts a wide range of projects that span cloud-native computing.
+          To find the perfect project for your skills and interests, check out our comprehensive
+          contribution guide at [Getting Started](https://contribute.cncf.io/contributors/getting-started/).
+          It provides you step-by-step instructions on getting started and offers valuable insights for
+          both newcomers and experienced contributors. Join our vibrant community and make your mark on
+          cloud-native innovation today!
+
+  - category: "Provisioning"
+    keywords: []
+    content: >
+      Provisioning is the first layer in the cloud native landscape. It encompasses tools that
+      are used to *create and harden* the foundation on which cloud native apps are built.
+      You'll find tools to automatically configure, create, and manage the infrastructure,
+      as well as  for scanning, signing, and storing container images. The layer also extends
+      to security with tools that enable policy setting and enforcement, embedded authentication
+      and authorization, and the handling of secrets distribution. That's a mouthful, so let's
+      discuss each category at a time.
+
+    subcategories:
+      - subcategory: "Automation & Configuration"
+        keywords:
+          - "Infrastructure-as-Code (IaC)"
+          - "Automation"
+          - "Declarative Configuration"
+        content: >
+          #### What it is
+
+          Automation and configuration tools speed up the creation and configuration of compute
+          resources (virtual machines, networks, firewall rules, load balancers, etc.). Tools in
+          this category either handle different parts of the provisioning process or try to control
+          everything end-to-end. Most provide the ability to integrate with other projects and
+          products in the space.
+
+          #### Problem it addresses
+
+          Traditionally, IT processes relied on lengthy and labor intensive manual release cycles,
+          typically between three to six months. Those cycles came with lots of human processes and
+          controls that slowed down changes to production environments. These slow release cycles
+          and static environments aren't compatible with cloud native development. To deliver on
+          rapid development cycles, infrastructure must be provisioned dynamically and without
+          human intervention.
+
+          #### How it helps
+
+          Tools of this category allow engineers to build computing environments without human
+          intervention. By codifying the environment setup it becomes reproducible with the click
+          of a button. While manual setup is error prone, once codified, environment creation
+          matches the exact desired state -- a huge advantage.
+
+          While tools may take different approaches, they all aim at reducing the required work
+          to provision resources through automation.
+
+          #### Technical 101
+
+          As we move from old-style human-driven provisioning to a new on-demand scaling model
+          driven by the cloud, the patterns and tools we used before no longer meet our needs.
+          Most organizations can't afford a large 24x7 staff to create, configure, and manage
+          servers. Automated tools like Terraform reduce the level of effort required to scale
+          tens of servers and networks with hundreds of firewall rules. Tools like Puppet, Chef,
+          and Ansible provision and/or configure these new servers and applications
+          programmatically as they are spun up and allow them to be consumed by developers.
+
+          Some tools interact directly with the infrastructure APIs provided by platforms like
+          AWS or vSphere, while others focus on configuring the individual machines to make them
+          part of a Kubernetes cluster. Many, like Chef and Terraform, can interoperate to provision
+          and configure the environment. Others, like OpenStack, exist to provide an
+          Infrastructure-as-a-Service (IaaS) environment that other tools could consume.
+          Fundamentally, you'll need one or more tools in this space as part of laying down the
+          computing environment, CPU, memory, storage, and networking, for your Kubernetes clusters.
+          You'll also need a subset of these to create and manage the Kubernetes clusters
+          themselves.
+
+          There are now over 5 CNCF projects in this space, more if you count projects like Cluster
+          API which don't appear on the landscape. There is also a very robust set of other open
+          source and vendor provided tools.
+

docs/config/settings.yml

@@ -1,4 +1,4 @@
-# Landscape2 settings file
+# Landscape2 settings
 #
 # This settings file allows customizing some aspects of the landscape.
 

src/build.rs

@@ -5,13 +5,14 @@
 use crate::{
     cache::Cache,
     crunchbase::collect_crunchbase_data,
-    data::{get_landscape_data, LandscapeData},
+    data::LandscapeData,
     datasets::Datasets,
     github::collect_github_data,
+    guide::LandscapeGuide,
     logos::prepare_logo,
     projects::{generate_projects_csv, Project, ProjectsMd},
-    settings::{get_landscape_settings, LandscapeSettings},
-    BuildArgs, LogosSource,
+    settings::LandscapeSettings,
+    BuildArgs, GuideSource, LogosSource,
 };
 use anyhow::{format_err, Result};
 use askama::Template;
@@ -62,15 +63,18 @@ pub(crate) async fn build(args: &BuildArgs) -> Result<()> {
     let cache = Cache::new(&args.cache_dir)?;
 
     // Get landscape data from the source provided
-    let mut landscape_data = get_landscape_data(&args.data_source).await?;
+    let mut landscape_data = LandscapeData::new(&args.data_source).await?;
 
     // Get landscape settings from the source provided
-    let settings = get_landscape_settings(&args.settings_source).await?;
+    let settings = LandscapeSettings::new(&args.settings_source).await?;
 
     // Add some extra information to the landscape based on the settings
     landscape_data.add_featured_items_data(&settings)?;
     landscape_data.add_member_subcategory(&settings.members_category);
 
+    // Prepare guide and copy it to the output directory
+    let includes_guide = prepare_guide(&args.guide_source, &args.output_dir).await?.is_some();
+
     // Prepare logos and copy them to the output directory
     prepare_logos(&cache, &args.logos_source, &mut landscape_data, &args.output_dir).await?;
 
@@ -88,7 +92,7 @@ pub(crate) async fn build(args: &BuildArgs) -> Result<()> {
     let datasets = generate_datasets(&landscape_data, &settings, &args.output_dir)?;
 
     // Render index file and write it to the output directory
-    render_index(&datasets, &args.output_dir)?;
+    render_index(&datasets, includes_guide, &args.output_dir)?;
 
     // Copy web assets files to the output directory
     copy_web_assets(&args.output_dir)?;
@@ -105,6 +109,8 @@ pub(crate) async fn build(args: &BuildArgs) -> Result<()> {
 /// Check web assets are present, to make sure the web app has been built.
 #[instrument(skip_all, err)]
 fn check_web_assets() -> Result<()> {
+    debug!("checking web assets are present");
+
     if !WebAssets::iter().any(|path| path.starts_with("assets/")) {
         return Err(format_err!(
             "web assets not found, please make sure they have been built"
@@ -117,6 +123,8 @@ fn check_web_assets() -> Result<()> {
 /// Copy web assets files to the output directory.
 #[instrument(skip_all, err)]
 fn copy_web_assets(output_dir: &Path) -> Result<()> {
+    debug!("copying web assets to output directory");
+
     for asset_path in WebAssets::iter() {
         // The index document is a template that we'll render, so we don't want
         // to copy it as is.
@@ -125,7 +133,6 @@ fn copy_web_assets(output_dir: &Path) -> Result<()> {
         }
 
         if let Some(embedded_file) = WebAssets::get(&asset_path) {
-            debug!(?asset_path, "copying file");
             if let Some(parent_path) = Path::new(asset_path.as_ref()).parent() {
                 fs::create_dir_all(output_dir.join(parent_path))?;
             }
@@ -148,9 +155,8 @@ fn generate_datasets(
     output_dir: &Path,
 ) -> Result<Datasets> {
     debug!("generating datasets");
-    let datasets = Datasets::new(landscape_data, settings)?;
 
-    debug!("copying datasets to output directory");
+    let datasets = Datasets::new(landscape_data, settings)?;
     let datasets_path = output_dir.join(DATASETS_PATH);
 
     // Base
@@ -167,7 +173,8 @@ fn generate_datasets(
 /// Generate the projects.md and projects.csv files from the landscape data.
 #[instrument(skip_all, err)]
 fn generate_projects_files(landscape_data: &LandscapeData, output_dir: &Path) -> Result<()> {
-    debug!("generating projects.* files");
+    debug!("generating projects files");
+
     let projects: Vec<Project> = landscape_data.into();
 
     // projects.md
@@ -183,6 +190,20 @@ fn generate_projects_files(landscape_data: &LandscapeData, output_dir: &Path) ->
     Ok(())
 }
 
+/// Prepare guide and copy it to the output directory.
+#[instrument(skip_all, err)]
+async fn prepare_guide(guide_source: &GuideSource, output_dir: &Path) -> Result<Option<()>> {
+    debug!("preparing guide");
+
+    let Some(guide) = LandscapeGuide::new(guide_source).await? else {
+        return Ok(None);
+    };
+    let path = output_dir.join(DATASETS_PATH).join("guide.json");
+    File::create(path)?.write_all(&serde_json::to_vec(&guide)?)?;
+
+    Ok(Some(()))
+}
+
 /// Prepare logos and copy them to the output directory, updating the logo
 /// reference on each landscape item.
 #[instrument(skip_all, err)]
@@ -257,13 +278,19 @@ async fn prepare_logos(
 #[template(path = "index.html", escape = "none")]
 struct Index<'a> {
     pub datasets: &'a Datasets,
+    pub includes_guide: bool,
 }
 
 /// Render index file and write it to the output directory.
 #[instrument(skip_all, err)]
-fn render_index(datasets: &Datasets, output_dir: &Path) -> Result<()> {
+fn render_index(datasets: &Datasets, includes_guide: bool, output_dir: &Path) -> Result<()> {
     debug!("rendering index.html file");
-    let index = Index { datasets }.render()?;
+
+    let index = Index {
+        datasets,
+        includes_guide,
+    }
+    .render()?;
     let mut file = File::create(output_dir.join("index.html"))?;
     file.write_all(index.as_bytes())?;
 
@@ -274,6 +301,8 @@ fn render_index(datasets: &Datasets, output_dir: &Path) -> Result<()> {
 /// paths inside it when needed.
 #[instrument(fields(?output_dir), skip_all, err)]
 fn setup_output_dir(output_dir: &Path) -> Result<()> {
+    debug!("setting up output directory");
+
     if !output_dir.exists() {
         debug!("creating output directory");
         fs::create_dir_all(output_dir)?;

src/data.rs

@@ -24,20 +24,6 @@ use uuid::Uuid;
 /// Format used for dates across the landscape data file.
 pub const DATE_FORMAT: &str = "%Y-%m-%d";
 
-/// Get landscape data from the source provided.
-#[instrument(skip_all, err)]
-pub(crate) async fn get_landscape_data(src: &DataSource) -> Result<LandscapeData> {
-    let data = if let Some(file) = &src.data_file {
-        debug!(?file, "getting landscape data from file");
-        LandscapeData::new_from_file(file)
-    } else {
-        debug!(url = ?src.data_url.as_ref().unwrap(), "getting landscape data from url");
-        LandscapeData::new_from_url(src.data_url.as_ref().unwrap()).await
-    }?;
-
-    Ok(data)
-}
-
 /// Landscape data.
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
 pub(crate) struct LandscapeData {
@@ -46,8 +32,26 @@ pub(crate) struct LandscapeData {
 }
 
 impl LandscapeData {
+    /// Create a new landscape data instance from the source provided.
+    #[instrument(skip_all, err)]
+    pub(crate) async fn new(src: &DataSource) -> Result<Self> {
+        // Try from file
+        if let Some(file) = &src.data_file {
+            debug!(?file, "getting landscape data from file");
+            return LandscapeData::new_from_file(file);
+        };
+
+        // Try from url
+        if let Some(url) = &src.data_url {
+            debug!(?url, "getting landscape data from url");
+            return LandscapeData::new_from_url(url).await;
+        };
+
+        Err(format_err!("data file or url not provided"))
+    }
+
     /// Create a new landscape data instance from the file provided.
-    pub(crate) fn new_from_file(file: &Path) -> Result<Self> {
+    fn new_from_file(file: &Path) -> Result<Self> {
         let raw_data = fs::read_to_string(file)?;
         let legacy_data: legacy::LandscapeData = serde_yaml::from_str(&raw_data)?;
         legacy_data.validate()?;
@@ -56,7 +60,7 @@ impl LandscapeData {
     }
 
     /// Create a new landscape data instance from the url provided.
-    pub(crate) async fn new_from_url(url: &str) -> Result<Self> {
+    async fn new_from_url(url: &str) -> Result<Self> {
         let resp = reqwest::get(url).await?;
         if resp.status() != StatusCode::OK {
             return Err(format_err!(