Merge pull request #3272 from owid/data-insights-in-sitemap

Data insights in sitemap and site nav

adminSiteServer/mockSiteRouter.tsx

@@ -53,6 +53,7 @@ import {
 import { GdocPost } from "../db/model/Gdoc/GdocPost.js"
 import { GdocDataInsight } from "../db/model/Gdoc/GdocDataInsight.js"
 import * as db from "../db/db.js"
+import { calculateDataInsightIndexPageCount } from "../db/model/Gdoc/gdocUtils.js"
 
 require("express-async-errors")
 
@@ -194,12 +195,16 @@ mockSiteRouter.get("/thank-you", async (req, res) =>
 )
 
 mockSiteRouter.get("/data-insights/:pageNumberOrSlug?", async (req, res) => {
+    const totalPageCount = calculateDataInsightIndexPageCount(
+        await db
+            .getPublishedDataInsights(db.knexInstance())
+            .then((insights) => insights.length)
+    )
     async function renderIndexPage(pageNumber: number) {
         const dataInsights =
             await GdocDataInsight.getPublishedDataInsights(pageNumber)
         // calling fetchImageMetadata 20 times makes me sad, would be nice if we could cache this
         await Promise.all(dataInsights.map((insight) => insight.loadState()))
-        const totalPageCount = await GdocDataInsight.getTotalPageCount()
         return renderDataInsightsIndexPage(
             dataInsights,
             pageNumber,
@@ -215,9 +220,9 @@ mockSiteRouter.get("/data-insights/:pageNumberOrSlug?", async (req, res) => {
     // pageNumber is 1-indexed, but DB operations are 0-indexed
     const pageNumber = parseInt(pageNumberOrSlug) - 1
     if (!isNaN(pageNumber)) {
-        if (pageNumber <= 0) return res.redirect("/data-insights")
-        const totalPages = await GdocDataInsight.getTotalPageCount()
-        if (pageNumber >= totalPages) return res.redirect("/data-insights")
+        if (pageNumber <= 0 || pageNumber >= totalPageCount) {
+            return res.redirect("/data-insights")
+        }
         return res.send(await renderIndexPage(pageNumber))
     }
 

baker/SiteBaker.tsx

@@ -95,7 +95,10 @@ import {
 } from "../settings/clientSettings.js"
 import pMap from "p-map"
 import { GdocDataInsight } from "../db/model/Gdoc/GdocDataInsight.js"
-import { fullGdocToMinimalGdoc } from "../db/model/Gdoc/gdocUtils.js"
+import {
+    calculateDataInsightIndexPageCount,
+    fullGdocToMinimalGdoc,
+} from "../db/model/Gdoc/gdocUtils.js"
 import {
     getVariableMetadata,
     getVariableOfDatapageIfApplicable,
@@ -698,7 +701,10 @@ export class SiteBaker {
 
     private async bakeDataInsights() {
         if (!this.bakeSteps.has("dataInsights")) return
-        const latestDataInsights = await db.getLatestDataInsights()
+        const latestDataInsights = await db.getPublishedDataInsights(
+            db.knexInstance(),
+            5
+        )
         const publishedDataInsights =
             await GdocDataInsight.getPublishedDataInsights()
 
@@ -740,7 +746,10 @@ export class SiteBaker {
             }
         }
 
-        const totalPageCount = await GdocDataInsight.getTotalPageCount()
+        const totalPageCount = calculateDataInsightIndexPageCount(
+            publishedDataInsights.length
+        )
+
         for (let pageNumber = 0; pageNumber < totalPageCount; pageNumber++) {
             const html = renderDataInsightsIndexPage(
                 publishedDataInsights.slice(

baker/sitemap.ts

@@ -14,6 +14,7 @@ import { ExplorerProgram } from "../explorer/ExplorerProgram.js"
 import { GdocPost } from "../db/model/Gdoc/GdocPost.js"
 import { getPostsFromSnapshots } from "../db/model/Post.js"
 import { Knex } from "knex"
+import { calculateDataInsightIndexPageCount } from "../db/model/Gdoc/gdocUtils.js"
 
 interface SitemapUrl {
     loc: string
@@ -70,6 +71,12 @@ export const makeSitemap = async (
         (postrow) => !alreadyPublishedViaGdocsSlugsSet.has(postrow.slug)
     )
     const gdocPosts = await GdocPost.getPublishedGdocs()
+
+    const publishedDataInsights = await db.getPublishedDataInsights(knex)
+    const dataInsightFeedPageCount = calculateDataInsightIndexPageCount(
+        publishedDataInsights.length
+    )
+
     const charts = (await db
         .knexTable(Chart.table)
         .select(knex.raw(`updatedAt, config->>"$.slug" AS slug`))
@@ -104,6 +111,25 @@ export const makeSitemap = async (
                 lastmod: dayjs(p.updatedAt).format("YYYY-MM-DD"),
             }))
         )
+        .concat(
+            publishedDataInsights.map((d) => ({
+                loc: urljoin(BAKED_BASE_URL, "data-insights", d.slug),
+                lastmod: dayjs(d.updatedAt).format("YYYY-MM-DD"),
+            }))
+        )
+        .concat(
+            // Data insight index pages: /data-insights, /data-insights/2, /data-insights/3, etc.
+            Array.from({ length: dataInsightFeedPageCount }, (_, i) => ({
+                loc: urljoin(
+                    BAKED_BASE_URL,
+                    "data-insights",
+                    i === 0 ? "" : `/${i + 1}`
+                ),
+                lastmod: dayjs(publishedDataInsights[0].updatedAt).format(
+                    "YYYY-MM-DD"
+                ),
+            }))
+        )
         .concat(
             charts.map((c) => ({
                 loc: urljoin(BAKED_GRAPHER_URL, c.slug),

db/db.ts

@@ -209,23 +209,25 @@ export const getPublishedExplorersBySlug = async (
     })
 }
 
-export const getLatestDataInsights = (
-    limit = 5
+export const getPublishedDataInsights = (
+    knex: Knex<any, any[]>,
+    limit = Number.MAX_SAFE_INTEGER // default to no limit
 ): Promise<MinimalDataInsightInterface[]> => {
     return knexRaw(
         `
         SELECT
             content->>'$.title' AS title,
             publishedAt,
+            updatedAt,
+            slug,
             ROW_NUMBER() OVER (ORDER BY publishedAt DESC) - 1 AS \`index\`
         FROM posts_gdocs
         WHERE content->>'$.type' = '${OwidGdocType.DataInsight}'
             AND published = TRUE
             AND publishedAt < NOW()
         ORDER BY publishedAt DESC
-        LIMIT ?
-        `,
-        knexInstance(),
+        LIMIT ?`,
+        knex,
         [limit]
     ).then((results) =>
         results.map((record: any) => ({
@@ -235,18 +237,6 @@ export const getLatestDataInsights = (
     ) as Promise<MinimalDataInsightInterface[]>
 }
 
-export const getPublishedDataInsightCount = (): Promise<number> => {
-    return knexRawFirst<{ count: number }>(
-        `
-        SELECT COUNT(*) AS count
-        FROM posts_gdocs
-        WHERE content->>'$.type' = '${OwidGdocType.DataInsight}'
-            AND published = TRUE
-            AND publishedAt < NOW()`,
-        knexInstance()
-    ).then((res) => res?.count ?? 0)
-}
-
 export const getTotalNumberOfCharts = (): Promise<number> => {
     return knexRawFirst<{ count: number }>(
         `

db/model/Gdoc/GdocDataInsight.ts

@@ -45,7 +45,11 @@ export class GdocDataInsight
     }
 
     _loadSubclassAttachments = async (): Promise<void> => {
-        this.latestDataInsights = await db.getLatestDataInsights()
+        // TODO: refactor these classes to properly use knex - not going to start it now
+        this.latestDataInsights = await db.getPublishedDataInsights(
+            db.knexInstance(),
+            5
+        )
     }
 
     static async getPublishedDataInsights(
@@ -69,13 +73,4 @@ export class GdocDataInsight
             relations: ["tags"],
         })
     }
-
-    /**
-     * Returns the number of pages that will exist on the data insights index page
-     * based on the number of published data insights and DATA_INSIGHTS_INDEX_PAGE_SIZE
-     */
-    static async getTotalPageCount(): Promise<number> {
-        const count = await db.getPublishedDataInsightCount()
-        return Math.ceil(count / DATA_INSIGHTS_INDEX_PAGE_SIZE)
-    }
 }

db/model/Gdoc/GdocHomepage.ts

@@ -60,6 +60,10 @@ export class GdocHomepage
             topicCount: UNIQUE_TOPIC_COUNT,
         }
 
-        this.latestDataInsights = await db.getLatestDataInsights(4)
+        // TODO: refactor these classes to properly use knex - not going to start it now
+        this.latestDataInsights = await db.getPublishedDataInsights(
+            db.knexInstance(),
+            4
+        )
     }
 }

db/model/Gdoc/gdocUtils.ts

@@ -7,6 +7,7 @@ import {
     OwidGdocType,
     formatDate,
     OwidGdocBaseInterface,
+    DATA_INSIGHTS_INDEX_PAGE_SIZE,
 } from "@ourworldindata/utils"
 import { match, P } from "ts-pattern"
 import cheerio from "cheerio"
@@ -172,3 +173,13 @@ export function fullGdocToMinimalGdoc(
         "featured-image": gdoc.content["featured-image"],
     }
 }
+
+/**
+ * Calculate the number of pages needed to display all data insights
+ * e.g. if there are 61 data insights and we want to display 20 per page, we need 4 pages
+ */
+export function calculateDataInsightIndexPageCount(
+    publishedDataInsightCount: number
+): number {
+    return Math.ceil(publishedDataInsightCount / DATA_INSIGHTS_INDEX_PAGE_SIZE)
+}

packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts

@@ -112,6 +112,8 @@ export type MinimalDataInsightInterface = Pick<
     "title"
 > & {
     publishedAt: string
+    updatedAt: string
+    slug: string
     // We select the 5 most recently published insights
     // We only display 4, but if you're on the DI page for one of them we hide it and show the next most recent
     index: 0 | 1 | 2 | 3 | 4

site/SiteResources.tsx

@@ -6,6 +6,9 @@ export const SiteResources = () => {
             <li>
                 <a href="/charts">Charts and Explorers</a>
             </li>
+            <li>
+                <a href="/data-insights">Data Insights</a>
+            </li>
             <li>
                 <a href="/sdgs">Sustainable Development Goals Tracker</a>
             </li>