Skip to content

Commit

Permalink
🔨 remove grapher static exports, replace with CF worker URL
Browse files Browse the repository at this point in the history
  • Loading branch information
@ikesau
ikesau committed Nov 5, 2024
1 parent 7abc704 commit 0c68bdb
Show file tree
Hide file tree
Showing 17 changed files with 110 additions and 344 deletions.
7 changes: 2 additions & 5 deletions adminSiteClient/ChartRow.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,7 @@ import { Link } from "./Link.js"
import { Timeago } from "./Forms.js"
import { EditableTags } from "./EditableTags.js"
import { AdminAppContext, AdminAppContextType } from "./AdminAppContext.js"
import {
BAKED_GRAPHER_EXPORTS_BASE_URL,
BAKED_GRAPHER_URL,
} from "../settings/clientSettings.js"
import { BAKED_GRAPHER_URL } from "../settings/clientSettings.js"
import { ChartListItem, showChartType } from "./ChartList.js"
import { TaggableType, DbChartTagJoin } from "@ourworldindata/utils"

Expand Down Expand Up @@ -52,7 +49,7 @@ export class ChartRow extends React.Component<{
{chart.isPublished ? (
<a href={`${BAKED_GRAPHER_URL}/${chart.slug}`}>
<img
src={`${BAKED_GRAPHER_EXPORTS_BASE_URL}/${chart.slug}.svg`}
src={`/grapher/exports/${chart.slug}.svg`}
className="chartPreview"
/>
</a>
Expand Down
10 changes: 2 additions & 8 deletions adminSiteServer/mockSiteRouter.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import {
import {
BAKED_BASE_URL,
BASE_DIR,
BAKED_SITE_DIR,
LEGACY_WORDPRESS_IMAGE_URL,
} from "../settings/serverSettings.js"

Expand Down Expand Up @@ -435,16 +434,11 @@ mockSiteRouter.use(
res.redirect(assetUrl)
}
)
mockSiteRouter.use(
"/exports",
express.static(path.join(BAKED_SITE_DIR, "exports"))
)

mockSiteRouter.use("/assets", express.static("dist/assets"))

// TODO: this used to be a mockSiteRouter.use call but otherwise it looked like a route and
// it didn't look like it was making use of any middleware - if this causese issues then
// this has to be reverted to a use call
// We've replaced static grapher exports with a Cloudflare Worker, so this is no longer "mocking" the real site
// But it's still useful to have for the /admin/charts page
getPlainRouteWithROTransaction(
mockSiteRouter,
"/grapher/exports/:slug.svg",
Expand Down
111 changes: 0 additions & 111 deletions baker/GrapherBakingUtils.ts
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,9 @@
import { glob } from "glob"
import path from "path"
import * as lodash from "lodash"
import {
OPTIMIZE_SVG_EXPORTS,
BAKED_SITE_DIR,
} from "../settings/serverSettings.js"
import { BAKED_SITE_EXPORTS_BASE_URL } from "../settings/clientSettings.js"

import * as db from "../db/db.js"
import { bakeGraphersToSvgs } from "../baker/GrapherImageBaker.js"
import { warn } from "../serverUtils/errorLog.js"
import { mapSlugsToIds } from "../db/model/Chart.js"
import md5 from "md5"
import { DbPlainTag, Url } from "@ourworldindata/utils"

interface ChartExportMeta {
key: string
svgUrl: string
version: number
width: number
height: number
}

// Splits a grapher URL like https://ourworldindata.org/grapher/soil-lifespans?tab=chart
// into its slug (soil-lifespans) and queryStr (?tab=chart)
export const grapherUrlToSlugAndQueryStr = (grapherUrl: string) => {
Expand All @@ -48,99 +30,6 @@ export const grapherSlugToExportFileKey = (
return `${slug}${queryStr ? `${separator}${maybeHashedQueryStr}` : ""}`
}

export interface GrapherExports {
get: (grapherUrl: string) => ChartExportMeta | undefined
}

export const bakeGrapherUrls = async (
knex: db.KnexReadonlyTransaction,
urls: string[]
) => {
const currentExports = await getGrapherExportsByUrl()
const slugToId = await mapSlugsToIds(knex)
const toBake = []

// Check that we need to bake this url, and don't already have an export
for (const url of urls) {
const current = currentExports.get(url)
if (!current) {
toBake.push(url)
continue
}

const slug = lodash.last(Url.fromURL(url).pathname?.split("/"))
if (!slug) {
warn(`Invalid chart url ${url}`)
continue
}

const chartId = slugToId[slug]
if (chartId === undefined) {
warn(`Couldn't find chart with slug ${slug}`)
continue
}

const rows = await db.knexRaw<{ version: number }>(
knex,
`-- sql
SELECT cc.full->>"$.version" AS version
FROM charts c
JOIN chart_configs cc ON c.configId = cc.id
WHERE c.id=?
`,
[chartId]
)
if (!rows.length) {
warn(`Mysteriously missing chart by id ${chartId}`)
continue
}

if (rows[0].version > current.version) toBake.push(url)
}

if (toBake.length > 0) {
await bakeGraphersToSvgs(
knex,
toBake,
`${BAKED_SITE_DIR}/exports`,
OPTIMIZE_SVG_EXPORTS
)
}
}

export const getGrapherExportsByUrl = async (): Promise<GrapherExports> => {
// Index the files to see what we have available, using the most recent version
// if multiple exports exist
const files = await glob(`${BAKED_SITE_DIR}/exports/*.svg`)
const exportsByKey = new Map<string, ChartExportMeta>()
for (const filepath of files) {
const filename = path.basename(filepath)
const [key, version, dims] = filename.toLowerCase().split("_")
const versionNumber = parseInt(version.split("v")[1])
const [width, height] = dims.split("x")

const current = exportsByKey.get(key)
if (!current || current.version < versionNumber) {
exportsByKey.set(key, {
key: key,
svgUrl: `${BAKED_SITE_EXPORTS_BASE_URL}/${filename}`,
version: versionNumber,
width: parseInt(width),
height: parseInt(height),
})
}
}

return {
get(grapherUrl: string) {
const { slug, queryStr } = grapherUrlToSlugAndQueryStr(grapherUrl)
return exportsByKey.get(
grapherSlugToExportFileKey(slug, queryStr).toLowerCase()
)
},
}
}

/**
* Returns a map that can resolve Tag names and Tag IDs to the Tag's slug
* e.g.
Expand Down
43 changes: 6 additions & 37 deletions baker/SiteBaker.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import fs from "fs-extra"
import path from "path"
import { glob } from "glob"
import { keyBy, without, uniq, mapValues, pick, chunk } from "lodash"
import { keyBy, without, mapValues, pick, chunk } from "lodash"
import ProgressBar from "progress"
import * as db from "../db/db.js"
import {
Expand Down Expand Up @@ -33,11 +33,6 @@ import {
makeDataInsightsAtomFeed,
renderGdocTombstone,
} from "../baker/siteRenderers.js"
import {
bakeGrapherUrls,
getGrapherExportsByUrl,
GrapherExports,
} from "../baker/GrapherBakingUtils.js"
import { makeSitemap } from "../baker/sitemap.js"
import { bakeCountries } from "../baker/countryProfiles.js"
import { bakeDriveImages } from "../baker/GDriveImagesBaker.js"
Expand Down Expand Up @@ -86,14 +81,11 @@ import { GdocPost } from "../db/model/Gdoc/GdocPost.js"
import { Image, getAllImages } from "../db/model/Image.js"
import { generateEmbedSnippet } from "../site/viteUtils.js"
import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js"
import {
getChartEmbedUrlsInPublishedWordpressPosts,
mapSlugsToConfigs,
} from "../db/model/Chart.js"
import { mapSlugsToConfigs } from "../db/model/Chart.js"
import {
BAKED_BASE_URL,
BAKED_GRAPHER_EXPORTS_BASE_URL,
FeatureFlagFeature,
GRAPHER_DYNAMIC_THUMBNAIL_URL,
} from "../settings/clientSettings.js"
import pMap from "p-map"
import { GdocDataInsight } from "../db/model/Gdoc/GdocDataInsight.js"
Expand Down Expand Up @@ -130,7 +122,6 @@ type PrefetchedAttachments = {
const wordpressSteps = [
"assets",
"blogIndex",
"embeds",
"redirects",
"rss",
"wordpressPosts",
Expand Down Expand Up @@ -186,7 +177,6 @@ function getProgressBarTotal(bakeSteps: BakeStepConfig): number {
}

export class SiteBaker {
private grapherExports!: GrapherExports
private bakedSiteDir: string
baseUrl: string
progressBar: ProgressBar
Expand Down Expand Up @@ -221,20 +211,6 @@ export class SiteBaker {
// }
}

private async bakeEmbeds(knex: db.KnexReadonlyTransaction) {
if (!this.bakeSteps.has("embeds")) return

// Find all grapher urls used as embeds in all Wordpress posts on the site
const grapherUrls = uniq(
await getChartEmbedUrlsInPublishedWordpressPosts(knex)
)

await bakeGrapherUrls(knex, grapherUrls)

this.grapherExports = await getGrapherExportsByUrl()
this.progressBar.tick({ name: "✅ baked embeds" })
}

private async bakeCountryProfiles(knex: db.KnexReadonlyTransaction) {
if (!this.bakeSteps.has("countryProfiles")) return
await Promise.all(
Expand All @@ -248,8 +224,7 @@ export class SiteBaker {
const html = await renderCountryProfile(
spec,
country,
knex,
this.grapherExports
knex
).catch(() =>
console.error(
`${country.name} country profile not baked for project "${spec.project}". Check that both pages "${spec.landingPageSlug}" and "${spec.genericProfileSlug}" exist and are published.`
Expand Down Expand Up @@ -296,12 +271,7 @@ export class SiteBaker {

// Bake an individual post/page
private async bakePost(post: FullPost, knex: db.KnexReadonlyTransaction) {
const html = await renderPost(
post,
knex,
this.baseUrl,
this.grapherExports
)
const html = await renderPost(post, knex, this.baseUrl)

const outPath = path.join(this.bakedSiteDir, `${post.slug}.html`)
await fs.mkdirp(path.dirname(outPath))
Expand Down Expand Up @@ -419,7 +389,7 @@ export class SiteBaker {
resolvedUrl: `${BAKED_GRAPHER_URL}/${chart.config.slug}`,
tab,
title: chart.config.title || "",
thumbnail: `${BAKED_GRAPHER_EXPORTS_BASE_URL}/${chart.config.slug}.svg`,
thumbnail: `${GRAPHER_DYNAMIC_THUMBNAIL_URL}/${chart.config.slug}.svg`,
indicatorId: datapageIndicator?.id,
tags: [],
})
Expand Down Expand Up @@ -1113,7 +1083,6 @@ export class SiteBaker {
// somewhere inside it we fetch images.
async bakeWordpressPages(knex: db.KnexReadWriteTransaction) {
await this.bakeRedirects(knex)
await this.bakeEmbeds(knex)
await this.bakeBlogIndex(knex)
await this.bakeRSS(knex)
await this.bakeAssets(knex)
Expand Down
Loading

0 comments on commit 0c68bdb

Please sign in to comment.