🔨 switch from R2 bindings to using fetch against a public bucket

owid · Aug 15, 2024 · 6474c78 · 6474c78
1 parent bf368a0
commit 6474c78
Show file tree

Hide file tree

Showing 6 changed files with 76 additions and 33 deletions.
diff --git a/.dev.vars.example b/.dev.vars.example
@@ -20,3 +20,5 @@ MAILGUN_SENDING_KEY=
 # optional
 SLACK_BOT_OAUTH_TOKEN=
 SLACK_ERROR_CHANNEL_ID=C016H0BNNB1 #bot-testing channel
+
+GRAPHER_CONFIG_R2_BUCKET_PATH=devs/YOURNAME
diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts
@@ -135,17 +135,36 @@ const extractOptions = (params: URLSearchParams): ImageOptions => {
     return options as ImageOptions
 }
 
-async function fetchAndRenderGrapherToSvg({
-    slug,
-    options,
-    searchParams,
-    env,
-}: {
-    slug: string
-    options: ImageOptions
-    searchParams: URLSearchParams
-    env: Env
-}) {
+const WORKER_CACHE_TIME_IN_SECONDS = 60
+
+async function fetchFromR2(
+    url: URL,
+    etag: string | undefined,
+    fallbackUrl?: URL
+) {
+    const headers = new Headers()
+    if (etag) headers.set("If-None-Match", etag)
+    const init = {
+        cf: {
+            cacheEverything: true,
+            cacheTtl: WORKER_CACHE_TIME_IN_SECONDS,
+        },
+        headers,
+    }
+    const primaryResponse = await fetch(url.toString(), init)
+    if (primaryResponse.status === 404 && fallbackUrl) {
+        return fetch(fallbackUrl.toString(), init)
+    }
+    return primaryResponse
+}
+
+async function fetchAndRenderGrapherToSvg(
+    slug: string,
+    options: ImageOptions,
+    searchParams: URLSearchParams,
+    env: Env,
+    etag?: string
+) {
     const grapherLogger = new TimeLogger("grapher")
 
     const url = new URL(`/grapher/${slug}`, env.url)
@@ -166,10 +185,31 @@ async function fetchAndRenderGrapherToSvg({
 
     console.log("fetching grapher config from this key", key)
 
+    const requestUrl = new URL(key, env.GRAPHER_CONFIG_R2_BUCKET_URL)
+
+    let fallbackUrl
+
+    if (
+        env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL &&
+        env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH
+    ) {
+        const topLevelDirectory = env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH
+        const fallbackKey = excludeUndefined([
+            ...topLevelDirectory,
+            R2GrapherConfigDirectory.publishedGrapherBySlug,
+            `${slugOnly}.json`,
+        ]).join("/")
+        fallbackUrl = new URL(
+            fallbackKey,
+            env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL
+        )
+    }
+
     // Fetch grapher config
-    const fetchResponse = await env.r2ChartConfigs.get(key)
+    const fetchResponse = await fetchFromR2(requestUrl, etag, fallbackUrl)
 
-    if (!fetchResponse) {
+    if (fetchResponse.status !== 200) {
+        console.log("Failed to fetch grapher config", fetchResponse.status)
         return null
     }
 
@@ -213,17 +253,20 @@ export const fetchAndRenderGrapher = async (
     slug: string,
     searchParams: URLSearchParams,
     outType: "png" | "svg",
-    env: Env
+    env: Env,
+    etag?: string
 ) => {
     const options = extractOptions(searchParams)
 
     console.log("Rendering", slug, outType, options)
-    const svg = await fetchAndRenderGrapherToSvg({
+    const svg = await fetchAndRenderGrapherToSvg(
         slug,
         options,
         searchParams,
         env,
-    })
+        etag
+    )
+    console.log("fetched svg")
 
     if (!svg) {
         return new Response("Not found", { status: 404 })

diff --git a/functions/grapher/thumbnail/[slug].ts b/functions/grapher/thumbnail/[slug].ts
@@ -5,11 +5,11 @@ export interface Env {
     ASSETS: {
         fetch: typeof fetch
     }
-    r2ChartConfigs: {
-        get: (url: string) => Promise<R2ObjectBody>
-    }
     url: URL
+    GRAPHER_CONFIG_R2_BUCKET_URL: string
+    GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL: string
     GRAPHER_CONFIG_R2_BUCKET_PATH: string
+    GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH: string
     CF_PAGES_BRANCH: string
     ENV: string
 }

diff --git a/package.json b/package.json
@@ -26,7 +26,7 @@
         "revertLastDbMigration": "tsx --tsconfig tsconfig.tsx.json node_modules/typeorm/cli.js migration:revert -d db/dataSource.ts",
         "startAdminServer": "node --enable-source-maps ./itsJustJavascript/adminSiteServer/app.js",
         "startAdminDevServer": "tsx watch --ignore '**.mjs' --tsconfig tsconfig.tsx.json adminSiteServer/app.tsx",
-        "startLocalCloudflareFunctions": "wrangler pages dev --local --persist-to ./cfstorage",
+        "startLocalCloudflareFunctions": "wrangler pages dev",
         "startDeployQueueServer": "node --enable-source-maps ./itsJustJavascript/baker/startDeployQueueServer.js",
         "startLernaWatcher": "lerna watch --scope '@ourworldindata/*' -- lerna run build --scope=\\$LERNA_PACKAGE_NAME --include-dependents",
         "startTmuxServer": "node_modules/tmex/tmex dev \"yarn startLernaWatcher\" \"yarn startAdminDevServer\" \"yarn startViteServer\"",

diff --git a/packages/@ourworldindata/types/src/domainTypes/Various.ts b/packages/@ourworldindata/types/src/domainTypes/Various.ts
@@ -68,5 +68,5 @@ export interface QueryParams {
 
 export enum R2GrapherConfigDirectory {
     byUUID = "config/by-uuid",
-    publishedGrapherBySlug = "grapher/by-slug",
+    publishedGrapherBySlug = "config/by-slug-published",
 }
diff --git a/wrangler.toml b/wrangler.toml
@@ -6,33 +6,31 @@ pages_build_output_dir = "./localBake"
 # Vars that should be available in all envs, including local dev
 [vars]
 ENV = "development"
+GRAPHER_CONFIG_R2_BUCKET_URL = "https://grapher-configs-staging.ourworldindata.org"
+GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL = "https://grapher-configs.ourworldindata.org"
+GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH = "v1"
+
 
 # Overrides for CF preview deployments
 [env.preview.vars]
 MAILGUN_DOMAIN = "mg.ourworldindata.org"
 SLACK_ERROR_CHANNEL_ID = "C016H0BNNB1"
 ENV = "preview"
-
-[[r2_buckets]]
-binding = "r2ChartConfigs"
-bucket_name = "owid-grapher-configs-staging"
+GRAPHER_CONFIG_R2_BUCKET_URL = "https://grapher-configs-staging.ourworldindata.org"
+GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL = "https://grapher-configs.ourworldindata.org"
+GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH = "v1"
 
 # Overrides for CF production deployment
 [env.production]
 compatibility_date = "2024-04-29"
 
-[[env.production.r2_buckets]]
-binding = "r2ChartConfigs"
-bucket_name = "owid-grapher-configs"
-
 [env.production.vars]
 ENV = "production"
 MAILGUN_DOMAIN = "mg.ourworldindata.org"
 SLACK_ERROR_CHANNEL_ID = "C5JJW19PS"
+GRAPHER_CONFIG_R2_BUCKET_URL = "https://grapher-configs.ourworldindata.org"
+GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL = ""
+GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH = ""
 GRAPHER_CONFIG_R2_BUCKET_PATH = "v1"
 
 
-[[env.preview.r2_buckets]]
-binding = "r2ChartConfigs"
-bucket_name = "owid-grapher-configs-staging"
-