Add caching, rss generation, and update screenshot

Author: ijjk

assets/table-view.png

@@ -1,5 +1,18 @@
+const fs = require('fs')
+const path = require('path')
 const { NODE_ENV, NOTION_TOKEN, BLOG_INDEX_ID } = process.env
 
+try {
+  fs.unlinkSync(path.resolve('.blog_index_data'))
+} catch (_) {
+  /* non fatal */
+}
+try {
+  fs.unlinkSync(path.resolve('.blog_index_data_previews'))
+} catch (_) {
+  /* non fatal */
+}
+
 const warnOrError =
   NODE_ENV !== 'production'
     ? console.warn
@@ -26,7 +39,22 @@ if (!BLOG_INDEX_ID) {
 }
 
 module.exports = {
+  target: 'experimental-serverless-trace',
+
   experimental: {
     css: true,
   },
+
+  webpack(cfg, { dev, isServer }) {
+    // only compile build-rss in production server build
+    if (dev || !isServer) return cfg
+
+    const originalEntry = cfg.entry
+    cfg.entry = async () => {
+      const entries = { ...(await originalEntry()) }
+      entries['./scripts/build-rss.js'] = './src/lib/build-rss.ts'
+      return entries
+    }
+    return cfg
+  },
 }

next.config.js

@@ -4,7 +4,7 @@
   "scripts": {
     "dev": "next dev",
     "start": "next start",
-    "build": "next build",
+    "build": "next build && node .next/serverless/scripts/build-rss.js",
     "format": "prettier --write \"**/*.{js,jsx,json,ts,tsx,md,mdx,css,html,yml,yaml,scss,sass}\" --ignore-path .gitignore"
   },
   "husky": {

package.json

@@ -9,3 +9,7 @@ export const getDateStr = date => {
     year: 'numeric',
   })
 }
+
+export const postIsReady = (post: any) => {
+  return process.env.NODE_ENV !== 'production' || post.Published === 'Yes'
+}

src/lib/blog-helpers.ts

@@ -0,0 +1,104 @@
+import { resolve } from 'path'
+import { writeFile } from './fs-helpers'
+import { renderToStaticMarkup } from 'react-dom/server'
+
+import { textBlock } from './notion/renderers'
+import getBlogIndex from './notion/getBlogIndex'
+import getNotionUsers from './notion/getNotionUsers'
+import { postIsReady, getBlogLink } from './blog-helpers'
+
+// must use weird syntax to bypass auto replacing of NODE_ENV
+process.env['NODE' + '_ENV'] = 'production'
+
+// constants
+const NOW = new Date().toJSON()
+
+function mapToAuthor(author) {
+  return `<author><name>${author.full_name}</name></author>`
+}
+
+function decode(string) {
+  return string
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;')
+}
+
+function mapToEntry(post) {
+  return `
+    <entry>
+      <id>${post.link}</id>
+      <title>${decode(post.title)}</title>
+      <link href="${post.link}"/>
+      <updated>${new Date(post.date).toJSON()}</updated>
+      <content type="xhtml">
+        <div xmlns="http://www.w3.org/1999/xhtml">
+          ${renderToStaticMarkup(
+            post.preview
+              ? (post.preview || []).map((block, idx) =>
+                  textBlock(block, false, post.title + idx)
+                )
+              : post.content
+          )}
+          <p class="more">
+            <a href="${post.link}">Read more</a>
+          </p>
+        </div>
+      </content>
+      ${(post.authors || []).map(mapToAuthor).join('\n      ')}
+    </entry>`
+}
+
+function concat(total, item) {
+  return total + item
+}
+
+function createRSS(blogPosts = []) {
+  const postsString = blogPosts.map(mapToEntry).reduce(concat, '')
+
+  return `<?xml version="1.0" encoding="utf-8"?>
+  <feed xmlns="http://www.w3.org/2005/Atom">
+    <title>My Blog</title>
+    <subtitle>Blog</subtitle>
+    <link href="/atom" rel="self" type="application/rss+xml"/>
+    <link href="/" />
+    <updated>${NOW}</updated>
+    <id>My Notion Blog</id>${postsString}
+  </feed>`
+}
+
+async function main() {
+  const postsTable = await getBlogIndex(true)
+  const neededAuthors = new Set<string>()
+
+  const blogPosts = Object.keys(postsTable)
+    .map(slug => {
+      const post = postsTable[slug]
+      if (!postIsReady(post)) return
+
+      post.authors = post.Authors || []
+
+      for (const author of post.authors) {
+        neededAuthors.add(author)
+      }
+      return post
+    })
+    .filter(Boolean)
+
+  const { users } = await getNotionUsers([...neededAuthors])
+
+  blogPosts.forEach(post => {
+    post.authors = post.authors.map(id => users[id])
+    post.link = getBlogLink(post.Slug)
+    post.title = post.Page
+    post.date = post.Date
+  })
+
+  const outputPath = './public/atom'
+  await writeFile(resolve(outputPath), createRSS(blogPosts))
+  console.log(`Atom feed file generated at \`${outputPath}\``)
+}
+
+main().catch(error => console.error(error))

src/lib/build-rss.ts

@@ -0,0 +1,5 @@
+import fs from 'fs'
+import { promisify } from 'util'
+
+export const readFile = promisify(fs.readFile)
+export const writeFile = promisify(fs.writeFile)

src/lib/fs-helpers.ts

@@ -2,45 +2,66 @@ import { Sema } from 'async-sema'
 import rpc, { values } from './rpc'
 import getTableData from './getTableData'
 import { getPostPreview } from './getPostPreview'
-import { BLOG_INDEX_ID } from './server-constants'
+import { readFile, writeFile } from '../fs-helpers'
+import { BLOG_INDEX_ID, BLOG_INDEX_CACHE } from './server-constants'
 
 export default async function getBlogIndex(previews = true) {
-  const data = await rpc('loadPageChunk', {
-    pageId: BLOG_INDEX_ID,
-    limit: 999, // TODO: figure out Notion's way of handling pagination
-    cursor: { stack: [] },
-    chunkNumber: 0,
-    verticalColumns: false,
-  })
-
-  // Parse table with posts
-  const tableBlock = values(data.recordMap.block).find(
-    (block: any) => block.value.type === 'collection_view'
-  )
-
-  const postsTable = await getTableData(tableBlock, true)
-  const postsKeys = Object.keys(postsTable)
-  const sema = new Sema(3, { capacity: postsKeys.length })
-
-  if (previews) {
-    await Promise.all(
-      postsKeys
-        .sort((a, b) => {
-          const postA = postsTable[a]
-          const postB = postsTable[b]
-          const timeA = postA.Date
-          const timeB = postB.Date
-          return Math.sign(timeB - timeA)
-        })
-        .map(async postKey => {
-          await sema.acquire()
-          const post = postsTable[postKey]
-          post.preview = post.id
-            ? await getPostPreview(postsTable[postKey].id)
-            : []
-          sema.release()
-        })
+  let postsTable: any = null
+  const isProd = process.env.NODE_ENV === 'production'
+  const cacheFile = `${BLOG_INDEX_CACHE}${previews ? '_previews' : ''}`
+
+  if (isProd) {
+    try {
+      postsTable = JSON.parse(await readFile(cacheFile, 'utf8'))
+    } catch (_) {
+      /* not fatal */
+    }
+  }
+
+  if (!postsTable) {
+    const data = await rpc('loadPageChunk', {
+      pageId: BLOG_INDEX_ID,
+      limit: 999, // TODO: figure out Notion's way of handling pagination
+      cursor: { stack: [] },
+      chunkNumber: 0,
+      verticalColumns: false,
+    })
+
+    // Parse table with posts
+    const tableBlock = values(data.recordMap.block).find(
+      (block: any) => block.value.type === 'collection_view'
     )
+
+    postsTable = await getTableData(tableBlock, true)
+    // only get 10 most recent post's previews
+    const postsKeys = Object.keys(postsTable).splice(0, 10)
+
+    const sema = new Sema(3, { capacity: postsKeys.length })
+
+    if (previews) {
+      await Promise.all(
+        postsKeys
+          .sort((a, b) => {
+            const postA = postsTable[a]
+            const postB = postsTable[b]
+            const timeA = postA.Date
+            const timeB = postB.Date
+            return Math.sign(timeB - timeA)
+          })
+          .map(async postKey => {
+            await sema.acquire()
+            const post = postsTable[postKey]
+            post.preview = post.id
+              ? await getPostPreview(postsTable[postKey].id)
+              : []
+            sema.release()
+          })
+      )
+    }
+
+    if (isProd) {
+      writeFile(cacheFile, JSON.stringify(postsTable), 'utf8').catch(() => {})
+    }
   }
 
   return postsTable

src/lib/notion/getBlogIndex.ts

@@ -11,8 +11,6 @@ export default function queryCollection({
     loadContentCover = true,
     type = 'table',
     userLocale = 'en',
-    // we use America/Phoenix since it doesn't do daylight savings and
-    // we can't use UTC here
     userTimeZone = 'America/Phoenix',
   } = loader
 

src/lib/notion/queryCollection.ts

@@ -1,3 +1,6 @@
+import path from 'path'
+
 export const NOTION_TOKEN = process.env.NOTION_TOKEN
 export const BLOG_INDEX_ID = process.env.BLOG_INDEX_ID
 export const API_ENDPOINT = 'https://www.notion.so/api/v3'
+export const BLOG_INDEX_CACHE = path.resolve('.blog_index_data')

src/lib/notion/server-constants.ts

@@ -4,7 +4,7 @@ import Header from '../../components/header'
 import blogStyles from '../../styles/blog.module.css'
 import sharedStyles from '../../styles/shared.module.css'
 
-import { getBlogLink, getDateStr } from '../../lib/blog-helpers'
+import { getBlogLink, getDateStr, postIsReady } from '../../lib/blog-helpers'
 import { textBlock } from '../../lib/notion/renderers'
 import getNotionUsers from '../../lib/notion/getNotionUsers'
 import getBlogIndex from '../../lib/notion/getBlogIndex'
@@ -17,7 +17,7 @@ export async function unstable_getStaticProps() {
     .map(slug => {
       const post = postsTable[slug]
       // remove draft posts in production
-      if (process.env.NODE_ENV === 'production' && post.Published !== 'Yes') {
+      if (!postIsReady(post)) {
         return null
       }
       post.Authors = post.Authors || []
@@ -61,7 +61,7 @@ export default ({ posts = [] }) => {
               <div className="authors">By: {post.Authors.join(' ')}</div>
               <div className="posted">Posted: {getDateStr(post.Date)}</div>
               <p>
-                {post.preview.map((block, idx) =>
+                {(post.preview || []).map((block, idx) =>
                   textBlock(block, true, `${post.Slug}${idx}`)
                 )}
               </p>