Reduced JPGs, implemented robots & added yarn sitemap

Integrated @astrojs/sitemap with a filter to exclude pages with query parameters. Added robots meta tag to blog posts to prevent indexing, subsequently updating the Layout.astro to support optional robots meta tag configuration. Finally, I compressed some blog post images to ensure they are not too large when turned into webp's.

Author: Yensid10

astro.config.ts

@@ -1,19 +1,26 @@
 import mdx from '@astrojs/mdx'
 import preact from '@astrojs/preact'
+import sitemap from '@astrojs/sitemap'
 import { defineConfig } from 'astro/config'
 import { h } from 'hastscript'
 import rehypeAutolinkHeadings from 'rehype-autolink-headings'
 import rehypeSlug from 'rehype-slug'
 import rehypeKatex from 'rehype-katex'
 import remarkMath from 'remark-math'
 import remarkGfm from 'remark-gfm'
-import inspectUrls from "@jsdevtools/rehype-url-inspector";
+import inspectUrls from '@jsdevtools/rehype-url-inspector'
 
 // https://astro.build/config
 export default defineConfig({
   integrations: [
     mdx(),
     preact(),
+    sitemap({
+      filter: (page) => {
+        // Exclude any pages we don't want in the sitemap, like search pages
+        return !page.includes('?')
+      }
+    })
   ],
   site: 'https://juxt.pro/',
   markdown: {
@@ -58,18 +65,18 @@ export default defineConfig({
       [
         inspectUrls,
         {
-          selectors: ["a[href]"],
+          selectors: ['a[href]'],
           inspectEach: (url) => {
-              // Ignore hash links
-              if (url.node.properties.href.startsWith("#")) {
-                  return;
-              }
-              var href = new URL(url.node.properties.href, "https://juxt.pro/");
-              // Add target blank to external links
-              if (href.host !== "juxt.pro") {
-                  url.node.properties.target = "_blank";
-                  url.node.properties.rel = "noopener noreferrer";
-              }
+            // Ignore hash links
+            if (url.node.properties.href.startsWith('#')) {
+              return
+            }
+            var href = new URL(url.node.properties.href, 'https://juxt.pro/')
+            // Add target blank to external links
+            if (href.host !== 'juxt.pro') {
+              url.node.properties.target = '_blank'
+              url.node.properties.rel = 'noopener noreferrer'
+            }
           }
         }
       ]
@@ -78,7 +85,7 @@ export default defineConfig({
   vite: {
     build: {
       rollupOptions: {
-        external: ['/pagefind/pagefind.js'],
+        external: ['/pagefind/pagefind.js']
       }
     }
   }

package.json

@@ -46,6 +46,7 @@
   "dependencies": {
     "@astrojs/check": "^0.2.0",
     "@astrojs/rss": "^3.0.0",
+    "@astrojs/sitemap": "^3.2.1",
     "@jsdevtools/rehype-url-inspector": "^2.0.2",
     "@preact/compat": "npm:@preact/compat",
     "@tailwindui/react": "^0.1.1",

src/assets/blog/kim-johannessen-xt24.jpg

@@ -186,6 +186,7 @@ const categoryHref = `/blog?${encodeURI(
   ogTitle={`JUXT Blog: ${title}`}
   ogDescription={description}
   ogImage={heroImageFilename}
+  robots='noindex,follow'
 >
   <DraftBanner draft={draft} pageName='Blog' />
   <main

src/assets/blog/oliver-xt24.jpg

@@ -13,6 +13,7 @@ type LayoutProps = {
   ogTitle?: string
   ogDescription?: string
   noFooter?: boolean
+  robots?: string
 }
 
 const url = new URL(Astro.request.url)
@@ -29,8 +30,16 @@ type Metadata = {
   seo: Record<string, { description: string; image?: string }>
 }
 
-const { title, navbar, navbarNoBg, ogImage, ogTitle, ogDescription, noFooter } =
-  Astro.props as LayoutProps
+const {
+  title,
+  navbar,
+  navbarNoBg,
+  ogImage,
+  ogTitle,
+  ogDescription,
+  noFooter,
+  robots
+} = Astro.props as LayoutProps
 
 const [{ seo }] = await Astro.glob<Metadata>('../data/metadata.json')
 
@@ -188,6 +197,7 @@ const metaTitle = ogTitle ?? title ?? currentPageName
       defer
       data-website-id='4a088aa2-1e6f-4f97-9e95-9619983468de'
       src='https://bunseki.juxt.pro/t.js'></script>
+    {robots && <meta name='robots' content={robots} />}
   </head>
   <body class='bg-gray-50 transition-colors'>
     {navbar && <Navbar navbarNoBg={navbarNoBg} />}

src/layouts/BlogPost.astro

@@ -143,6 +143,15 @@
     fast-xml-parser "^4.2.7"
     kleur "^4.1.5"
 
+"@astrojs/sitemap@^3.2.1":
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/@astrojs/sitemap/-/sitemap-3.2.1.tgz#ed3874861fbca83f9ca3e66ac24a0f7ae3f9cf49"
+  integrity sha512-uxMfO8f7pALq0ADL6Lk68UV6dNYjJ2xGUzyjjVj60JLBs5a6smtlkBYv3tQ0DzoqwS7c9n4FUx5lgv0yPo/fgA==
+  dependencies:
+    sitemap "^8.0.0"
+    stream-replace-string "^2.0.0"
+    zod "^3.23.8"
+
 "@astrojs/telemetry@3.0.3":
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/@astrojs/telemetry/-/telemetry-3.0.3.tgz#a7a87a40de74bfeaae78fc4cbec1f6ec1cbf1c36"
@@ -790,7 +799,7 @@
     picocolors "^1.0.0"
     tslib "^2.6.0"
 
-"@preact/compat@npm:@preact/compat", "react-dom@npm:@preact/compat", "react@npm:@preact/compat":
+"@preact/compat@npm:@preact/compat":
   version "17.1.2"
   resolved "https://registry.yarnpkg.com/@preact/compat/-/compat-17.1.2.tgz#928756713b07af6faf7812f6a56840d8ce6fed37"
   integrity sha512-7pOZN9lMDDRQ+6aWvjwTp483KR8/zOpfS83wmOo3zfuLKdngS8/5RLbsFWzFZMGdYlotAhX980hJ75bjOHTwWg==
@@ -1016,6 +1025,11 @@
   dependencies:
     undici-types "~5.25.1"
 
+"@types/node@^17.0.5":
+  version "17.0.45"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190"
+  integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==
+
 "@types/node@^18.16.3":
   version "18.18.6"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.18.6.tgz#26da694f75cdb057750f49d099da5e3f3824cb3e"
@@ -1031,6 +1045,13 @@
   resolved "https://registry.yarnpkg.com/@types/ps-tree/-/ps-tree-1.1.4.tgz#762883fb62e03080ced3fc15e702321968bf0009"
   integrity sha512-CJyu2BqU/aZN/s8Ili3jiMctqXfTjCaWXirEcjRD8y1lUQZJ8eNohnal8+LXeWFs1VbdAOrCIdgATFsv+lnQ5Q==
 
+"@types/sax@^1.2.1":
+  version "1.2.7"
+  resolved "https://registry.yarnpkg.com/@types/sax/-/sax-1.2.7.tgz#ba5fe7df9aa9c89b6dff7688a19023dd2963091d"
+  integrity sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==
+  dependencies:
+    "@types/node" "*"
+
 "@types/semver@^7.5.0":
   version "7.5.4"
   resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.4.tgz#0a41252ad431c473158b22f9bfb9a63df7541cff"
@@ -1317,7 +1338,7 @@ anymatch@~3.1.2:
     normalize-path "^3.0.0"
     picomatch "^2.0.4"
 
-arg@^5.0.2:
+arg@^5.0.0, arg@^5.0.2:
   version "5.0.2"
   resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c"
   integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==
@@ -5142,11 +5163,21 @@ rc@^1.2.7:
     minimist "^1.2.0"
     strip-json-comments "~2.0.1"
 
+"react-dom@npm:@preact/compat":
+  version "17.1.2"
+  resolved "https://registry.yarnpkg.com/@preact/compat/-/compat-17.1.2.tgz#928756713b07af6faf7812f6a56840d8ce6fed37"
+  integrity sha512-7pOZN9lMDDRQ+6aWvjwTp483KR8/zOpfS83wmOo3zfuLKdngS8/5RLbsFWzFZMGdYlotAhX980hJ75bjOHTwWg==
+
 react-hook-form@^7.43.1:
   version "7.51.1"
   resolved "https://registry.yarnpkg.com/react-hook-form/-/react-hook-form-7.51.1.tgz#3ce5f8b5ef41903b4054a641cef8c0dc8bf8ae85"
   integrity sha512-ifnBjl+kW0ksINHd+8C/Gp6a4eZOdWyvRv0UBaByShwU8JbVx5hTcTWEcd5VdybvmPTATkVVXk9npXArHmo56w==
 
+"react@npm:@preact/compat":
+  version "17.1.2"
+  resolved "https://registry.yarnpkg.com/@preact/compat/-/compat-17.1.2.tgz#928756713b07af6faf7812f6a56840d8ce6fed37"
+  integrity sha512-7pOZN9lMDDRQ+6aWvjwTp483KR8/zOpfS83wmOo3zfuLKdngS8/5RLbsFWzFZMGdYlotAhX980hJ75bjOHTwWg==
+
 read-cache@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/read-cache/-/read-cache-1.0.0.tgz#e664ef31161166c9751cdbe8dbcf86b5fb58f774"
@@ -5612,6 +5643,16 @@ sisteransi@^1.0.5:
   resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed"
   integrity sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==
 
+sitemap@^8.0.0:
+  version "8.0.0"
+  resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-8.0.0.tgz#eb6ea48f95787cd680b83683c555d6f6b5a903fd"
+  integrity sha512-+AbdxhM9kJsHtruUF39bwS/B0Fytw6Fr1o4ZAIAEqA6cke2xcoO2GleBw9Zw7nRzILVEgz7zBM5GiTJjie1G9A==
+  dependencies:
+    "@types/node" "^17.0.5"
+    "@types/sax" "^1.2.1"
+    arg "^5.0.0"
+    sax "^1.2.4"
+
 slash@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
@@ -5683,6 +5724,11 @@ stream-parser@~0.3.1:
   dependencies:
     debug "2"
 
+stream-replace-string@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/stream-replace-string/-/stream-replace-string-2.0.0.tgz#e49fd584bd1c633613e010bc73b9db49cb5024ad"
+  integrity sha512-TlnjJ1C0QrmxRNrON00JvaFFlNh5TTG00APw23j74ET7gkQpTASi6/L2fuiav8pzK715HXtUeClpBTw2NPSn6w==
+
 streamx@^2.15.0:
   version "2.15.1"
   resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.15.1.tgz#396ad286d8bc3eeef8f5cea3f029e81237c024c6"
@@ -6680,6 +6726,11 @@ zod@3.21.1:
   resolved "https://registry.yarnpkg.com/zod/-/zod-3.21.1.tgz#ac5bb7cf68876281ebd02f95ac4bb9a080370282"
   integrity sha512-+dTu2m6gmCbO9Ahm4ZBDapx2O6ZY9QSPXst2WXjcznPMwf2YNpn3RevLx4KkZp1OPW/ouFcoBtBzFz/LeY69oA==
 
+zod@^3.23.8:
+  version "3.24.2"
+  resolved "https://registry.yarnpkg.com/zod/-/zod-3.24.2.tgz#8efa74126287c675e92f46871cfc8d15c34372b3"
+  integrity sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==
+
 zwitch@^2.0.0, zwitch@^2.0.4:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-2.0.4.tgz#c827d4b0acb76fc3e685a4c6ec2902d51070e9d7"