feat: sitemap from app directory

package.json

@@ -7,8 +7,7 @@
   "packageManager": "[email protected]",
   "scripts": {
     "analyze": "ANALYZE=true next build",
-    "build": "next build && next-sitemap",
-    "postbuild": "next-sitemap",
+    "build": "next build",
     "dev": "NODE_OPTIONS='--inspect' next dev --turbo",
     "email": "email dev --dir ./src/utils/email/templates",
     "format": "prettier --write \"**/*.{js,jsx,ts,tsx,json,md}\"",
@@ -99,7 +98,6 @@
     "eslint-plugin-react-hooks": "^5.0.0",
     "github-slugger": "^2.0.0",
     "jiti": "^2.4.0",
-    "next-sitemap": "^4.2.3",
     "prettier": "^3.3.3",
     "prettier-plugin-tailwindcss": "^0.6.9",
     "schema-dts": "^1.1.2",

src/app/robots.ts

@@ -0,0 +1,32 @@
+import type { MetadataRoute } from 'next';
+
+import { BASE_SITE_URL } from '@/config';
+
+const noIndexPaths = [
+  '/ingest', // posthog's reverse proxy
+  '/ingest/*', // posthog's reverse proxy
+];
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [
+      {
+        userAgent: '*',
+        disallow: '/api/',
+      },
+      {
+        userAgent: '*',
+        disallow: '/_next/',
+      },
+      {
+        userAgent: '*',
+        disallow: '/public/',
+      },
+      ...noIndexPaths.map((path) => ({
+        userAgent: '*',
+        disallow: path,
+      })),
+    ],
+    sitemap: `${BASE_SITE_URL}/sitemap.xml`,
+  };
+}

src/app/sitemap.ts

@@ -0,0 +1,63 @@
+import type { MetadataRoute } from 'next';
+import { getAllPosts } from '@lib/blog';
+import { getAllNewsletters } from '@lib/newsletters';
+import { getAllTags } from '@lib/tags';
+
+import { BASE_SITE_URL } from '@/config';
+
+export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
+  // Define your base URL
+  const baseUrl = BASE_SITE_URL;
+
+  const newsletters = await getAllNewsletters();
+  const newslettersSitemap = newsletters.map((newsletter) => ({
+    url: `${baseUrl}/newsletter/${newsletter.slug}`,
+    lastModified: new Date(newsletter.frontmatter.date),
+    changeFrequency: 'weekly' as const,
+    priority: 0.8,
+  }));
+
+  const posts = await getAllPosts();
+  const postsSitemap = posts.map((post) => ({
+    url: `${baseUrl}/blog/${post.slug}`,
+    lastModified: new Date(post.frontmatter.date),
+    changeFrequency: 'weekly' as const,
+    priority: 0.8,
+  }));
+
+  const { allTags } = await getAllTags();
+  const tagsSitemap = allTags.map((tag) => ({
+    url: `${baseUrl}/tags/${tag}`,
+    lastModified: new Date(),
+    changeFrequency: 'weekly' as const,
+    priority: 0.5,
+  }));
+
+  // Define your static routes
+  const routes: string[] = [
+    '', // home page
+    '/about',
+    '/integrity',
+    '/newsletter',
+    '/podcast',
+    '/posts',
+    '/tags',
+    '/work',
+    '/shop',
+  ];
+
+  // Create sitemap entries for static routes
+  const staticRoutesSitemap = routes.map((route) => ({
+    url: `${baseUrl}${route}`,
+    lastModified: new Date(),
+    changeFrequency: 'weekly' as const,
+    priority: route === '' ? 1 : 0.8,
+  }));
+
+  return [
+    ...staticRoutesSitemap,
+    ...newslettersSitemap,
+    ...postsSitemap,
+    ...tagsSitemap,
+  ];
+}

src/config.ts

@@ -1,4 +1,4 @@
-const BASE_SITE_URL = 'https://mikebifulco.com';
+export const BASE_SITE_URL = 'https://mikebifulco.com';
 
 const config = {
   employer: {