Automatic robot.txt & sitemap.xml (#382)

* Create robots.txt

* Delete abomination

* Dynamical robots.txt but bug with sitemap.tsx

* sitemap.xml is automatically created but with scss paths

* only .tsx in sitemap

* url without /page

* changeFrequency = weekly

* Solving error

* It "works" but can't find tournament ids

* comment problem

* Why there is no tournaments??

* Sitemap works but tournamentId must be set by hand

* pnpm lint

* fix some errors

---------

Co-authored-by: Antoine Dufils <[email protected]>

src/app/robots.tsx

@@ -0,0 +1,21 @@
+import { MetadataRoute } from 'next';
+
+export default function robots(): MetadataRoute.Robots {
+  if (process.env.NODE_ENV === 'production') {
+    return {
+      rules: {
+        userAgent: '*',
+        disallow: ['/dashboard/', '/admin/', '/uploads/'],
+      },
+      sitemap: `${process.env.NEXT_PUBLIC_URL}/sitemap.xml`,
+    };
+  } else {
+    return {
+      rules: {
+        userAgent: '*',
+        disallow: '/',
+      },
+      sitemap: `${process.env.NEXT_PUBLIC_URL}/sitemap.xml`,
+    };
+  }
+}

src/app/sitemap.tsx

@@ -0,0 +1,89 @@
+import { MetadataRoute } from 'next';
+import fs from 'fs';
+import path from 'path';
+
+export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
+  //const tournamentId = await tournamentsId();
+
+  //console.log(tournamentId);
+
+  const tournamentId = ['lol', 'cs2', 'osu', 'ssbu', 'tft', 'open'];
+
+  // Path to the directory containing your TSX files
+  const siteDirectory = path.join(process.cwd(), '/src/app');
+
+  // Retrieve all file paths recursively
+  const filePaths = getAllFilePaths(siteDirectory);
+
+  const absoluteAppPaths = path.resolve('./src');
+
+  const allowedFilePaths = filePaths.filter((filePath) => {
+    const relativeFilePaths = path.relative(absoluteAppPaths, filePath);
+    if (
+      relativeFilePaths.slice(3, 15) !== '/(dashboard)' &&
+      relativeFilePaths.slice(3, 9) !== '/oauth' &&
+      relativeFilePaths.slice(3, 9) !== '/reset' &&
+      relativeFilePaths.slice(3, 12) !== '/validate' &&
+      relativeFilePaths.slice(3, 12) !== '/page.tsx'
+    ) {
+      return filePath;
+    }
+  });
+
+  // Select only .tsx files
+  const tsxFilePaths = allowedFilePaths.filter((filePath) => {
+    if (filePath.slice(-8) === 'page.tsx' && filePath.indexOf('[') === -1) {
+      return filePath;
+    }
+  });
+
+  // Generate URLs and add them to the sitemap
+  const sitemap = tsxFilePaths.map((filePath) => {
+    const category = path.basename(path.dirname(filePath));
+    const url = `${process.env.NEXT_PUBLIC_URL}/${category}`;
+    const changeFrequency = 'weekly' as const;
+    const lastModified = fs.statSync(filePath).mtime;
+    const priority = 0.5;
+    return {
+      url,
+      lastModified,
+      changeFrequency,
+      priority,
+    };
+  });
+
+  // Add other URLs to the sitemap
+  for (const id of tournamentId) {
+    sitemap.push({
+      url: `${process.env.NEXT_PUBLIC_URL}/tournament/${id}`,
+      lastModified: new Date(),
+      changeFrequency: 'weekly' as const,
+      priority: 0.8,
+    });
+  }
+
+  sitemap.push({
+    url: `${process.env.NEXT_PUBLIC_URL}`,
+    lastModified: new Date(),
+    changeFrequency: 'weekly' as const,
+    priority: 1.0,
+  });
+
+  return sitemap;
+}
+
+// Recursively retrieve all file paths
+function getAllFilePaths(directory: string): string[] {
+  const fileNames = fs.readdirSync(directory);
+  const filePaths = fileNames.map((fileName) => {
+    const filePath = path.join(directory, fileName);
+    const stat = fs.statSync(filePath);
+    if (stat.isDirectory()) {
+      return getAllFilePaths(filePath);
+    } else {
+      return filePath;
+    }
+  });
+
+  return Array.prototype.concat(...filePaths);
+}

src/modules/tournament.ts

@@ -46,6 +46,14 @@ export const fetchTournaments = (): AppThunk => async (dispatch) => {
   dispatch(setTournaments(res));
 };
 
+export const tournamentsId = async (): Promise<string[]> => {
+  const tournaments: Tournament[] = await API.get('tournaments');
+  const tournamentsId = tournaments.map((tournament) => {
+    return tournament.id;
+  });
+  return tournamentsId;
+};
+
 export const fetchSlots = (): AppThunk => async (dispatch) => {
   const res = await API.get('tournaments?paidOnly=true');
   const slots = res.reduce(