Merge pull request #83 from NixOS/joerg-ci

add robots.txt

Author: Mic92

modules/nixos-wiki/default.nix

@@ -161,6 +161,7 @@ in
       forceSSL = lib.mkDefault true;
       locations."=/nixos.png".alias = ./nixos.png;
       locations."=/favicon.ico".alias = ./favicon.ico;
+      locations."=/robots.txt".alias = ./robots.txt;
     };
   };
 }

modules/nixos-wiki/robots.txt

@@ -0,0 +1,20 @@
+User-agent: *
+Disallow: /index.php?diff=
+Disallow: /index.php?oldid=
+Disallow: /index.php?title=Help
+Disallow: /index.php?title=Image
+Disallow: /index.php?title=MediaWiki
+Disallow: /index.php?title=Special:
+Disallow: /index.php?title=Template
+Disallow: /wiki/Special:
+Disallow: /api.php
+Disallow: /index.php?returnto=
+Disallow: /index.php?*userExpLevel=
+
+# Allows robots to find all the pages containing content
+Allow: /wiki/Special:AllPages
+
+# Allow the Internet Archiver to index action=raw and thereby store the raw wikitext of pages
+User-agent: ia_archiver
+Allow: /*&action=raw
+Allow: /*&action=edit