robots-txt.config.js

module.exports = {
  host: "https://thepmgrp.com",
  sitemap: "https://thepmgrp.com/sitemap.xml",
  policy: [{ userAgent: "*", allow: "/" }],
  env: {
    development: {
      policy: [{ userAgent: "*", disallow: ["/"] }],
    },
    production: {
      policy: [
        { userAgent: "*", allow: "/" },
        { userAgent: "*", disallow: ["/login/"] },
        { userAgent: "*", disallow: ["/cgi-bin/"] },
        { userAgent: "*", disallow: ["/wp-admin/"] },
        { userAgent: "*", disallow: ["/wp-login.php"] },
        { userAgent: "*", disallow: ["/wp-admin/admin-ajax.php"] },
        { userAgent: "*", disallow: ["/wp-"] },
        { userAgent: "*", disallow: ["/?s="] },
        { userAgent: "*", disallow: ["*&s="] },
        { userAgent: "*", disallow: ["/search"] },
        { userAgent: "*", disallow: ["/author/*"] },
        { userAgent: "*", disallow: ["*?attachment_id="] },
        { userAgent: "*", disallow: ["*/feed"] },
        { userAgent: "*", disallow: ["*/rss"] },
        { userAgent: "*", disallow: ["*/embed"] },
        { userAgent: "*", disallow: ["/wp-admin/"] },
        { userAgent: "*", disallow: ["/trackback/"] },
        { userAgent: "*", disallow: ["/tag/*"] },
        { userAgent: "*", disallow: ["/product/*"] },
        { userAgent: "*", disallow: ["/category/*"] },
        { userAgent: "*", disallow: ["/test/"] },
        { userAgent: "*", disallow: ["/thank-you/"] },
        { userAgent: "*", disallow: ["/thank-you-careers/"] },
      ],
    },
  },
}