feat: bot detection logic

package.json

@@ -84,7 +84,7 @@
     "@adobe/spacecat-shared-rum-api-client": "2.40.3",
     "@adobe/spacecat-shared-scrape-client": "2.3.6",
     "@adobe/spacecat-shared-slack-client": "1.5.32",
-    "@adobe/spacecat-shared-utils": "1.85.2",
+    "@adobe/spacecat-shared-utils": "https://gist.github.com/tkotthakota-adobe/0bcfeb9e5daac09bb328ae94bc9dfdd7/raw/b63b067b1b5b516b65784280aa6770290626f974/adobe-spacecat-shared-utils-1.86.0.tgz",
     "@aws-sdk/client-cloudwatch-logs": "3.946.0",
     "@aws-sdk/client-lambda": "3.946.0",
     "@aws-sdk/client-sqs": "3.946.0",

src/tasks/opportunity-status-processor/handler.js

@@ -16,7 +16,7 @@ import RUMAPIClient from '@adobe/spacecat-shared-rum-api-client';
 import GoogleClient from '@adobe/spacecat-shared-google-client';
 import { ScrapeClient } from '@adobe/spacecat-shared-scrape-client';
 import { resolveCanonicalUrl } from '@adobe/spacecat-shared-utils';
-import { say } from '../../utils/slack-utils.js';
+import { say, formatBotProtectionSlackMessage } from '../../utils/slack-utils.js';
 import { getOpportunitiesForAudit } from './audit-opportunity-map.js';
 import { OPPORTUNITY_DEPENDENCY_MAP } from './opportunity-dependency-map.js';
 
@@ -202,6 +202,49 @@ async function isScrapingAvailable(baseUrl, context) {
   }
 }
 
+/**
+ * Checks scrape results for bot protection blocking
+ * @param {Array} scrapeResults - Array of scrape URL results
+ * @param {object} context - The context object with log
+ * @returns {object|null} Bot protection details if detected, null otherwise
+ */
+async function checkBotProtectionInScrapes(scrapeResults, context) {
+  const { log } = context;
+
+  if (!scrapeResults || scrapeResults.length === 0) {
+    return null;
+  }
+
+  // Count URLs with bot protection
+  const blockedResults = scrapeResults.filter((result) => {
+    const metadata = result.metadata || {};
+    const { botProtection } = metadata;
+
+    return botProtection && (botProtection.blocked || !botProtection.crawlable);
+  });
+
+  if (blockedResults.length === 0) {
+    return null;
+  }
+
+  // Get details from first blocked result
+  const firstBlocked = blockedResults[0];
+  const { botProtection } = firstBlocked.metadata;
+
+  log.warn(`Bot protection detected: ${blockedResults.length}/${scrapeResults.length} URLs blocked`);
+  log.warn(`Type: ${botProtection.type}, Confidence: ${(botProtection.confidence * 100).toFixed(0)}%`);
+
+  return {
+    detected: true,
+    type: botProtection.type,
+    confidence: botProtection.confidence,
+    blockedCount: blockedResults.length,
+    totalCount: scrapeResults.length,
+    reason: botProtection.reason,
+    details: botProtection.details,
+  };
+}
+
 /**
  * Searches CloudWatch logs for audit execution
  * @param {string} auditType - The audit type to search for
@@ -507,6 +550,35 @@ export async function runOpportunityStatusProcessor(message, context) {
               await say(env, log, slackContext, statsMessage);
             }
           }
+
+          // Check for bot protection in scrape results
+          if (scrapingCheck.results && scrapingCheck.results.length > 0 && slackContext) {
+            const botProtection = await checkBotProtectionInScrapes(
+              scrapingCheck.results,
+              context,
+            );
+
+            if (botProtection) {
+              log.warn(`Bot protection blocking scrapes for ${siteUrl}`);
+
+              // Determine environment from AWS_REGION or env variable
+              const environment = env.AWS_REGION?.includes('us-east') ? 'prod' : 'dev';
+
+              // Send detailed bot protection alert
+              await say(
+                env,
+                log,
+                slackContext,
+                formatBotProtectionSlackMessage({
+                  siteUrl,
+                  botProtection,
+                  environment,
+                  blockedCount: botProtection.blockedCount,
+                  totalCount: botProtection.totalCount,
+                }),
+              );
+            }
+          }
         }
       } catch (error) {
         log.warn(`Could not resolve canonical URL or parse siteUrl for data source checks: ${siteUrl}`, error);

src/utils/slack-utils.js

@@ -11,7 +11,7 @@
  */
 
 // eslint-disable-next-line import/no-unresolved
-import { hasText } from '@adobe/spacecat-shared-utils';
+import { hasText, SPACECAT_BOT_USER_AGENT, SPACECAT_BOT_IPS } from '@adobe/spacecat-shared-utils';
 import { BaseSlackClient, SLACK_TARGETS } from '@adobe/spacecat-shared-slack-client';
 /**
  * Sends a message to Slack using the provided client and context
@@ -50,3 +50,65 @@ export async function say(env, log, slackContext, message) {
     });
   }
 }
+
+/**
+ * Formats bot protection details for Slack notifications
+ * @param {Object} options - Options
+ * @param {string} options.siteUrl - Site URL
+ * @param {Object} options.botProtection - Bot protection details
+ * @param {string} [options.auditType] - Audit type (optional, for context)
+ * @param {string} [options.environment='prod'] - Environment ('prod' or 'dev')
+ * @param {number} [options.blockedCount] - Number of blocked URLs (optional)
+ * @param {number} [options.totalCount] - Total number of URLs (optional)
+ * @returns {string} Formatted Slack message
+ */
+export function formatBotProtectionSlackMessage({
+  siteUrl,
+  botProtection,
+  auditType,
+  environment = 'prod',
+  blockedCount,
+  totalCount,
+}) {
+  const ips = environment === 'prod'
+    ? SPACECAT_BOT_IPS.production
+    : SPACECAT_BOT_IPS.development;
+  const ipList = ips.map((ip) => `• \`${ip}\``).join('\n');
+
+  const auditInfo = auditType ? ` during ${auditType} audit` : '';
+  const envLabel = environment === 'prod' ? 'Production' : 'Development';
+
+  let message = `:warning: *Bot Protection Detected${auditInfo}*\n\n`
+    + `*Site:* ${siteUrl}\n`
+    + `*Protection Type:* ${botProtection.type}\n`
+    + `*Confidence:* ${(botProtection.confidence * 100).toFixed(0)}%\n`;
+
+  // Add blocked count if provided
+  if (blockedCount !== undefined && totalCount !== undefined) {
+    const blockedPercent = ((blockedCount / totalCount) * 100).toFixed(0);
+    message += `*Blocked URLs:* ${blockedCount}/${totalCount} (${blockedPercent}%)\n`;
+  }
+
+  if (botProtection.reason) {
+    message += `*Reason:* ${botProtection.reason}\n`;
+  }
+
+  message += '\n'
+    + '*Impact on Audit Results:*\n'
+    + '• Scraper received challenge pages instead of real content\n'
+    + '• Audit results may be incorrect or incomplete\n'
+    + '• Opportunities may be inaccurate or missing\n'
+    + '\n'
+    + '*Action Required:*\n'
+    + `Customer must allowlist SpaceCat in their ${botProtection.type} configuration:\n`
+    + '\n'
+    + '*User-Agent to allowlist:*\n'
+    + `\`${SPACECAT_BOT_USER_AGENT}\`\n`
+    + '\n'
+    + `*${envLabel} IPs to allowlist:*\n`
+    + `${ipList}\n`
+    + '\n'
+    + '_After allowlisting, re-run audits to get accurate results._';
+
+  return message;
+}