Merge pull request #6870 from psiinon/spider/stats

Client and AJAX spider stats

Author: kingthorin

addOns/callhome/CHANGELOG.md

@@ -4,7 +4,8 @@ All notable changes to this add-on will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
-
+### Added
+- AJAX Spider stats to telemetry.
 
 ## [0.16.0] - 2025-10-22
 ### Added

addOns/callhome/src/main/java/org/zaproxy/addon/callhome/ExtensionCallHome.java

@@ -327,6 +327,7 @@ public boolean test(Entry<String, Long> t) {
                     || key.startsWith("stats.selenium.")
                     || key.startsWith("stats.sequence.")
                     || key.startsWith("stats.spider.")
+                    || key.startsWith("stats.spiderAjax.")
                     || key.startsWith("stats.tech.")
                     || key.startsWith("stats.ui.")
                     || key.startsWith("stats.websockets.")

addOns/client/CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Add optional parameters for the Client Spider API action `scan`:
   - `numberOfBrowsers` - control concurrency (number of browser windows).
   - `scopeCheck` - select Scope Check (Flexible or Strict).
+- Spider stats.
 
 ## [0.17.0] - 2025-09-02
 ### Added

addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java

@@ -249,7 +249,9 @@ public void run() {
         if (options.getMaxDuration() > 0) {
             maxTime = startTime + TimeUnit.MINUTES.toMillis(options.getMaxDuration());
         }
+        Stats.incCounter("stats.client.spider.started");
         if (user != null) {
+            Stats.incCounter("stats.client.spider.started.user");
             synchronized (this.extClient.getAuthenticationHandlers()) {
                 this.extClient
                         .getAuthenticationHandlers()
@@ -662,6 +664,7 @@ private void finished() {
             listener.scanFinshed(scanId, displayName);
         }
 
+        Stats.incCounter("stats.client.spider.time", timeTaken);
         Stats.incCounter("stats.client.spider.urls", crawledUrls.size());
         Stats.incCounter(
                 "stats.client.spider.nodes", addedNodesModel.getRowCount() + contentLoaded);

addOns/spiderAjax/CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this add-on will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
+### Added
+- Spider stats.
+
 ### Fixed
 - Correctly validate browser IDs.
 

addOns/spiderAjax/src/main/java/org/zaproxy/zap/extension/spiderAjax/SpiderThread.java

@@ -68,6 +68,7 @@
 import org.zaproxy.zap.model.ScanEventPublisher;
 import org.zaproxy.zap.network.HttpResponseBody;
 import org.zaproxy.zap.users.User;
+import org.zaproxy.zap.utils.Stats;
 
 public class SpiderThread implements Runnable {
 
@@ -92,6 +93,7 @@ public class SpiderThread implements Runnable {
     private boolean running;
     private final Session session;
     private static final Logger LOGGER = LogManager.getLogger(SpiderThread.class);
+    private long startTime;
 
     private HttpResponseHeader outOfScopeResponseHeader;
     private HttpResponseBody outOfScopeResponseBody;
@@ -339,16 +341,19 @@ public void run() {
         LOGGER.info(
                 "Running Crawljax (with {}): {}", target.getOptions().getBrowserId(), displayName);
         this.running = true;
+        this.startTime = System.currentTimeMillis();
         notifyListenersSpiderStarted();
         SpiderEventPublisher.publishScanEvent(
                 ScanEventPublisher.SCAN_STARTED_EVENT,
                 0,
                 this.target.toTarget(),
                 target.getStartUri().toString(),
                 this.target.getUser());
+        Stats.incCounter("stats.spiderAjax.started");
 
         User user = target.getUser();
         if (user != null) {
+            Stats.incCounter("stats.spiderAjax.started.user");
             for (AuthenticationHandler ah : extension.getAuthenticationHandlers()) {
                 if (ah.enableAuthentication(user)) {
                     authHandler = ah;
@@ -376,6 +381,7 @@ public void run() {
             LOGGER.error(e, e);
         } finally {
             this.running = false;
+            Stats.incCounter("stats.spiderAjax.time", System.currentTimeMillis() - this.startTime);
             LOGGER.info("Stopping proxy...");
             stopProxy();
             LOGGER.info("Proxy stopped.");
@@ -440,6 +446,7 @@ public void handleMessage(HttpMessageHandlerContext ctx, HttpMessage httpMessage
                     checkState(httpMessage.getRequestHeader().getURI().getEscapedURI());
 
             if (!ctx.isFromClient()) {
+                Stats.incCounter("stats.spiderAjax.urls.added");
                 notifyMessage(
                         httpMessage,
                         HistoryReference.TYPE_SPIDER_AJAX,

addOns/spiderAjax/src/main/java/org/zaproxy/zap/extension/spiderAjax/automation/AjaxSpiderJob.java

@@ -353,6 +353,8 @@ public void runJob(AutomationEnvironment env, AutomationProgress progress) {
             this.sleep(500);
 
             numUrlsFound = listener.getMessagesFound();
+            // Should remove this at some point, but its almost certainly being used by existing AF
+            // jobs
             Stats.incCounter("spiderAjax.urls.added", numUrlsFound - lastCount);
             lastCount = numUrlsFound;
 

addOns/spiderAjax/src/main/javahelp/org/zaproxy/zap/extension/spiderAjax/resources/help/contents/automation.html

@@ -51,7 +51,7 @@ <H2>Job: spiderAjax</H2>
     tests:
       - name: 'At least 100 URLs found'      # String: Name of the test, default: statistic + operator + value
         type: 'stats'                        # String: Type of test, only 'stats' is supported for now
-        statistic: 'spiderAjax.urls.added'   # String: Name of an integer / long statistic, currently supported: 'spiderAjax.urls.added'
+        statistic: 'stats.spiderAjax.urls.added'   # String: Name of an integer / long statistic, currently supported: 'stats.spiderAjax.urls.added'
         operator: '&gt;='                       # String ['==', '!=', '&gt;=', '&gt;', '&lt;', '&lt;=']: Operator used for testing
         value: 100                           # Int: Change this to the number of URLs you expect to find
         onFail: 'info'                       # String [warn, error, info]: Change this to 'warn' or 'error' for the test to take effect
@@ -63,5 +63,8 @@ <H2>Job: spiderAjax</H2>
 rule installed and enabled. If either of those things are not done then the ajax spider will always run and a warning output.
 If they are both done and no "Modern Web Application" alert is raised then the assumption is made that this is a tradition app 
 and therefore the ajax spider is not needed.
+
+Previously the statistic "spiderAjax.urls.started" was specified. This is no longer recommended and will be removed at some point in the future.
+
 </BODY>
 </HTML>

addOns/spiderAjax/src/main/resources/org/zaproxy/zap/extension/spiderAjax/resources/spiderAjax-max.yaml

@@ -24,7 +24,7 @@
     tests:
       - name: 'At least X URLs found'        # String: Name of the test, default: statistic + operator + value
         type: 'stats'                        # String: Type of test, only 'stats' is supported for now
-        statistic: 'spiderAjax.urls.added'   # String: Name of an integer / long statistic, currently supported: 'spiderAjax.urls.added'
+        statistic: 'stats.spiderAjax.urls.added'   # String: Name of an integer / long statistic, currently supported: 'stats.spiderAjax.urls.added'
         operator: '>='                       # String ['==', '!=', '>=', '>', '<', '<=']: Operator used for testing
         value: 100                           # Int: Change this to the number of URLs you expect to find
         onFail: 'info'                       # String [warn, error, info]: Change this to 'warn' or 'error' for the test to take effect

addOns/spiderAjax/src/main/resources/org/zaproxy/zap/extension/spiderAjax/resources/spiderAjax-min.yaml

@@ -11,7 +11,7 @@
     tests:
       - name: 'At least X URLs found'        # String: Name of the test, default: statistic + operator + value
         type: 'stats'                        # String: Type of test, only 'stats' is supported for now
-        statistic: 'spiderAjax.urls.added'   # String: Name of an integer / long statistic, currently supported: 'spiderAjax.urls.added'
+        statistic: 'stats.spiderAjax.urls.added'   # String: Name of an integer / long statistic, currently supported: 'stats.spiderAjax.urls.added'
         operator: '>='                       # String ['==', '!=', '>=', '>', '<', '<=']: Operator used for testing
         value: 100                           # Int: Change this to the number of URLs you expect to find
         onFail: 'info'                       # String [warn, error, info]: Change this to 'warn' or 'error' for the test to take effect