Adds detection for various bots (#7480)

* Adds detection for phpMyAdmin * Adds detection for Cloudflare SSL Detector * Change producer url for Cloudflare Health Checks * Improves detection for Cloudflare Diagnostics * Adds detection for Cloudflare Custom Hostname Verification * Adds detection for Cloudflare Traffic Manager * Adds detection for Cloudflare Security Insights
matomo-org · Oct 6, 2023 · 7d3606c · 7d3606c
1 parent b1ec060
commit 7d3606c
Show file tree

Hide file tree

Showing 2 changed files with 97 additions and 9 deletions.
diff --git a/Tests/fixtures/bots.yml b/Tests/fixtures/bots.yml
@@ -584,10 +584,46 @@
   bot:
     name: CloudFlare Always Online
     category: Site Monitor
-    url: http://www.cloudflare.com/always-online
+    url: https://www.cloudflare.com/always-online
     producer:
       name: CloudFlare
-      url: http://www.cloudflare.com
+      url: https://www.cloudflare.com/
+-
+  user_agent: Cloudflare-SSLDetector
+  bot:
+    name: Cloudflare SSL Detector
+    category: Site Monitor
+    url: https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/
+    producer:
+      name: CloudFlare
+      url: https://www.cloudflare.com/
+-
+  user_agent: Cloudflare Custom Hostname Verification
+  bot:
+    name: Cloudflare Custom Hostname Verification
+    category: Service Agent
+    url: https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/
+    producer:
+      name: CloudFlare
+      url: https://www.cloudflare.com/
+-
+  user_agent: 'Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/; pool-id: <POOLID>)'
+  bot:
+    name: Cloudflare Traffic Manager
+    category: Site Monitor
+    url: https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/
+    producer:
+      name: CloudFlare
+      url: https://www.cloudflare.com/
+-
+  user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36 (compatible; +https://developers.cloudflare.com/security-center/)
+  bot:
+    name: Cloudflare Security Insights
+    category: Site Monitor
+    url: https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/
+    producer:
+      name: CloudFlare
+      url: https://www.cloudflare.com/
 -
   user_agent: collectd/5.5.1
   bot:
@@ -3926,7 +3962,16 @@
     url: https://www.cloudflare.com/
     producer:
       name: Cloudflare
-      url: https://www.cloudflare.com
+      url: https://www.cloudflare.com/
+-
+  user_agent: Cloudflare-diagnostics
+  bot:
+    name: Cloudflare Diagnostics
+    category: Site Monitor
+    url: https://www.cloudflare.com/
+    producer:
+      name: Cloudflare
+      url: https://www.cloudflare.com/
 -
   user_agent: Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) adbeat.com/policy AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/69.0.3497.105 Mobile/15E148 Safari/605.1
   bot:
@@ -5565,7 +5610,7 @@
     url: https://developers.cloudflare.com/health-checks/
     producer:
       name: CloudFlare
-      url: http://www.cloudflare.com
+      url: https://www.cloudflare.com/
 -
   user_agent: Mozilla/5.0 (Morningscore/1.0)
   bot:
@@ -5777,3 +5822,9 @@
     producer:
       name: webwiki GmbH
       url: https://www.webwiki.com/
+-
+  user_agent: phpMyAdmin
+  bot:
+    name: phpMyAdmin
+    category: Service Agent
+    url: https://www.phpmyadmin.net/
diff --git a/regexes/bots.yml b/regexes/bots.yml
@@ -19,7 +19,7 @@
   url: 'https://developers.cloudflare.com/health-checks/'
   producer:
     name: 'CloudFlare'
-    url: 'http://www.cloudflare.com'
+    url: 'https://www.cloudflare.com/'
 
 - regex: '360Spider'
   name: '360Spider'
@@ -331,21 +331,53 @@
     name: 'CloudFlare'
     url: 'http://www.cloudflare.com'
 
-- regex: 'CloudflareDiagnostics'
+- regex: 'Cloudflare-?Diagnostics'
   name: 'Cloudflare Diagnostics'
   category: 'Site Monitor'
   url: 'https://www.cloudflare.com/'
   producer:
     name: 'Cloudflare'
-    url: 'https://www.cloudflare.com'
+    url: 'https://www.cloudflare.com/'
 
 - regex: 'CloudFlare-AlwaysOnline'
   name: 'CloudFlare Always Online'
   category: 'Site Monitor'
-  url: 'http://www.cloudflare.com/always-online'
+  url: 'https://www.cloudflare.com/always-online'
   producer:
     name: 'CloudFlare'
-    url: 'http://www.cloudflare.com'
+    url: 'https://www.cloudflare.com/'
+
+- regex: 'Cloudflare-SSLDetector'
+  name: 'Cloudflare SSL Detector'
+  category: 'Site Monitor'
+  url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/'
+  producer:
+    name: 'CloudFlare'
+    url: 'https://www.cloudflare.com/'
+
+- regex: 'Cloudflare Custom Hostname Verification'
+  name: 'Cloudflare Custom Hostname Verification'
+  category: 'Service Agent'
+  url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/'
+  producer:
+    name: 'CloudFlare'
+    url: 'https://www.cloudflare.com/'
+
+- regex: 'Cloudflare-Traffic-Manager'
+  name: 'Cloudflare Traffic Manager'
+  category: 'Site Monitor'
+  url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/'
+  producer:
+    name: 'CloudFlare'
+    url: 'https://www.cloudflare.com/'
+
+- regex: 'https://developers.cloudflare.com/security-center/'
+  name: 'Cloudflare Security Insights'
+  category: 'Site Monitor'
+  url: 'https://developers.cloudflare.com/fundamentals/reference/cloudflare-site-crawling/'
+  producer:
+    name: 'CloudFlare'
+    url: 'https://www.cloudflare.com/'
 
 - regex: 'coccoc.com'
   name: 'Cốc Cốc Bot'
@@ -3435,6 +3467,11 @@
     name: 'webwiki GmbH'
     url: 'https://www.webwiki.com/'
 
+- regex: 'phpMyAdmin'
+  name: 'phpMyAdmin'
+  category: 'Service Agent'
+  url: 'https://www.phpmyadmin.net/'
+
 # Generic detections
 - regex: '[a-z0-9\-_]*((?<!cu|power[ _]|m[ _])bot(?![ _]TAB|[ _]?5[0-9]|[ _]Senior|[ _]Junior)|crawler|crawl|checker|archiver|transcoder|spider)([^a-z]|$)'
   name: 'Generic Bot'