Merge branch 'main' into fcappelli/subscription_oauth_api_v2

# Conflicts:
#	DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

Configuration/BuildNumber.xcconfig

@@ -1 +1 @@
-CURRENT_PROJECT_VERSION = 316
+CURRENT_PROJECT_VERSION = 317

DuckDuckGo.xcodeproj/project.pbxproj

@@ -15187,7 +15187,7 @@
 			repositoryURL = "https://github.com/duckduckgo/BrowserServicesKit";
 			requirement = {
 				kind = exactVersion;
-				version = 211.1.3;
+				version = 212.0.0;
 			};
 		};
 		9FF521422BAA8FF300B9819B /* XCRemoteSwiftPackageReference "lottie-spm" */ = {

DuckDuckGo.xcodeproj/xcshareddata/xcschemes/DuckDuckGo Privacy Browser App Store.xcscheme

@@ -154,6 +154,30 @@
                <Test
                   Identifier = "NavigationProtectionIntegrationTests/testReferrerTrimming()">
                </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testFeatureDisabledAndPhishingDetection_tabIsNotMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingDetectedThenDDGLoaded_tabIsNotMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingDetectedThenNotDetected_tabIsNotMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingDetectedViaHTTPRedirectChain_tabIsMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingDetectedViaJSRedirectChain_tabIsMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingDetected_tabIsMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingNotDetected_tabIsNotMarkedPhishing()">
+               </Test>
                <Test
                   Identifier = "TabContentTests">
                </Test>

DuckDuckGo.xcodeproj/xcshareddata/xcschemes/DuckDuckGo Privacy Browser.xcscheme

@@ -186,6 +186,30 @@
                <Test
                   Identifier = "NavigationProtectionIntegrationTests/testReferrerTrimming()">
                </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testFeatureDisabledAndPhishingDetection_tabIsNotMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingDetectedThenDDGLoaded_tabIsNotMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingDetectedThenNotDetected_tabIsNotMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingDetectedViaHTTPRedirectChain_tabIsMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingDetectedViaJSRedirectChain_tabIsMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingDetected_tabIsMarkedPhishing()">
+               </Test>
+               <Test
+                  Identifier = "PhishingDetectionIntegrationTests/testPhishingNotDetected_tabIsNotMarkedPhishing()">
+               </Test>
                <Test
                   Identifier = "TabContentTests/testWhenPDFContextMenuPrintChosen_printDialogOpens()">
                </Test>

DuckDuckGo/Autoconsent/AutoconsentUserScript.swift

@@ -33,6 +33,10 @@ protocol UserScriptWithAutoconsent: UserScript {
 
 final class AutoconsentUserScript: NSObject, WKScriptMessageHandlerWithReply, UserScriptWithAutoconsent {
 
+    private struct Constants {
+        static let filterListCmpName = "filterList" // special CMP name used for reports from the cosmetic filterlist
+    }
+
     static let newSitePopupHiddenNotification = Notification.Name("newSitePopupHidden")
 
     var injectionTime: WKUserScriptInjectionTime { .atDocumentStart }
@@ -111,36 +115,36 @@ extension AutoconsentUserScript {
 
     struct PopupFoundMessage: Codable {
         let type: String
-        let cmp: String
+        let cmp: String // name of the Autoconsent rule that matched
         let url: String
     }
 
     struct OptOutResultMessage: Codable {
         let type: String
-        let cmp: String
+        let cmp: String // name of the Autoconsent rule that matched
         let result: Bool
         let scheduleSelfTest: Bool
         let url: String
     }
 
     struct OptInResultMessage: Codable {
         let type: String
-        let cmp: String
+        let cmp: String // name of the Autoconsent rule that matched
         let result: Bool
         let scheduleSelfTest: Bool
         let url: String
     }
 
     struct SelfTestResultMessage: Codable {
         let type: String
-        let cmp: String
+        let cmp: String // name of the Autoconsent rule that matched
         let result: Bool
         let url: String
     }
 
     struct AutoconsentDoneMessage: Codable {
         let type: String
-        let cmp: String
+        let cmp: String // name of the Autoconsent rule that matched
         let url: String
         let isCosmetic: Bool
     }
@@ -171,8 +175,7 @@ extension AutoconsentUserScript {
         case MessageName.eval:
             handleEval(message: message, replyHandler: replyHandler)
         case MessageName.popupFound:
-            Logger.autoconsent.debug("Autoconsent popup found")
-            replyHandler([ "type": "ok" ], nil) // this is just to prevent a Promise rejection
+            handlePopupFound(message: message, replyHandler: replyHandler)
         case MessageName.optOutResult:
             handleOptOutResult(message: message, replyHandler: replyHandler)
         case MessageName.optInResult:
@@ -187,21 +190,19 @@ extension AutoconsentUserScript {
         case MessageName.autoconsentDone:
             handleAutoconsentDone(message: message, replyHandler: replyHandler)
         case MessageName.autoconsentError:
-            Logger.autoconsent.debug("Autoconsent error: \(String(describing: message.body))")
+            Logger.autoconsent.error("Autoconsent error: \(String(describing: message.body))")
             replyHandler([ "type": "ok" ], nil) // this is just to prevent a Promise rejection
         }
     }
 
     @MainActor
     func handleInit(message: WKScriptMessage, replyHandler: @escaping (Any?, String?) -> Void) {
-        guard let messageData: InitMessage = decodeMessageBody(from: message.body) else {
+        guard let messageData: InitMessage = decodeMessageBody(from: message.body),
+              let url = URL(string: messageData.url) else {
+            assertionFailure("Received a malformed message from autoconsent")
             replyHandler(nil, "cannot decode message")
             return
         }
-        guard let url = URL(string: messageData.url) else {
-            replyHandler(nil, "cannot decode init request")
-            return
-        }
 
         guard url.navigationalScheme?.isHypertextScheme == true else {
             // ignore special schemes
@@ -256,6 +257,7 @@ extension AutoconsentUserScript {
     @MainActor
     func handleEval(message: WKScriptMessage, replyHandler: @escaping (Any?, String?) -> Void) {
         guard let messageData: EvalMessage = decodeMessageBody(from: message.body) else {
+            assertionFailure("Received a malformed message from autoconsent")
             replyHandler(nil, "cannot decode message")
             return
         }
@@ -291,9 +293,39 @@ extension AutoconsentUserScript {
         }
     }
 
+    @MainActor
+    func handlePopupFound(message: WKScriptMessage, replyHandler: @escaping (Any?, String?) -> Void) {
+        guard let messageData: PopupFoundMessage = decodeMessageBody(from: message.body),
+              let url = URL(string: messageData.url),
+              let host = url.host else {
+            assertionFailure("Received a malformed message from autoconsent")
+            replyHandler(nil, "cannot decode message")
+            return
+        }
+        Logger.autoconsent.debug("Cookie popup found: \(String(describing: messageData))")
+
+        // if popupFound is sent with "filterList", it indicates that cosmetic filterlist matched in the prehide stage,
+        // but a real opt-out may still follow. See https://github.com/duckduckgo/autoconsent/blob/main/api.md#messaging-api
+        if messageData.cmp == Constants.filterListCmpName {
+            refreshDashboardState(consentManaged: true, cosmetic: true, optoutFailed: false, selftestFailed: nil)
+            // trigger animation, but do not cache it because it can still be overridden
+            if !management.sitesNotifiedCache.contains(host) {
+                Logger.autoconsent.debug("Starting animation for cosmetic filters")
+                // post popover notification
+                NotificationCenter.default.post(name: Self.newSitePopupHiddenNotification, object: self, userInfo: [
+                    "topUrl": self.topUrl ?? url,
+                    "isCosmetic": true
+                ])
+            }
+        }
+
+        replyHandler([ "type": "ok" ], nil) // this is just to prevent a Promise rejection
+    }
+
     @MainActor
     func handleOptOutResult(message: WKScriptMessage, replyHandler: @escaping (Any?, String?) -> Void) {
         guard let messageData: OptOutResultMessage = decodeMessageBody(from: message.body) else {
+            assertionFailure("Received a malformed message from autoconsent")
             replyHandler(nil, "cannot decode message")
             return
         }
@@ -313,26 +345,24 @@ extension AutoconsentUserScript {
     @MainActor
     func handleAutoconsentDone(message: WKScriptMessage, replyHandler: @escaping (Any?, String?) -> Void) {
         // report a managed popup
-        guard let messageData: AutoconsentDoneMessage = decodeMessageBody(from: message.body) else {
-            replyHandler(nil, "cannot decode message")
-            return
-        }
-        Logger.autoconsent.debug("opt-out successful: \(String(describing: messageData))")
-
-        guard let url = URL(string: messageData.url),
+        guard let messageData: AutoconsentDoneMessage = decodeMessageBody(from: message.body),
+              let url = URL(string: messageData.url),
               let host = url.host else {
+            assertionFailure("Received a malformed message from autoconsent")
             replyHandler(nil, "cannot decode message")
             return
         }
 
+        Logger.autoconsent.debug("opt-out successful: \(String(describing: messageData))")
+
         refreshDashboardState(consentManaged: true, cosmetic: messageData.isCosmetic, optoutFailed: false, selftestFailed: nil)
 
         // trigger popup once per domain
         if !management.sitesNotifiedCache.contains(host) {
-            Logger.autoconsent.debug("bragging that we closed a popup")
             management.sitesNotifiedCache.insert(host)
-            // post popover notification on main thread
-            DispatchQueue.main.async {
+            if messageData.cmp != Constants.filterListCmpName { // filterlist animation should have been triggered already (see handlePopupFound)
+                Logger.autoconsent.debug("Starting animation for the handled cookie popup")
+                // post popover notification
                 NotificationCenter.default.post(name: Self.newSitePopupHiddenNotification, object: self, userInfo: [
                     "topUrl": self.topUrl ?? url,
                     "isCosmetic": messageData.isCosmetic
@@ -359,15 +389,17 @@ extension AutoconsentUserScript {
                 }
             )
         } else {
-            Logger.autoconsent.debug("no self-test scheduled in this tab")
+            Logger.autoconsent.error("no self-test scheduled in this tab")
         }
         selfTestWebView = nil
         selfTestFrameInfo = nil
     }
 
     @MainActor
     func handleSelfTestResult(message: WKScriptMessage, replyHandler: @escaping (Any?, String?) -> Void) {
-        guard let messageData: SelfTestResultMessage = decodeMessageBody(from: message.body) else {
+        guard let messageData: SelfTestResultMessage = decodeMessageBody(from: message.body),
+              let url = URL(string: messageData.url) else {
+            assertionFailure("Received a malformed message from autoconsent")
             replyHandler(nil, "cannot decode message")
             return
         }

DuckDuckGo/ContentBlocker/ContentBlocking.swift

@@ -137,6 +137,17 @@ final class AppContentBlocking {
 
             domainEvent = .contentBlockingCompilationFailed(listType: listType, component: component)
 
+        case .contentBlockingLookupRulesSucceeded:
+                    domainEvent = .contentBlockingLookupRulesSucceeded
+
+        case .contentBlockingFetchLRCSucceeded:
+                    domainEvent = .contentBlockingFetchLRCSucceeded
+
+        case .contentBlockingNoMatchInLRC:
+                    domainEvent = .contentBlockingNoMatchInLRC
+
+        case .contentBlockingLRCMissing:
+                    domainEvent = .contentBlockingLRCMissing
         case .contentBlockingCompilationTime:
             // Temporarily avoid firing this pixel. This can be re-enabled if it's determined to be necessary later.
             // domainEvent = .contentBlockingCompilationTime

DuckDuckGo/HomePage/Model/HomePageAddressBarModel.swift

@@ -101,6 +101,7 @@ extension HomePage.Models {
         func setUpExperimentIfNeeded() {
             if isExperimentActive {
                 let ntpExperiment = NewTabPageSearchBoxExperiment()
+                ntpExperiment.assignUserToCohort()
                 shouldShowAddressBar = ntpExperiment.cohort?.isExperiment == true
             }
         }

DuckDuckGo/Statistics/GeneralPixel.swift

@@ -314,6 +314,10 @@ enum GeneralPixel: PixelKitEventV2 {
     case contentBlockingCompilationFailed(listType: CompileRulesListType, component: ContentBlockerDebugEvents.Component)
 
     case contentBlockingCompilationTime
+    case contentBlockingLookupRulesSucceeded
+    case contentBlockingFetchLRCSucceeded
+    case contentBlockingNoMatchInLRC
+    case contentBlockingLRCMissing
 
     case secureVaultInitError(error: Error)
     case secureVaultError(error: Error)
@@ -930,6 +934,15 @@ enum GeneralPixel: PixelKitEventV2 {
         case .contentBlockingCompilationTime:
             return "content_blocking_compilation_time"
 
+        case .contentBlockingLookupRulesSucceeded:
+            return "content_blocking_lookup_rules_succeeded"
+        case .contentBlockingFetchLRCSucceeded:
+            return "content_blocking_fetch_lrc_succeeded"
+        case .contentBlockingNoMatchInLRC:
+            return "content_blocking_no_match_in_lrc"
+        case .contentBlockingLRCMissing:
+            return "content_blocking_lrc_missing"
+
         case .secureVaultInitError:
             return "secure_vault_init_error"
         case .secureVaultError:

IntegrationTests/PhishingDetection/PhishingDetectionIntegrationTests.swift

@@ -119,7 +119,7 @@ class PhishingDetectionIntegrationTests: XCTestCase {
     }
 
     @MainActor
-    func testPhishingDetectedViaJSRedirectChain_tabIsMarkedPhishing() async throws {
+    func disabled_testPhishingDetectedViaJSRedirectChain_tabIsMarkedPhishing() async throws {
         loadUrl("http://privacy-test-pages.site/security/badware/phishing-js-redirector.html")
         try await waitForTabToFinishLoading()
         let tabErrorCode = tabViewModel.tab.error?.errorCode

LocalPackages/DataBrokerProtection/Package.swift

@@ -29,7 +29,7 @@ let package = Package(
             targets: ["DataBrokerProtection"])
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "211.1.3"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "212.0.0"),
         .package(path: "../SwiftUIExtensions"),
         .package(path: "../AppKitExtensions"),
         .package(path: "../XPCHelper"),

LocalPackages/FeatureFlags/Package.swift

@@ -32,7 +32,7 @@ let package = Package(
             targets: ["FeatureFlags"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "211.1.3"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "212.0.0"),
     ],
     targets: [
         // Targets are the basic building blocks of a package, defining a module or a test suite.

LocalPackages/NetworkProtectionMac/Package.swift

@@ -32,7 +32,7 @@ let package = Package(
         .library(name: "VPNAppLauncher", targets: ["VPNAppLauncher"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "211.1.3"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "212.0.0"),
         .package(url: "https://github.com/airbnb/lottie-spm", exact: "4.4.3"),
         .package(path: "../AppLauncher"),
         .package(path: "../UDSHelper"),

LocalPackages/SubscriptionUI/Package.swift

@@ -12,7 +12,7 @@ let package = Package(
             targets: ["SubscriptionUI"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "211.1.3"),
+        .package(url: "https://github.com/duckduckgo/BrowserServicesKit", exact: "212.0.0"),
         .package(path: "../SwiftUIExtensions")
     ],
     targets: [

UnitTests/Autoconsent/AutoconsentMessageProtocolTests.swift

@@ -119,7 +119,7 @@ class AutoconsentMessageProtocolTests: XCTestCase {
         let message = MockWKScriptMessage(name: "popupFound", body: [
             "type": "popupFound",
             "cmp": "some cmp",
-            "url": "some url"
+            "url": "https://example.com"
         ])
         userScript.handleMessage(
             replyHandler: {(msg: Any?, _: String?) in