Skip to content

Commit

Permalink
tds override (#3726)
Browse files Browse the repository at this point in the history 
Task/Issue URL:
https://app.asana.com/0/1204186595873227/1208547600159200/f
Tech Design URL:
https://app.asana.com/0/481882893211075/1208547600159199
CC:

**Description**: Makes the tds fetch dependent on the privacy
configuration fetch (so that the url for tds can be taken from the
configuration). Sets up experiment for tds. Note metrics will be added
in a separate PR.
  • Loading branch information
@SabrinaTardio
SabrinaTardio authored Jan 17, 2025
1 parent ae3a141 commit 4f1655b
Show file tree
Hide file tree
Showing 18 changed files with 431 additions and 32 deletions.
4 changes: 4 additions & 0 deletions DBPE2ETests/DBPEndToEndTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,10 @@ private extension DBPEndToEndTests {
[String: Any]()
}

func settings(for subfeature: any BrowserServicesKit.PrivacySubfeature) -> PrivacyConfigurationData.PrivacyFeature.SubfeatureSettings? {
nil
}

func userEnabledProtection(forDomain: String) {

}
Expand Down
22 changes: 21 additions & 1 deletion DuckDuckGo.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1682,6 +1682,8 @@
562532A12BC069190034D316 /* ZoomPopoverViewModelTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5625329D2BC069100034D316 /* ZoomPopoverViewModelTests.swift */; };
562984702AC4610100AC20EB /* SyncPreferencesTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5629846E2AC4610100AC20EB /* SyncPreferencesTests.swift */; };
562984712AC469E400AC20EB /* SyncPreferencesTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5629846E2AC4610100AC20EB /* SyncPreferencesTests.swift */; };
563A3CFA2D37AE2A001966FD /* ConfigurationManagerIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 563A3CF92D37AE2A001966FD /* ConfigurationManagerIntegrationTests.swift */; };
563A3CFB2D37AE2A001966FD /* ConfigurationManagerIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 563A3CF92D37AE2A001966FD /* ConfigurationManagerIntegrationTests.swift */; };
56406D4B2C636A8900BF8FA2 /* SpecialPagesUserScriptExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56406D4A2C636A8900BF8FA2 /* SpecialPagesUserScriptExtension.swift */; };
56406D4C2C636A8900BF8FA2 /* SpecialPagesUserScriptExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56406D4A2C636A8900BF8FA2 /* SpecialPagesUserScriptExtension.swift */; };
5641734B2CFE168700F4B716 /* PixelExperimentKit in Frameworks */ = {isa = PBXBuildFile; productRef = 5641734A2CFE168700F4B716 /* PixelExperimentKit */; };
Expand Down Expand Up @@ -1775,6 +1777,8 @@
56BA1E802BAB2E43001CF69F /* ErrorPageTabExtensionTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56BA1E7C2BAB290E001CF69F /* ErrorPageTabExtensionTest.swift */; };
56BA1E8A2BB1CB5B001CF69F /* CertificateTrustEvaluator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56BA1E892BB1CB5B001CF69F /* CertificateTrustEvaluator.swift */; };
56BA1E8B2BB1CB5B001CF69F /* CertificateTrustEvaluator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56BA1E892BB1CB5B001CF69F /* CertificateTrustEvaluator.swift */; };
56BC8F012D312B320046059D /* ConfigurationManagerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56BC8F002D312B320046059D /* ConfigurationManagerTests.swift */; };
56BC8F022D312B320046059D /* ConfigurationManagerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56BC8F002D312B320046059D /* ConfigurationManagerTests.swift */; };
56CE77612C7DFCF800AC1ED2 /* OnboardingSuggestedSearchesProviderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56CE77602C7DFCF800AC1ED2 /* OnboardingSuggestedSearchesProviderTests.swift */; };
56CE77622C7DFCF800AC1ED2 /* OnboardingSuggestedSearchesProviderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 56CE77602C7DFCF800AC1ED2 /* OnboardingSuggestedSearchesProviderTests.swift */; };
56CEE90E2B7A725B00CF10AA /* InfoPlist.xcstrings in Resources */ = {isa = PBXBuildFile; fileRef = 56CEE90D2B7A6DE100CF10AA /* InfoPlist.xcstrings */; };
Expand Down Expand Up @@ -4085,6 +4089,7 @@
561D66692B95C45A008ACC5C /* Suggestion.storyboard */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; path = Suggestion.storyboard; sourceTree = "<group>"; };
5625329D2BC069100034D316 /* ZoomPopoverViewModelTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ZoomPopoverViewModelTests.swift; sourceTree = "<group>"; };
5629846E2AC4610100AC20EB /* SyncPreferencesTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SyncPreferencesTests.swift; sourceTree = "<group>"; };
563A3CF92D37AE2A001966FD /* ConfigurationManagerIntegrationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConfigurationManagerIntegrationTests.swift; sourceTree = "<group>"; };
56406D4A2C636A8900BF8FA2 /* SpecialPagesUserScriptExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpecialPagesUserScriptExtension.swift; sourceTree = "<group>"; };
56534DEC29DF252C00121467 /* CapturingDefaultBrowserProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CapturingDefaultBrowserProvider.swift; sourceTree = "<group>"; };
565E46DD2B2725DC0013AC2A /* SyncE2EUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = SyncE2EUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
Expand Down Expand Up @@ -4131,6 +4136,7 @@
56BA1E742BAAF70F001CF69F /* SpecialErrorPageTabExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpecialErrorPageTabExtension.swift; sourceTree = "<group>"; };
56BA1E7C2BAB290E001CF69F /* ErrorPageTabExtensionTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ErrorPageTabExtensionTest.swift; sourceTree = "<group>"; };
56BA1E892BB1CB5B001CF69F /* CertificateTrustEvaluator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CertificateTrustEvaluator.swift; sourceTree = "<group>"; };
56BC8F002D312B320046059D /* ConfigurationManagerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConfigurationManagerTests.swift; sourceTree = "<group>"; };
56CE77602C7DFCF800AC1ED2 /* OnboardingSuggestedSearchesProviderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OnboardingSuggestedSearchesProviderTests.swift; sourceTree = "<group>"; };
56CEE9092B7A66C500CF10AA /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
56CEE90D2B7A6DE100CF10AA /* InfoPlist.xcstrings */ = {isa = PBXFileReference; lastKnownFileType = text.json.xcstrings; path = InfoPlist.xcstrings; sourceTree = "<group>"; };
Expand Down Expand Up @@ -6234,6 +6240,7 @@
4B1AD89E25FC27E200261379 /* IntegrationTests */ = {
isa = PBXGroup;
children = (
563A3CF82D37ADFA001966FD /* Configurations */,
560C6ECB2CCA5B9D00D411E2 /* Onboarding */,
84537A072C99C1EF008723BC /* App */,
EEE0E1CB2C32F53C0058E148 /* DataImport */,
Expand Down Expand Up @@ -7075,6 +7082,14 @@
path = Sync;
sourceTree = "<group>";
};
563A3CF82D37ADFA001966FD /* Configurations */ = {
isa = PBXGroup;
children = (
563A3CF92D37AE2A001966FD /* ConfigurationManagerIntegrationTests.swift */,
);
path = Configurations;
sourceTree = "<group>";
};
56534DEB29DF251C00121467 /* Mocks */ = {
isa = PBXGroup;
children = (
Expand Down Expand Up @@ -7496,6 +7511,7 @@
isa = PBXGroup;
children = (
85AC3B4825DAC9BD00C7D2AA /* ConfigurationStorageTests.swift */,
56BC8F002D312B320046059D /* ConfigurationManagerTests.swift */,
);
path = Configuration;
sourceTree = "<group>";
Expand Down Expand Up @@ -12454,6 +12470,7 @@
3706FE2F293F661700E42796 /* WebViewMock.swift in Sources */,
3706FE30293F661700E42796 /* CollectionExtension.swift in Sources */,
B630E80129C887ED00363609 /* NSErrorAdditionalInfo.swift in Sources */,
56BC8F012D312B320046059D /* ConfigurationManagerTests.swift in Sources */,
3706FE31293F661700E42796 /* TabCollectionViewModelDelegateMock.swift in Sources */,
3706FE32293F661700E42796 /* BookmarksHTMLReaderTests.swift in Sources */,
9F0FFFBC2BCCAEC2007C87DD /* AddEditBookmarkFolderDialogViewModelMock.swift in Sources */,
Expand Down Expand Up @@ -12670,6 +12687,7 @@
3706FEA6293F662100E42796 /* EncryptionKeyStoreTests.swift in Sources */,
B6F5656A299A414300A04298 /* WKWebViewMockingExtension.swift in Sources */,
EEE0E1CD2C32F5690058E148 /* CSVImporterIntegrationTests.swift in Sources */,
563A3CFB2D37AE2A001966FD /* ConfigurationManagerIntegrationTests.swift in Sources */,
B693766F2B6B5F27005BD9D4 /* ErrorPageTests.swift in Sources */,
56A054362C205820007D8FAB /* OnboardingPageTests.swift in Sources */,
);
Expand Down Expand Up @@ -12719,6 +12737,7 @@
4B1AD8D525FC38DD00261379 /* EncryptionKeyStoreTests.swift in Sources */,
B6F56568299A414300A04298 /* WKWebViewMockingExtension.swift in Sources */,
EEE0E1D12C32F8620058E148 /* CSVImporterIntegrationTests.swift in Sources */,
563A3CFA2D37AE2A001966FD /* ConfigurationManagerIntegrationTests.swift in Sources */,
B693766E2B6B5F27005BD9D4 /* ErrorPageTests.swift in Sources */,
56A054352C20581F007D8FAB /* OnboardingPageTests.swift in Sources */,
);
Expand Down Expand Up @@ -14293,6 +14312,7 @@
B6CA4824298CDC2E0067ECCE /* AdClickAttributionTabExtensionTests.swift in Sources */,
AAEC74B22642C57200C2EFBC /* HistoryCoordinatingMock.swift in Sources */,
37D046A12C7DA9A200AEAA50 /* UserBackgroundImagesManagerTests.swift in Sources */,
56BC8F022D312B320046059D /* ConfigurationManagerTests.swift in Sources */,
56A214AF2CB583BF00E5BC0E /* TrackerMessageProviderTests.swift in Sources */,
37CD54B927F1F8AC00F1F7B9 /* AppearancePreferencesTests.swift in Sources */,
EEF53E182950CED5002D78F4 /* JSAlertViewModelTests.swift in Sources */,
Expand Down Expand Up @@ -15366,7 +15386,7 @@
repositoryURL = "https://github.com/duckduckgo/BrowserServicesKit";
requirement = {
kind = exactVersion;
version = 224.7.2;
version = 225.0.0;
};
};
9FF521422BAA8FF300B9819B /* XCRemoteSwiftPackageReference "lottie-spm" */ = {
Expand Down
12 changes: 6 additions & 6 deletions DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,17 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/duckduckgo/BrowserServicesKit",
"state" : {
"revision" : "b3a8ea5ef9821203fe88a12e3f15ad48b7278a6a",
"version" : "224.7.2"
"revision" : "20e6eaf0b1e423d9a270e2d460cae284c08f73d8",
"version" : "225.0.0"
}
},
{
"identity" : "content-scope-scripts",
"kind" : "remoteSourceControl",
"location" : "https://github.com/duckduckgo/content-scope-scripts",
"state" : {
"revision" : "0502ed7de4130bd8705daebaca9aeb20d3e62d15",
"version" : "7.5.0"
"revision" : "7958ddab724c26326333cae13fe81478290607fa",
"version" : "7.6.0"
}
},
{
Expand Down Expand Up @@ -167,8 +167,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/duckduckgo/sync_crypto",
"state" : {
"revision" : "0c8bf3c0e75591bc366407b9d7a73a9fcfc7736f",
"version" : "0.3.0"
"revision" : "cc726cebb67367466bc31ced4784e16d44ac68d1",
"version" : "0.4.0"
}
},
{
Expand Down
31 changes: 27 additions & 4 deletions DuckDuckGo/Application/AppConfigurationURLProvider.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,17 @@

import Configuration
import Foundation
import BrowserServicesKit
import os.log

struct AppConfigurationURLProvider: ConfigurationURLProviding {

// MARK: - Debug

internal init(customPrivacyConfiguration: URL? = nil) {
internal init(privacyConfigurationManager: PrivacyConfigurationManaging = ContentBlocking.shared.privacyConfigurationManager,
featureFlagger: FeatureFlagger = Application.appDelegate.featureFlagger,
customPrivacyConfiguration: URL? = nil) {
let trackerDataUrlProvider = TrackerDataURLOverrider(privacyConfigurationManager: privacyConfigurationManager, featureFlagger: featureFlagger)
self.init(trackerDataUrlProvider: trackerDataUrlProvider)
if let customPrivacyConfiguration {
// Overwrite custom privacy configuration if provided
self.customPrivacyConfiguration = customPrivacyConfiguration.absoluteString
Expand All @@ -47,16 +52,34 @@ struct AppConfigurationURLProvider: ConfigurationURLProviding {

// MARK: - Main

private var trackerDataUrlProvider: TrackerDataURLProviding

public enum Constants {
public static let baseTdsURLString = "https://staticcdn.duckduckgo.com/trackerblocking/"
public static let defaultTrackerDataURL = URL(string: "https://staticcdn.duckduckgo.com/trackerblocking/v6/current/macos-tds.json")!
public static let defaultPrivacyConfigurationURL = URL(string: "https://staticcdn.duckduckgo.com/trackerblocking/config/v4/macos-config.json")!
}

init(privacyConfigurationManager: PrivacyConfigurationManaging = ContentBlocking.shared.privacyConfigurationManager,
featureFlagger: FeatureFlagger = Application.appDelegate.featureFlagger) {
self.trackerDataUrlProvider = TrackerDataURLOverrider(privacyConfigurationManager: privacyConfigurationManager, featureFlagger: featureFlagger)
}

init(trackerDataUrlProvider: TrackerDataURLProviding) {
self.trackerDataUrlProvider = trackerDataUrlProvider
}

func url(for configuration: Configuration) -> URL {
// URLs for privacyConfiguration and trackerDataSet shall match the ones in update_embedded.sh.
// Danger checks that the URLs match on every PR. If the code changes, the regex that Danger uses may need an update.
switch configuration {
case .bloomFilterBinary: return URL(string: "https://staticcdn.duckduckgo.com/https/https-mobile-v2-bloom.bin")!
case .bloomFilterSpec: return URL(string: "https://staticcdn.duckduckgo.com/https/https-mobile-v2-bloom-spec.json")!
case .bloomFilterExcludedDomains: return URL(string: "https://staticcdn.duckduckgo.com/https/https-mobile-v2-false-positives.json")!
case .privacyConfiguration: return customPrivacyConfigurationUrl ?? URL(string: "https://staticcdn.duckduckgo.com/trackerblocking/config/v4/macos-config.json")!
case .privacyConfiguration: return customPrivacyConfigurationUrl ?? Constants.defaultPrivacyConfigurationURL
case .surrogates: return URL(string: "https://staticcdn.duckduckgo.com/surrogates.txt")!
case .trackerDataSet: return URL(string: "https://staticcdn.duckduckgo.com/trackerblocking/v6/current/macos-tds.json")!
case .trackerDataSet:
return trackerDataUrlProvider.trackerDataURL ?? Constants.defaultTrackerDataURL
// In archived repo, to be refactored shortly (https://staticcdn.duckduckgo.com/useragents/social_ctp_configuration.json)
case .remoteMessagingConfig: return RemoteMessagingClient.Constants.endpoint
}
Expand Down
4 changes: 2 additions & 2 deletions DuckDuckGo/Application/AppDelegate.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,8 +198,6 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
let internalUserDeciderStore = InternalUserDeciderStore(fileStore: fileStore)
internalUserDecider = DefaultInternalUserDecider(store: internalUserDeciderStore)

configurationManager = ConfigurationManager(store: configurationStore)

if NSApplication.runType.requiresEnvironment {
Self.configurePixelKit()

Expand Down Expand Up @@ -279,6 +277,8 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
)
}

configurationManager = ConfigurationManager(store: configurationStore)

featureFlagger = DefaultFeatureFlagger(
internalUserDecider: internalUserDecider,
privacyConfigManager: AppPrivacyFeatures.shared.contentBlocking.privacyConfigurationManager,
Expand Down
57 changes: 45 additions & 12 deletions DuckDuckGo/Configuration/ConfigurationManager.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ import PixelKit

final class ConfigurationManager: DefaultConfigurationManager {

private let trackerDataManager: TrackerDataManager
private let privacyConfigurationManager: PrivacyConfigurationManaging
private var contentBlockingManager: ContentBlockerRulesManagerProtocol

private enum Constants {
static let lastConfigurationInstallDateKey = "config.last.installed"
}
Expand All @@ -53,10 +57,18 @@ final class ConfigurationManager: DefaultConfigurationManager {
PixelKit.fire(DebugEvent(domainEvent, error: error))
}

override init(fetcher: ConfigurationFetching = ConfigurationFetcher(store: ConfigurationStore(), eventMapping: configurationDebugEvents),
store: ConfigurationStoring = ConfigurationStore(),
defaults: KeyValueStoring = UserDefaults.appConfiguration) {
init(fetcher: ConfigurationFetching = ConfigurationFetcher(store: ConfigurationStore(), eventMapping: configurationDebugEvents),
store: ConfigurationStoring = ConfigurationStore(),
defaults: KeyValueStoring = UserDefaults.appConfiguration,
trackerDataManager: TrackerDataManager = ContentBlocking.shared.trackerDataManager,
privacyConfigurationManager: PrivacyConfigurationManaging = ContentBlocking.shared.privacyConfigurationManager,
contentBlockingManager: ContentBlockerRulesManagerProtocol = ContentBlocking.shared.contentBlockingManager) {

self.trackerDataManager = trackerDataManager
self.privacyConfigurationManager = privacyConfigurationManager
self.contentBlockingManager = contentBlockingManager
self.defaults = defaults

super.init(fetcher: fetcher, store: store, defaults: defaults)
}

Expand Down Expand Up @@ -107,10 +119,30 @@ final class ConfigurationManager: DefaultConfigurationManager {
private func fetchTrackerBlockingDependencies(isDebug: Bool) async -> Bool {
var didFetchAnyTrackerBlockingDependencies = false

var tasks = [Configuration: Task<(), Swift.Error>]()
tasks[.trackerDataSet] = Task { try await fetcher.fetch(.trackerDataSet, isDebug: isDebug) }
tasks[.surrogates] = Task { try await fetcher.fetch(.surrogates, isDebug: isDebug) }
tasks[.privacyConfiguration] = Task { try await fetcher.fetch(.privacyConfiguration, isDebug: isDebug) }
// Start surrogates fetch task
let surrogatesTask = Task { try await fetcher.fetch(.surrogates, isDebug: isDebug) }

// Perform privacyConfiguration fetch and update
do {
try await fetcher.fetch(.privacyConfiguration, isDebug: isDebug)
didFetchAnyTrackerBlockingDependencies = true
privacyConfigurationManager.reload(etag: store.loadEtag(for: .privacyConfiguration),
data: store.loadData(for: .privacyConfiguration))
} catch {
Logger.config.error(
"Failed to complete configuration update to \(Configuration.privacyConfiguration.rawValue, privacy: .public): \(error.localizedDescription, privacy: .public)"
)
tryAgainSoon()
}

// Start trackerDataSet fetch task after privacyConfiguration completes
let trackerDataSetTask = Task { try await fetcher.fetch(.trackerDataSet, isDebug: isDebug) }

// Wait for surrogates and trackerDataSet tasks
let tasks: [(Configuration, Task<(), Swift.Error>)] = [
(.surrogates, surrogatesTask),
(.trackerDataSet, trackerDataSetTask)
]

for (configuration, task) in tasks {
do {
Expand Down Expand Up @@ -141,11 +173,12 @@ final class ConfigurationManager: DefaultConfigurationManager {

private func updateTrackerBlockingDependencies() {
lastConfigurationInstallDate = Date()
ContentBlocking.shared.trackerDataManager.reload(etag: store.loadEtag(for: .trackerDataSet),
data: store.loadData(for: .trackerDataSet))
ContentBlocking.shared.privacyConfigurationManager.reload(etag: store.loadEtag(for: .privacyConfiguration),
data: store.loadData(for: .privacyConfiguration))
ContentBlocking.shared.contentBlockingManager.scheduleCompilation()

trackerDataManager.reload(etag: store.loadEtag(for: .trackerDataSet),
data: store.loadData(for: .trackerDataSet))
privacyConfigurationManager.reload(etag: store.loadEtag(for: .privacyConfiguration),
data: store.loadData(for: .privacyConfiguration))
contentBlockingManager.scheduleCompilation()
}

private func updateBloomFilter() async throws {
Expand Down
Loading

0 comments on commit 4f1655b

Please sign in to comment.