diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift index a04ab7d0b4..d24eefb993 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift @@ -115,7 +115,7 @@ class DataBrokerOperation: Operation, @unchecked Sendable { } } - private func filterAndSortOperationsData(brokerProfileQueriesData: [BrokerProfileQueryData], operationType: OperationType, priorityDate: Date?) -> [BrokerJobData] { + static func filterAndSortOperationsData(brokerProfileQueriesData: [BrokerProfileQueryData], operationType: OperationType, priorityDate: Date?) -> [BrokerJobData] { let operationsData: [BrokerJobData] switch operationType { @@ -131,8 +131,8 @@ class DataBrokerOperation: Operation, @unchecked Sendable { if let priorityDate = priorityDate { filteredAndSortedOperationsData = operationsData - .filter { $0.preferredRunDate != nil && $0.preferredRunDate! <= priorityDate } - .sorted { $0.preferredRunDate! < $1.preferredRunDate! } + .filtered(using: priorityDate) + .sortedByPreferredRunDate() } else { filteredAndSortedOperationsData = operationsData } @@ -152,9 +152,9 @@ class DataBrokerOperation: Operation, @unchecked Sendable { let brokerProfileQueriesData = allBrokerProfileQueryData.filter { $0.dataBroker.id == dataBrokerID } - let filteredAndSortedOperationsData = filterAndSortOperationsData(brokerProfileQueriesData: brokerProfileQueriesData, - operationType: operationType, - priorityDate: priorityDate) + let filteredAndSortedOperationsData = Self.filterAndSortOperationsData(brokerProfileQueriesData: brokerProfileQueriesData, + operationType: operationType, + priorityDate: priorityDate) Logger.dataBrokerProtection.debug("filteredAndSortedOperationsData count: \(filteredAndSortedOperationsData.count, privacy: .public) for brokerID \(self.dataBrokerID, privacy: .public)") @@ -215,3 +215,40 @@ class DataBrokerOperation: Operation, @unchecked Sendable { } } // swiftlint:enable explicit_non_final_class + +extension Array where Element == BrokerJobData { + /// Filters jobs based on their preferred run date: + /// - Opt-out jobs with no preferred run date are included. + /// - Jobs with a preferred run date on or before the priority date are included. + /// + /// Note: Opt-out jobs without a preferred run date may be: + /// 1. From child brokers (skipped during runOptOutOperation) + /// 2. From former child brokers now acting as parent brokers (processed if extractedProfile hasn't been removed) + func filtered(using priorityDate: Date) -> [BrokerJobData] { + filter { jobData in + guard let preferredRunDate = jobData.preferredRunDate else { + return jobData is OptOutJobData + } + + return preferredRunDate <= priorityDate + } + } + + /// Sorts BrokerJobData array based on their preferred run dates. + /// - Jobs with non-nil preferred run dates are sorted in ascending order (earliest date first). + /// - Opt-out jobs with nil preferred run dates come last, maintaining their original relative order. + func sortedByPreferredRunDate() -> [BrokerJobData] { + sorted { lhs, rhs in + switch (lhs.preferredRunDate, rhs.preferredRunDate) { + case (nil, nil): + return false + case (_, nil): + return true + case (nil, _): + return false + case (let lhsRunDate?, let rhsRunDate?): + return lhsRunDate < rhsRunDate + } + } + } +} diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift index 6bb37ceb99..eef9280e4c 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift @@ -291,11 +291,11 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager { } guard extractedProfile.removedDate == nil else { - Logger.dataBrokerProtection.debug("Profile already extracted, skipping...") + Logger.dataBrokerProtection.debug("Profile already removed, skipping...") return } - guard let optOutStep = brokerProfileQueryData.dataBroker.optOutStep(), optOutStep.optOutType != .parentSiteOptOut else { + guard !brokerProfileQueryData.dataBroker.performsOptOutWithinParent() else { Logger.dataBrokerProtection.debug("Broker opts out in parent, skipping...") return } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift index dae2ad1b09..e787f119b7 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift @@ -81,8 +81,10 @@ struct OperationPreferredDateCalculator { return date.now.addingTimeInterval(calculateNextRunDateOnError(schedulingConfig: schedulingConfig, historyEvents: historyEvents)) case .optOutStarted, .scanStarted, .noMatchFound: return currentPreferredRunDate - case .optOutConfirmed, .optOutRequested: + case .optOutConfirmed: return nil + case .optOutRequested: + return date.now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift index 7534b98ee9..22798d7eb3 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift @@ -172,6 +172,10 @@ extension Date { static func nowMinus(hours: Int) -> Date { Calendar.current.date(byAdding: .hour, value: -hours, to: Date()) ?? Date() } + + static func nowPlus(hours: Int) -> Date { + nowMinus(hours: -hours) + } } final class DataBrokerProtectionStatsPixels: StatsPixels { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json index cbe6abc734..b90a1fae58 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json @@ -1,76 +1,147 @@ { - "name": "Kwold", - "url": "kwold.com", - "version": "0.4.0", - "parent": "verecor.com", - "addedDatetime": 1702965600000, - "optOutUrl": "https://kwold.com/ns/control/privacy", - "steps": [ - { - "stepType": "scan", - "scanType": "templatedUrl", - "actions": [ + "name": "Kwold", + "url": "kwold.com", + "version": "0.5.0", + "addedDatetime": 1702965600000, + "optOutUrl": "https://kwold.com/ns/control/privacy", + "steps": [ { - "actionType": "navigate", - "id": "878e00ab-dbad-4ca9-a303-645702a36ee2", - "url": "https://kwold.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}", - "ageRange": [ - "18-30", - "31-40", - "41-50", - "51-60", - "61-70", - "71-80", - "81+" - ] + "stepType": "scan", + "scanType": "templatedUrl", + "actions": [ + { + "actionType": "navigate", + "id": "878e00ab-dbad-4ca9-a303-645702a36ee2", + "url": "https://kwold.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}", + "ageRange": [ + "18-30", + "31-40", + "41-50", + "51-60", + "61-70", + "71-80", + "81+" + ] + }, + { + "actionType": "extract", + "id": "ec9f8ae6-199e-441b-9722-ffc6737b4595", + "selector": ".card", + "noResultsSelector": "//div[@class='page-404' and h1[starts-with(text(), 'Sorry')]]", + "profile": { + "name": { + "selector": ".card-title", + "beforeText": " ~" + }, + "alternativeNamesList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Known as:']]/dd/ul[@class='list-inline m-0']/li", + "findElements": true + }, + "age": { + "beforeText": "years old", + "selector": ".card-title", + "afterText": " ~" + }, + "addressCityStateList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Has lived in:']]/dd/ul[@class='list-inline m-0']/li", + "findElements": true + }, + "relativesList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Related to:']]/dd/ul[@class='list-inline m-0']/li", + "beforeText": ",", + "findElements": true + }, + "profileUrl": { + "selector": "a", + "identifierType": "path", + "identifier": "https://kwold.com/pp/${id}" + } + } + } + ] }, { - "actionType": "extract", - "id": "ec9f8ae6-199e-441b-9722-ffc6737b4595", - "selector": ".card", - "noResultsSelector": "//div[@class='page-404' and h1[starts-with(text(), 'Sorry')]]", - "profile": { - "name": { - "selector": ".card-title", - "beforeText": " ~" - }, - "alternativeNamesList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Known as:']]/dd/ul[@class='list-inline m-0']/li", - "findElements": true - }, - "age": { - "beforeText": "years old", - "selector": ".card-title", - "afterText": " ~" - }, - "addressCityStateList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Has lived in:']]/dd/ul[@class='list-inline m-0']/li", - "findElements": true - }, - "relativesList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Related to:']]/dd/ul[@class='list-inline m-0']/li", - "beforeText": ",", - "findElements": true - }, - "profileUrl": { - "selector": "a", - "identifierType": "path", - "identifier": "https://kwold.com/pp/${id}" - } - } + "stepType": "optOut", + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://kwold.com/ns/control/privacy", + "id": "037f7920-b9e7-4214-a937-171ec641d641" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "5b9de12f-a52e-4bd0-b6ac-6884377d309b" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "48e5e7a8-af33-4629-a849-2cf926a518a3" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "bc2d26dc-3eef-478a-a04b-5671a1dbdf8b" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "7f2a685e-ddad-4c5a-8e80-a6d3a690851f" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "3a8a6e9d-c9a0-4e59-a8a4-fe4a05f3ce68" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "93ccf84a-a5ce-4dcf-8a78-143610723488" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "fcddc35b-6298-4f2b-a04c-08a2d6f7ceaa" + } + ] } - ] - }, - { - "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + ], + "schedulingConfig": { + "retryError": 48, + "confirmOptOutScan": 72, + "maintenanceScan": 120, + "maxAttempts": -1 } - ], - "schedulingConfig": { - "retryError": 48, - "confirmOptOutScan": 72, - "maintenanceScan": 120, - "maxAttempts": -1 - } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift new file mode 100644 index 0000000000..ad5a8aedb1 --- /dev/null +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift @@ -0,0 +1,99 @@ +// +// DataBrokerOperationTests.swift +// +// Copyright © 2024 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +@testable import DataBrokerProtection +import XCTest + +final class DataBrokerOperationTests: XCTestCase { + lazy var mockOptOutQueryData: [BrokerProfileQueryData] = { + let brokerId: Int64 = 1 + + let mockNilPreferredRunDateQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: nil, optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: nil)]) + } + let mockPastQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowMinus(hours: $0), optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: .nowMinus(hours: $0))]) + } + let mockFutureQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowPlus(hours: $0), optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: .nowPlus(hours: $0))]) + } + + return mockNilPreferredRunDateQueryData + mockPastQueryData + mockFutureQueryData + }() + + lazy var mockScanQueryData: [BrokerProfileQueryData] = { + let mockNilPreferredRunDateQueryData = Array(1...10).map { _ in + BrokerProfileQueryData.mock(preferredRunDate: nil) + } + let mockPastQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowMinus(hours: $0)) + } + let mockFutureQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowPlus(hours: $0)) + } + + return mockNilPreferredRunDateQueryData + mockPastQueryData + mockFutureQueryData + }() + + func testWhenFilteringOptOutOperationData_thenAllButFuturePreferredRunDateIsReturned() { + let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: nil) + let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .now) + let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .distantPast) + let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .distantFuture) + + XCTAssertEqual(operationData1.count, 30) // all jobs + XCTAssertEqual(operationData2.count, 20) // nil preferred run date + past jobs + XCTAssertEqual(operationData3.count, 10) // nil preferred run date jobs + XCTAssertEqual(operationData4.count, 30) // all jobs + } + + func testWhenFilteringScanOperationData_thenPreferredRunDatePriorToPriorityDateIsReturned() { + let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .scheduledScan, priorityDate: nil) + let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .manualScan, priorityDate: .now) + let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .scheduledScan, priorityDate: .distantPast) + let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .manualScan, priorityDate: .distantFuture) + + XCTAssertEqual(operationData1.count, 30) // all jobs + XCTAssertEqual(operationData2.count, 10) // past jobs + XCTAssertEqual(operationData3.count, 0) // no jobs + XCTAssertEqual(operationData4.count, 20) // past + future jobs + } + + func testFilteringAllOperationData() { + let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: nil) + let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .now) + let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .distantPast) + let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .distantFuture) + + XCTAssertEqual(operationData1.filter { $0 is ScanJobData }.count, 30) // all jobs + XCTAssertEqual(operationData1.filter { $0 is OptOutJobData }.count, 30) // all jobs + XCTAssertEqual(operationData1.count, 30+30) + + XCTAssertEqual(operationData2.filter { $0 is ScanJobData }.count, 10) // past jobs + XCTAssertEqual(operationData2.filter { $0 is OptOutJobData }.count, 20) // nil preferred run date + past jobs + XCTAssertEqual(operationData2.count, 10+20) + + XCTAssertEqual(operationData3.filter { $0 is ScanJobData }.count, 0) // no jobs + XCTAssertEqual(operationData3.filter { $0 is OptOutJobData }.count, 10) // nil preferred run date jobs + XCTAssertEqual(operationData3.count, 0+10) + + XCTAssertEqual(operationData4.filter { $0 is ScanJobData }.count, 20) // past + future jobs + XCTAssertEqual(operationData4.filter { $0 is OptOutJobData }.count, 30) // all jobs + XCTAssertEqual(operationData4.count, 20+30) + } +} diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift index 5e772ac0ee..869c1e855a 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift @@ -369,24 +369,17 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { } } - func testWhenRemovedProfileIsFound_thenOptOutConfirmedIsAddedRemoveDateIsUpdatedAndPreferredRunDateIsSetToNil() async { + func testWhenRemovedProfileIsFound_thenOptOutConfirmedIsAddedRemoveDateIsUpdated() async { do { - let extractedProfileId: Int64 = 1 - let brokerId: Int64 = 1 - let profileQueryId: Int64 = 1 - let mockHistoryEvent = HistoryEvent(extractedProfileId: extractedProfileId, brokerId: brokerId, profileQueryId: profileQueryId, type: .optOutRequested) - let mockBrokerProfileQuery = BrokerProfileQueryData( - dataBroker: .mock, - profileQuery: .mock, - scanJobData: .mock, - optOutJobData: [.mock(with: .mockWithoutRemovedDate, preferredRunDate: Date(), historyEvents: [mockHistoryEvent])] - ) - mockWebOperationRunner.scanResults = [.mockWithoutId] - mockDatabase.brokerProfileQueryDataToReturn = [mockBrokerProfileQuery] _ = try await sut.runScanOperation( on: mockWebOperationRunner, - brokerProfileQueryData: mockBrokerProfileQuery, + brokerProfileQueryData: .init( + dataBroker: .mock, + profileQuery: .mock, + scanJobData: .mock, + optOutJobData: [OptOutJobData.mock(with: .mockWithoutRemovedDate)] + ), database: mockDatabase, notificationCenter: .default, pixelHandler: MockDataBrokerProtectionPixelsHandler(), @@ -396,8 +389,6 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertTrue(mockDatabase.optOutEvents.contains(where: { $0.type == .optOutConfirmed })) XCTAssertTrue(mockDatabase.wasUpdateRemoveDateCalled) XCTAssertNotNil(mockDatabase.extractedProfileRemovedDate) - XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) - XCTAssertNil(mockDatabase.lastPreferredRunDateOnOptOut) } catch { XCTFail("Should not throw") } @@ -841,7 +832,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: Date().addingTimeInterval(schedulingConfig.confirmOptOutScan.hoursToSeconds))) } - func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToNil() throws { + func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToMaintenance() throws { let brokerId: Int64 = 1 let profileQueryId: Int64 = 1 let extractedProfileId: Int64 = 1 @@ -851,7 +842,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { try sut.updateOperationDataDates(origin: .scan, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId, schedulingConfig: schedulingConfig, database: mockDatabase) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertNil(mockDatabase.lastPreferredRunDateOnOptOut) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds))) } func testWhenUpdatingDatesAndLastEventIsMatchesFound_thenWeSetScanPreferredDateToMaintanence() throws { @@ -918,7 +909,9 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { // If the date is not going to be set, we don't call the database function XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) + + XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.maintenanceScan.hoursToSeconds))) } func testUpdatingScanDateFromScan_thenScanDoesNotRespectMostRecentDate() throws { @@ -942,8 +935,10 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { try sut.updateOperationDataDates(origin: .scan, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId, schedulingConfig: config, database: mockDatabase) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: expectedPreferredRunDate), "\(String(describing: mockDatabase.lastPreferredRunDateOnScan)) is not equal to \(expectedPreferredRunDate)") + + XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.maintenanceScan.hoursToSeconds))) } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift index 1308534ca3..0fbc1936ed 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift @@ -117,6 +117,13 @@ extension BrokerProfileQueryData { return [broker1Data, broker2Data, broker3Data] } + static func createOptOutJobData(extractedProfileId: Int64, brokerId: Int64, profileQueryId: Int64, preferredRunDate: Date?) -> OptOutJobData { + + let extractedProfile = ExtractedProfile(id: extractedProfileId) + + return OptOutJobData(brokerId: brokerId, profileQueryId: profileQueryId, createdDate: .now, preferredRunDate: preferredRunDate, historyEvents: [], attemptCount: 0, extractedProfile: extractedProfile) + } + static func createOptOutJobData(extractedProfileId: Int64, brokerId: Int64, profileQueryId: Int64, startEventHoursAgo: Int, requestEventHoursAgo: Int, jobCreatedHoursAgo: Int) -> OptOutJobData { let extractedProfile = ExtractedProfile(id: extractedProfileId) diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift index 555c19a9e7..c569cf2eea 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift @@ -25,7 +25,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { private let schedulingConfig = DataBrokerScheduleConfig( retryError: 48, confirmOptOutScan: 2000, - maintenanceScan: 3000, + maintenanceScan: 120, maxAttempts: 3 ) @@ -498,9 +498,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) } - func testOptOutConfirmedWithCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil - + func testOptOutConfirmedWithCurrentPreferredDate_thenOptOutIsNotScheduled() throws { let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -509,18 +507,16 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertNil(proposedOptOutDate) } - func testOptOutConfirmedWithoutCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil - + func testOptOutConfirmedWithoutCurrentPreferredDate_thenOptOutIsNotScheduled() throws { let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -529,17 +525,17 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertNil(proposedOptOutDate) } - func testOptOutRequestedWithCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil + func testOptOutRequestedWithCurrentPreferredDate_thenOptOutIsNotScheduled() throws { + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -549,17 +545,18 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) } - func testOptOutRequestedWithoutCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil + func testOptOutRequestedWithoutCurrentPreferredDate_thenOptOutIsNotScheduled() throws { + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -569,13 +566,14 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) } func testScanStarted_thenOptOutDoesNotChange() throws { @@ -758,6 +756,69 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } } + func testChildBrokerTurnsParentBroker_whenFirstOptOutSucceeds_thenOptOutDateIsNotScheduled() throws { + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) + + let historyEvents = [ + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .optOutRequested), + ] + let calculator = OperationPreferredDateCalculator() + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1, + date: MockDate()) + + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) + } + + func testChildBrokerTurnsParentBroker_whenFirstOptOutFails_thenOptOutIsScheduled() throws { + let expectedOptOutDate = Calendar.current.date(byAdding: .hour, value: 2, to: Date())! + + let historyEvents = [ + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .error(error: .malformedURL)), + ] + let calculator = OperationPreferredDateCalculator() + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1) + + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: proposedOptOutDate)) + } + + func testRequestedOptOut_whenProfileReappears_thenOptOutIsScheduled() throws { + let expectedOptOutDate = Date() + + let historyEvents = [ + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .optOutRequested, + date: .nowMinus(hours: 24*10)), + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .reAppearence), + ] + let calculator = OperationPreferredDateCalculator() + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: .distantFuture, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1) + + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: proposedOptOutDate)) + } + func testOptOutStartedWithRecentDate_thenOptOutDateDoesNotChange() throws { let expectedOptOutDate = Date() @@ -779,8 +840,6 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } func testOptOutConfirmedWithRecentDate_thenOptOutDateDoesNotChange() throws { - let expectedOptOutDate: Date? = nil - let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -789,17 +848,17 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertNil(proposedOptOutDate) } - func testOptOutRequestedWithRecentDate_thenOptOutDateDoesNotChange() throws { - let expectedOptOutDate: Date? = nil + func testOptOutRequestedWithRecentDate_thenOutOutIsNotScheduled() throws { + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -809,13 +868,14 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) } func testScanStartedWithRecentDate_thenOptOutDateDoesNotChange() throws {