From a6a4021859c9b8afd08cd6483ba21849cc7509a6 Mon Sep 17 00:00:00 2001 From: Brian Posey <15091170+btposey@users.noreply.github.com> Date: Fri, 28 Feb 2025 17:22:39 -0500 Subject: [PATCH 01/10] Upgrade case sync to use last sync date Jira ticket: CAMS-421 Co-authored-by: James Brooks <12275865+jamesobrooks@users.noreply.github.com> Co-authored-by: Brian Posey <15091170+btposey@users.noreply.github.com> --- .../dataflows/import/sync-cases.ts | 7 +- .../adapters/gateways/cases.local.gateway.ts | 12 +-- .../gateways/dxtr/cases.dxtr.gateway.test.ts | 90 +++++++++---------- .../gateways/dxtr/cases.dxtr.gateway.ts | 65 ++++++-------- .../lib/use-cases/cases/cases.interface.d.ts | 5 +- .../dataflows/cases-runtime-state.test.ts | 67 +------------- .../dataflows/cases-runtime-state.ts | 31 ++----- .../use-cases/dataflows/sync-cases.test.ts | 60 +++++-------- backend/lib/use-cases/dataflows/sync-cases.ts | 20 +++-- backend/lib/use-cases/gateways.types.ts | 2 +- common/src/feature-flags.ts | 1 + 11 files changed, 126 insertions(+), 234 deletions(-) diff --git a/backend/function-apps/dataflows/import/sync-cases.ts b/backend/function-apps/dataflows/import/sync-cases.ts index 95d791d6d..84e256769 100644 --- a/backend/function-apps/dataflows/import/sync-cases.ts +++ b/backend/function-apps/dataflows/import/sync-cases.ts @@ -49,7 +49,10 @@ const TIMER_TRIGGER = buildFunctionName(MODULE_NAME, 'timerTrigger'); async function handleStart(startMessage: StartMessage, invocationContext: InvocationContext) { try { const context = await ContextCreator.getApplicationContext({ invocationContext }); - const { events, lastTxId } = await SyncCases.getCaseIds(context, startMessage['lastTxId']); + const { events, lastSyncDate } = await SyncCases.getCaseIds( + context, + startMessage['lastSyncDate'], + ); if (!events.length) return; @@ -64,7 +67,7 @@ async function handleStart(startMessage: StartMessage, invocationContext: Invoca } invocationContext.extraOutputs.set(PAGE, pages); - await CasesRuntimeState.storeRuntimeState(context, lastTxId); + await CasesRuntimeState.storeRuntimeState(context, lastSyncDate); } catch (originalError) { invocationContext.extraOutputs.set( DLQ, diff --git a/backend/lib/adapters/gateways/cases.local.gateway.ts b/backend/lib/adapters/gateways/cases.local.gateway.ts index a872cc6a9..9be889702 100644 --- a/backend/lib/adapters/gateways/cases.local.gateway.ts +++ b/backend/lib/adapters/gateways/cases.local.gateway.ts @@ -1,8 +1,4 @@ -import { - CasesInterface, - CasesSyncMeta, - TransactionIdRangeForDate, -} from '../../use-cases/cases/cases.interface'; +import { CasesInterface, TransactionIdRangeForDate } from '../../use-cases/cases/cases.interface'; import { ApplicationContext } from '../types/basic'; import { GatewayHelper } from './gateway-helper'; import { getMonthDayYearStringFromDate } from '../utils/date-helper'; @@ -124,10 +120,10 @@ export class CasesLocalGateway implements CasesInterface { } } - public async getCaseIdsAndMaxTxIdToSync( + public async getUpdatedCaseIds( _applicationContext: ApplicationContext, - _lastTxId: string, - ): Promise { + _start: string, + ): Promise { throw new Error('Not implemented'); } diff --git a/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.test.ts b/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.test.ts index 332ed97ec..92f24c5e5 100644 --- a/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.test.ts +++ b/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.test.ts @@ -757,56 +757,6 @@ describe('Test DXTR Gateway', () => { }); }); - describe('getCaseIdsAndMaxTxIdToSync tests', () => { - test('should return array of case ids and a max tx id', async () => { - const mockRecords = [ - { caseId: MockData.getCaseBasics().caseId, maxTxId: 10 }, - { caseId: MockData.getCaseBasics().caseId, maxTxId: 9 }, - { caseId: MockData.getCaseBasics().caseId, maxTxId: 8 }, - { caseId: MockData.getCaseBasics().caseId, maxTxId: 7 }, - ]; - const mockResults: QueryResults = { - success: true, - results: { - recordset: mockRecords, - }, - message: '', - }; - const expectedReturn = { - caseIds: mockRecords.map((rec) => rec.caseId), - lastTxId: '10', - }; - - querySpy.mockImplementationOnce(async () => { - return Promise.resolve(mockResults); - }); - - const result = await testCasesDxtrGateway.getCaseIdsAndMaxTxIdToSync(applicationContext, '0'); - expect(result).toEqual(expectedReturn); - }); - - test('should return an empty array and the existing max tx id', async () => { - const mockResults: QueryResults = { - success: true, - results: { - recordset: [], - }, - message: '', - }; - const expectedReturn = { - caseIds: [], - lastTxId: '0', - }; - - querySpy.mockImplementationOnce(async () => { - return Promise.resolve(mockResults); - }); - - const result = await testCasesDxtrGateway.getCaseIdsAndMaxTxIdToSync(applicationContext, '0'); - expect(result).toEqual(expectedReturn); - }); - }); - describe('findTransactionIdRangeForDate', () => { const dateRangeMock = (_context, _config, query: string, params: DbTableFieldSpec[]) => { const mockTxMap = new Map([ @@ -898,4 +848,44 @@ describe('Test DXTR Gateway', () => { }, ); }); + + describe('getUpdatedCaseIds', () => { + test('should return a list of updated case ids', async () => { + const recordset = MockData.buildArray(MockData.randomCaseId, 100).map((caseId) => { + return { caseId }; + }); + + const executeResults: QueryResults = { + success: true, + results: { + recordset, + }, + message: '', + }; + + const expectedReturn = recordset.map((record) => record.caseId); + + querySpy.mockImplementationOnce(async () => { + return executeResults; + }); + + const startDate = new Date().toISOString(); + const actual = await testCasesDxtrGateway.getUpdatedCaseIds(applicationContext, startDate); + expect(actual).toEqual(expectedReturn); + }); + + test('should return an empty array', async () => { + const mockResults: QueryResults = { + success: true, + results: { + recordset: [], + }, + message: '', + }; + + querySpy.mockReturnValue(mockResults); + const actual = await testCasesDxtrGateway.getUpdatedCaseIds(applicationContext, 'foo'); + expect(actual).toEqual([]); + }); + }); }); diff --git a/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.ts b/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.ts index 690a85178..41d59021c 100644 --- a/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.ts +++ b/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.ts @@ -1,7 +1,6 @@ import * as mssql from 'mssql'; import { CasesInterface, - CasesSyncMeta, TransactionIdRangeForDate, } from '../../../use-cases/cases/cases.interface'; import { ApplicationContext } from '../../types/basic'; @@ -30,6 +29,7 @@ const orderToTransferCode = 'CTO'; const NOT_FOUND = -1; type RawCaseIdAndMaxId = { caseId: string; maxTxId: number }; +type CaseIdRecord = { caseId: string }; export function getCaseIdParts(caseId: string) { const parts = caseId.split('-'); @@ -472,58 +472,45 @@ export default class CasesDxtrGateway implements CasesInterface { return bCase; } - async getCaseIdsAndMaxTxIdToSync( - context: ApplicationContext, - lastTxId: string, - ): Promise { - const input: DbTableFieldSpec[] = []; - - input.push({ - name: 'txId', - type: mssql.BigInt, - value: parseInt(lastTxId), + /** + * getUpdatedCaseIds + * + * Gets the case ids for all cases with LAST_UPDATE_DATE values greater than the provided date. + * 2025-02-23 06:35:30.453 + * + * @param {string} start The date and time to begin checking for LAST_UPDATE_DATE values. + * @returns {string[]} A list of case ids for updated cases. + */ + async getUpdatedCaseIds(context: ApplicationContext, start: string): Promise { + const params: DbTableFieldSpec[] = []; + params.push({ + name: 'start', + type: mssql.DateTime, + value: start, }); const query = ` - SELECT - CONCAT(CS_DIV.CS_DIV_ACMS, '-', C.CASE_ID) AS caseId, - MAX(T.TX_ID) as maxTxId - FROM AO_TX T - JOIN AO_CS C ON C.CS_CASEID = T.CS_CASEID AND C.COURT_ID = T.COURT_ID + SELECT CONCAT(CS_DIV.CS_DIV_ACMS, '-', C.CASE_ID) AS caseId + FROM AO_CS C JOIN AO_CS_DIV AS CS_DIV ON C.CS_DIV = CS_DIV.CS_DIV - WHERE T.TX_ID > @txId - GROUP BY CS_DIV.CS_DIV_ACMS, C.CASE_ID - ORDER BY MAX(T.TX_ID) DESC + WHERE C.LAST_UPDATE_DATE > '@start' `; const queryResult: QueryResults = await executeQuery( context, context.config.dxtrDbConfig, query, - input, + params, ); - const results = handleQueryResult( + const results = handleQueryResult( context, queryResult, MODULE_NAME, - this.caseIdsAndMaxTxIdCallback, + this.getUpdatedCaseIdsCallback, ); - let meta: CasesSyncMeta; - if (results.length) { - meta = { - caseIds: results.map((bCase) => bCase.caseId), - lastTxId: results[0].maxTxId.toString(), - }; - } else { - meta = { - caseIds: [], - lastTxId, - }; - } - - return meta; + return results.map((record) => record.caseId); } private async queryCase( @@ -902,4 +889,10 @@ export default class CasesDxtrGateway implements CasesInterface { return (queryResult.results as mssql.IResult).recordset; } + + getUpdatedCaseIdsCallback(applicationContext: ApplicationContext, queryResult: QueryResults) { + applicationContext.logger.debug(MODULE_NAME, `Results received from DXTR`); + + return (queryResult.results as mssql.IResult).recordset; + } } diff --git a/backend/lib/use-cases/cases/cases.interface.d.ts b/backend/lib/use-cases/cases/cases.interface.d.ts index 2c2658e6c..18f5ce9a3 100644 --- a/backend/lib/use-cases/cases/cases.interface.d.ts +++ b/backend/lib/use-cases/cases/cases.interface.d.ts @@ -26,10 +26,7 @@ export interface CasesInterface { getSuggestedCases(applicationContext: ApplicationContext, caseId: string): Promise; - getCaseIdsAndMaxTxIdToSync( - applicationContext: ApplicationContext, - lastTxId: string, - ): Promise; + getUpdatedCaseIds(applicationContext: ApplicationContext, caseId: string): Promise; findTransactionIdRangeForDate( context: ApplicationContext, diff --git a/backend/lib/use-cases/dataflows/cases-runtime-state.test.ts b/backend/lib/use-cases/dataflows/cases-runtime-state.test.ts index ae4793875..a48dfc63f 100644 --- a/backend/lib/use-cases/dataflows/cases-runtime-state.test.ts +++ b/backend/lib/use-cases/dataflows/cases-runtime-state.test.ts @@ -1,7 +1,6 @@ import { ApplicationContext } from '../../adapters/types/basic'; import { createMockApplicationContext } from '../../testing/testing-utilities'; import { MockMongoRepository } from '../../testing/mock-gateways/mock-mongo.repository'; -import { CasesLocalGateway } from '../../adapters/gateways/cases.local.gateway'; import { CasesSyncState } from '../gateways.types'; import CasesRuntimeState from './cases-runtime-state'; @@ -16,85 +15,27 @@ describe('storeRuntimeState tests', () => { jest.restoreAllMocks(); }); - test('should persist a new sync state when transaction id is not provided', async () => { - jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(undefined); - const upsertSpy = jest - .spyOn(MockMongoRepository.prototype, 'upsert') - .mockResolvedValue(undefined); - const txId = '1001'; - jest.spyOn(CasesLocalGateway.prototype, 'findMaxTransactionId').mockResolvedValue(txId); - - await CasesRuntimeState.storeRuntimeState(context); - expect(upsertSpy).toHaveBeenCalledWith({ documentType: 'CASES_SYNC_STATE', txId }); - }); - test('should persist a new sync state', async () => { jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(undefined); const upsertSpy = jest .spyOn(MockMongoRepository.prototype, 'upsert') .mockResolvedValue(undefined); - const txId = '1001'; - await CasesRuntimeState.storeRuntimeState(context, txId); + const lastSyncDate = new Date().toISOString(); + await CasesRuntimeState.storeRuntimeState(context, lastSyncDate); expect(upsertSpy).toHaveBeenCalledWith({ documentType: 'CASES_SYNC_STATE', - txId, + lastSyncDate, }); }); - test('should persist a higher transaction id', async () => { - const original: CasesSyncState = { - documentType: 'CASES_SYNC_STATE', - txId: '1000', - }; - jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(original); - const upsertSpy = jest - .spyOn(MockMongoRepository.prototype, 'upsert') - .mockResolvedValue(undefined); - - const txId = '1001'; - await CasesRuntimeState.storeRuntimeState(context, txId); - expect(upsertSpy).toHaveBeenCalledWith({ ...original, txId }); - }); - - test('should not persist a lower transaction id', async () => { - const original: CasesSyncState = { - documentType: 'CASES_SYNC_STATE', - txId: '1000', - }; - jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(original); - const upsertSpy = jest - .spyOn(MockMongoRepository.prototype, 'upsert') - .mockRejectedValue(new Error('this should not be called')); - - const txId = '1'; - await CasesRuntimeState.storeRuntimeState(context, txId); - expect(upsertSpy).not.toHaveBeenCalled(); - }); - test('should throw CamsError', async () => { const original: CasesSyncState = { documentType: 'CASES_SYNC_STATE', - txId: '1000', + lastSyncDate: '1000', }; jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(original); jest.spyOn(MockMongoRepository.prototype, 'upsert').mockRejectedValue(new Error('some error')); await expect(CasesRuntimeState.storeRuntimeState(context, '1001')).resolves.toBeUndefined(); }); - - test('should throw error if gateway throws', async () => { - jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(undefined); - jest - .spyOn(CasesLocalGateway.prototype, 'findMaxTransactionId') - .mockRejectedValue(new Error('some error')); - - await expect(CasesRuntimeState.storeRuntimeState(context)).resolves.toBeUndefined(); - }); - - test('should throw error if max transaction id cannot be determined', async () => { - jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(undefined); - jest.spyOn(CasesLocalGateway.prototype, 'findMaxTransactionId').mockResolvedValue(undefined); - - await expect(CasesRuntimeState.storeRuntimeState(context)).resolves.toBeUndefined(); - }); }); diff --git a/backend/lib/use-cases/dataflows/cases-runtime-state.ts b/backend/lib/use-cases/dataflows/cases-runtime-state.ts index 07209fcca..5926c7199 100644 --- a/backend/lib/use-cases/dataflows/cases-runtime-state.ts +++ b/backend/lib/use-cases/dataflows/cases-runtime-state.ts @@ -1,34 +1,19 @@ import { ApplicationContext } from '../../adapters/types/basic'; -import Factory, { getCasesGateway } from '../../factory'; -import { UnknownError } from '../../common-errors/unknown-error'; +import Factory from '../../factory'; import { CasesSyncState } from '../gateways.types'; import { getCamsError } from '../../common-errors/error-utilities'; const MODULE_NAME = 'CASE-RUNTIME-STATE'; -async function storeRuntimeState(context: ApplicationContext, lastTxId?: string) { +async function storeRuntimeState(context: ApplicationContext, lastSyncDate: string) { const runtimeStateRepo = Factory.getCasesSyncStateRepo(context); try { - const syncState = await runtimeStateRepo.read('CASES_SYNC_STATE'); - context.logger.info(MODULE_NAME, `Retrieved runtime state: ${syncState}.`); - if (!lastTxId) { - const gateway = getCasesGateway(context); - lastTxId = await gateway.findMaxTransactionId(context); - if (!lastTxId) { - throw new UnknownError(MODULE_NAME, { - message: 'Failed to determine the maximum transaction id.', - }); - } - } - if (!syncState || lastTxId > syncState.txId) { - const newSyncState: CasesSyncState = { - ...syncState, - documentType: 'CASES_SYNC_STATE', - txId: lastTxId, - }; - await runtimeStateRepo.upsert(newSyncState); - context.logger.info(MODULE_NAME, `Wrote runtime state: ${newSyncState}.`); - } + const newSyncState: CasesSyncState = { + documentType: 'CASES_SYNC_STATE', + lastSyncDate, + }; + await runtimeStateRepo.upsert(newSyncState); + context.logger.info(MODULE_NAME, `Wrote runtime state: ${newSyncState}.`); } catch (originalError) { const error = getCamsError( originalError, diff --git a/backend/lib/use-cases/dataflows/sync-cases.test.ts b/backend/lib/use-cases/dataflows/sync-cases.test.ts index a1a755c43..627b1f436 100644 --- a/backend/lib/use-cases/dataflows/sync-cases.test.ts +++ b/backend/lib/use-cases/dataflows/sync-cases.test.ts @@ -3,7 +3,6 @@ import { MockMongoRepository } from '../../testing/mock-gateways/mock-mongo.repo import { createMockApplicationContext } from '../../testing/testing-utilities'; import { CasesSyncState } from '../gateways.types'; import { CasesLocalGateway } from '../../adapters/gateways/cases.local.gateway'; -import { CasesSyncMeta } from '../cases/cases.interface'; import MockData from '../../../../common/src/cams/test-utilities/mock-data'; import { ApplicationContext } from '../../adapters/types/basic'; @@ -20,66 +19,51 @@ describe('getCaseIds tests', () => { expect(actual).toEqual({ events: [] }); }); - test('should return events to sync and last transaction id', async () => { - const initialTxId = '0'; - const lastTxId = '1000'; - const gatewayResponse: CasesSyncMeta = { - caseIds: MockData.buildArray(MockData.randomCaseId, 3), - lastTxId, - }; + test('should return events to sync and last sync date', async () => { + const lastSyncDate = '2025-01-01'; + const gatewayResponse = MockData.buildArray(MockData.randomCaseId, 3); const getIdSpy = jest - .spyOn(CasesLocalGateway.prototype, 'getCaseIdsAndMaxTxIdToSync') + .spyOn(CasesLocalGateway.prototype, 'getUpdatedCaseIds') .mockResolvedValue(gatewayResponse); const syncState: CasesSyncState = { documentType: 'CASES_SYNC_STATE', - txId: initialTxId, + lastSyncDate, }; jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(syncState); - jest - .spyOn(MockMongoRepository.prototype, 'upsert') - .mockRejectedValue(new Error('this should not be called')); const actual = await SyncCases.getCaseIds(context); const expected = { - events: gatewayResponse.caseIds.map((caseId) => { + events: gatewayResponse.map((caseId) => { return { caseId, type: 'CASE_CHANGED' }; }), - lastTxId, }; - expect(getIdSpy).toHaveBeenCalledWith(expect.anything(), syncState.txId); - expect(actual).toEqual(expected); + expect(getIdSpy).toHaveBeenCalledWith(expect.anything(), lastSyncDate); + expect(actual).toEqual(expect.objectContaining(expected)); + expect(Date.parse(actual.lastSyncDate)).toBeGreaterThan(Date.parse(lastSyncDate)); }); - test('should use provided lastRunTxId', async () => { + test('should use provided lastSyncDate if provided', async () => { + const lastSyncDate = '2025-02-01 23:59:59'; jest .spyOn(MockMongoRepository.prototype, 'read') .mockRejectedValue(new Error('this should not be called')); - const lastRunTxId = '12345678901234567890'; - const lastTxId = '98765432109876543210'; - const gatewayResponse: CasesSyncMeta = { - caseIds: MockData.buildArray(MockData.randomCaseId, 3), - lastTxId, - }; - - const getIdSpy = jest - .spyOn(CasesLocalGateway.prototype, 'getCaseIdsAndMaxTxIdToSync') - .mockResolvedValue(gatewayResponse); - - const actual = await SyncCases.getCaseIds(context, '12345678901234567890'); + const mockCaseIds = MockData.buildArray(MockData.randomCaseId, 3); - const expected = { - events: gatewayResponse.caseIds.map((caseId) => { - return { caseId, type: 'CASE_CHANGED' }; - }), - lastTxId, - }; + const getUpdatedSpy = jest + .spyOn(CasesLocalGateway.prototype, 'getUpdatedCaseIds') + .mockResolvedValue(mockCaseIds); - expect(getIdSpy).toHaveBeenCalledWith(expect.anything(), lastRunTxId); - expect(actual).toEqual(expected); + const actual = await SyncCases.getCaseIds(context, lastSyncDate); + expect(getUpdatedSpy).toHaveBeenCalled(); + expect(actual).toEqual({ + events: expect.anything(), + lastSyncDate: expect.any(String), + }); + expect(Date.parse(actual.lastSyncDate)).toBeGreaterThan(Date.parse(lastSyncDate)); }); }); diff --git a/backend/lib/use-cases/dataflows/sync-cases.ts b/backend/lib/use-cases/dataflows/sync-cases.ts index 17ec21b56..0f259e97f 100644 --- a/backend/lib/use-cases/dataflows/sync-cases.ts +++ b/backend/lib/use-cases/dataflows/sync-cases.ts @@ -4,34 +4,36 @@ import Factory, { getCasesGateway } from '../../factory'; import { getCamsError } from '../../common-errors/error-utilities'; import { CasesSyncState } from '../gateways.types'; import { randomUUID } from 'node:crypto'; +import { CamsError } from '../../common-errors/cams-error'; const MODULE_NAME = 'SYNC-CASES-USE-CASE'; -async function getCaseIds(context: ApplicationContext, lastRunTxId?: string) { +async function getCaseIds(context: ApplicationContext, lastSyncDate?: string) { try { - const runtimeStateRepo = Factory.getCasesSyncStateRepo(context); + const now = new Date().toISOString(); let syncState: CasesSyncState; - if (lastRunTxId) { + if (lastSyncDate) { syncState = { id: randomUUID(), documentType: 'CASES_SYNC_STATE', - txId: lastRunTxId, + lastSyncDate, }; } else { + const runtimeStateRepo = Factory.getCasesSyncStateRepo(context); syncState = await runtimeStateRepo.read('CASES_SYNC_STATE'); } const casesGateway = getCasesGateway(context); - const { caseIds, lastTxId } = await casesGateway.getCaseIdsAndMaxTxIdToSync( - context, - syncState.txId, - ); + const start = syncState.lastSyncDate; + if (!start) throw new CamsError(MODULE_NAME); + const caseIds = await casesGateway.getUpdatedCaseIds(context, syncState.lastSyncDate); const events: CaseSyncEvent[] = caseIds.map((caseId) => { return { type: 'CASE_CHANGED', caseId }; }); - return { events, lastTxId: lastTxId }; + + return { events, lastSyncDate: now }; } catch (originalError) { const error = getCamsError(originalError, MODULE_NAME); context.logger.camsError(error); diff --git a/backend/lib/use-cases/gateways.types.ts b/backend/lib/use-cases/gateways.types.ts index 27ee3be7c..c4870b491 100644 --- a/backend/lib/use-cases/gateways.types.ts +++ b/backend/lib/use-cases/gateways.types.ts @@ -177,7 +177,7 @@ export type OrderSyncState = RuntimeState & { export type CasesSyncState = RuntimeState & { documentType: 'CASES_SYNC_STATE'; - txId: string; + lastSyncDate?: string; }; export type OfficeStaffSyncState = RuntimeState & { diff --git a/common/src/feature-flags.ts b/common/src/feature-flags.ts index 41e1b72f4..710974d6c 100644 --- a/common/src/feature-flags.ts +++ b/common/src/feature-flags.ts @@ -15,4 +15,5 @@ export const testFeatureFlags: FeatureFlagSet = { 'case-search-enabled': true, 'case-notes-enabled': true, 'privileged-identity-management': true, + 'enhanced-case-sync': true, }; From 0ef0b002638cacbeaf2eec5b155371b499364ca2 Mon Sep 17 00:00:00 2001 From: Brian Posey <15091170+btposey@users.noreply.github.com> Date: Fri, 28 Feb 2025 17:31:03 -0500 Subject: [PATCH 02/10] Pass today's date to store in case sync runtime state Jira ticket: CAMS-421 --- backend/function-apps/dataflows/import/migrate-cases.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/function-apps/dataflows/import/migrate-cases.ts b/backend/function-apps/dataflows/import/migrate-cases.ts index 02d82211f..d78b5afac 100644 --- a/backend/function-apps/dataflows/import/migrate-cases.ts +++ b/backend/function-apps/dataflows/import/migrate-cases.ts @@ -17,6 +17,7 @@ import ExportAndLoadCase from '../../../lib/use-cases/dataflows/export-and-load- import { isNotFoundError } from '../../../lib/common-errors/not-found-error'; import ApplicationContextCreator from '../../azure/application-context-creator'; import { UnknownError } from '../../../lib/common-errors/unknown-error'; +import { getTodaysIsoDate } from '../../../../common/src/date-helper'; const MODULE_NAME = 'MIGRATE-CASES'; const PAGE_SIZE = 100; @@ -236,7 +237,7 @@ async function getCaseIdsToMigrate( */ async function storeRuntimeState(invocationContext: InvocationContext) { const appContext = await ContextCreator.getApplicationContext({ invocationContext }); - return CasesRuntimeState.storeRuntimeState(appContext); + return CasesRuntimeState.storeRuntimeState(appContext, getTodaysIsoDate()); } export function setupMigrateCases() { From 34ecb077c00756ff47110a000c70a175e7ae779f Mon Sep 17 00:00:00 2001 From: James Brooks <12275865+jamesobrooks@users.noreply.github.com> Date: Tue, 4 Mar 2025 09:41:51 -0600 Subject: [PATCH 03/10] Fix query for updated case ids Jira ticket: CAMS-421 Co-authored-by: Arthur Morrow <133667008+amorrow-flexion@users.noreply.github.com> Co-authored-by: Brian Posey <15091170+btposey@users.noreply.github.com> --- backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.ts b/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.ts index 41d59021c..85a9ce69d 100644 --- a/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.ts +++ b/backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.ts @@ -493,7 +493,7 @@ export default class CasesDxtrGateway implements CasesInterface { SELECT CONCAT(CS_DIV.CS_DIV_ACMS, '-', C.CASE_ID) AS caseId FROM AO_CS C JOIN AO_CS_DIV AS CS_DIV ON C.CS_DIV = CS_DIV.CS_DIV - WHERE C.LAST_UPDATE_DATE > '@start' + WHERE C.LAST_UPDATE_DATE > @start `; const queryResult: QueryResults = await executeQuery( From 3dd10c06cbafd399b3538d5d8a3d39aa86195df3 Mon Sep 17 00:00:00 2001 From: Arthur Morrow Date: Tue, 4 Mar 2025 09:49:21 -0600 Subject: [PATCH 04/10] removed old storage account queues Jira ticket: CAMS-421 Co-authored-by: Arthur Morrow <133667008+amorrow-flexion@users.noreply.github.com> Co-authored-by: James Brooks <12275865+jamesobrooks@users.noreply.github.com> Co-authored-by: Brian Posey <15091170+btposey@users.noreply.github.com> --- .../lib/storage/storage-queues.bicep | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/ops/cloud-deployment/lib/storage/storage-queues.bicep b/ops/cloud-deployment/lib/storage/storage-queues.bicep index dad6d500c..fb28029a0 100644 --- a/ops/cloud-deployment/lib/storage/storage-queues.bicep +++ b/ops/cloud-deployment/lib/storage/storage-queues.bicep @@ -1,7 +1,5 @@ param storageAccountName string -param migrationTaskName string = 'migration-task' - resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' existing = { name: storageAccountName } @@ -15,36 +13,3 @@ resource storageAccountQueueServices 'Microsoft.Storage/storageAccounts/queueSer } } } - -resource migrationBaseQueue 'Microsoft.Storage/storageAccounts/queueServices/queues@2023-05-01' = { - parent: storageAccountQueueServices - name: migrationTaskName - properties: { - metadata: {} - } - dependsOn: [ - storageAccount - ] -} - -resource migrationFailureQueue 'Microsoft.Storage/storageAccounts/queueServices/queues@2023-05-01' = { - parent: storageAccountQueueServices - name: '${migrationTaskName}-fail' - properties: { - metadata: {} - } - dependsOn: [ - storageAccount - ] -} - -resource migrationSuccessQueue 'Microsoft.Storage/storageAccounts/queueServices/queues@2023-05-01' = { - parent: storageAccountQueueServices - name: '${migrationTaskName}-success' - properties: { - metadata: {} - } - dependsOn: [ - storageAccount - ] -} From 16818739eb4a220384805e2c535b14703eb31547 Mon Sep 17 00:00:00 2001 From: Arthur Morrow Date: Tue, 4 Mar 2025 10:25:20 -0600 Subject: [PATCH 05/10] replaced isodate with proper function cporrect interfact implementation Jira ticket: CAMS-421 --- backend/lib/use-cases/cases/cases.interface.d.ts | 2 +- backend/lib/use-cases/dataflows/sync-cases.ts | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/lib/use-cases/cases/cases.interface.d.ts b/backend/lib/use-cases/cases/cases.interface.d.ts index 18f5ce9a3..6d254bfab 100644 --- a/backend/lib/use-cases/cases/cases.interface.d.ts +++ b/backend/lib/use-cases/cases/cases.interface.d.ts @@ -26,7 +26,7 @@ export interface CasesInterface { getSuggestedCases(applicationContext: ApplicationContext, caseId: string): Promise; - getUpdatedCaseIds(applicationContext: ApplicationContext, caseId: string): Promise; + getUpdatedCaseIds(applicationContext: ApplicationContext, start: string): Promise; findTransactionIdRangeForDate( context: ApplicationContext, diff --git a/backend/lib/use-cases/dataflows/sync-cases.ts b/backend/lib/use-cases/dataflows/sync-cases.ts index 0f259e97f..65de8e890 100644 --- a/backend/lib/use-cases/dataflows/sync-cases.ts +++ b/backend/lib/use-cases/dataflows/sync-cases.ts @@ -5,13 +5,13 @@ import { getCamsError } from '../../common-errors/error-utilities'; import { CasesSyncState } from '../gateways.types'; import { randomUUID } from 'node:crypto'; import { CamsError } from '../../common-errors/cams-error'; +import { getIsoDate } from '../../../../common/src/date-helper'; const MODULE_NAME = 'SYNC-CASES-USE-CASE'; async function getCaseIds(context: ApplicationContext, lastSyncDate?: string) { try { - const now = new Date().toISOString(); - + const now = getIsoDate(new Date()); let syncState: CasesSyncState; if (lastSyncDate) { syncState = { @@ -26,7 +26,9 @@ async function getCaseIds(context: ApplicationContext, lastSyncDate?: string) { const casesGateway = getCasesGateway(context); const start = syncState.lastSyncDate; - if (!start) throw new CamsError(MODULE_NAME); + if (!start) { + throw new CamsError(MODULE_NAME); + } const caseIds = await casesGateway.getUpdatedCaseIds(context, syncState.lastSyncDate); const events: CaseSyncEvent[] = caseIds.map((caseId) => { From be172ebd832b3e6eeb58ed5ceae17d12861de8ca Mon Sep 17 00:00:00 2001 From: Arthur Morrow Date: Tue, 4 Mar 2025 10:27:49 -0600 Subject: [PATCH 06/10] reverted change to date Jira ticket: CAMS-421 Co-authored-by: James Brooks <12275865+jamesobrooks@users.noreply.github.com> Co-authored-by: Brian Posey <15091170+btposey@users.noreply.github.com> --- backend/lib/use-cases/dataflows/sync-cases.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/lib/use-cases/dataflows/sync-cases.ts b/backend/lib/use-cases/dataflows/sync-cases.ts index 65de8e890..f1d8967a2 100644 --- a/backend/lib/use-cases/dataflows/sync-cases.ts +++ b/backend/lib/use-cases/dataflows/sync-cases.ts @@ -5,13 +5,13 @@ import { getCamsError } from '../../common-errors/error-utilities'; import { CasesSyncState } from '../gateways.types'; import { randomUUID } from 'node:crypto'; import { CamsError } from '../../common-errors/cams-error'; -import { getIsoDate } from '../../../../common/src/date-helper'; const MODULE_NAME = 'SYNC-CASES-USE-CASE'; async function getCaseIds(context: ApplicationContext, lastSyncDate?: string) { try { - const now = getIsoDate(new Date()); + const now = new Date().toISOString(); + let syncState: CasesSyncState; if (lastSyncDate) { syncState = { From 2ba160c3b6626d5f8728f0e32f0795e6eff5a019 Mon Sep 17 00:00:00 2001 From: James Brooks <12275865+jamesobrooks@users.noreply.github.com> Date: Tue, 4 Mar 2025 10:29:29 -0600 Subject: [PATCH 07/10] Remove defunct feature flag --- common/src/feature-flags.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/common/src/feature-flags.ts b/common/src/feature-flags.ts index 710974d6c..41e1b72f4 100644 --- a/common/src/feature-flags.ts +++ b/common/src/feature-flags.ts @@ -15,5 +15,4 @@ export const testFeatureFlags: FeatureFlagSet = { 'case-search-enabled': true, 'case-notes-enabled': true, 'privileged-identity-management': true, - 'enhanced-case-sync': true, }; From a570f1052c24e07c5044deca7e6d8f9403f17fc1 Mon Sep 17 00:00:00 2001 From: Arthur Morrow Date: Tue, 4 Mar 2025 10:35:24 -0600 Subject: [PATCH 08/10] removed refs to case search feature flag Jira ticket: CAMS-421 --- common/src/feature-flags.ts | 1 - user-interface/src/lib/components/Header.test.tsx | 1 - 2 files changed, 2 deletions(-) diff --git a/common/src/feature-flags.ts b/common/src/feature-flags.ts index 41e1b72f4..8f7a08835 100644 --- a/common/src/feature-flags.ts +++ b/common/src/feature-flags.ts @@ -12,7 +12,6 @@ export const testFeatureFlags: FeatureFlagSet = { 'chapter-eleven-enabled': true, 'transfer-orders-enabled': true, 'consolidations-enabled': true, - 'case-search-enabled': true, 'case-notes-enabled': true, 'privileged-identity-management': true, }; diff --git a/user-interface/src/lib/components/Header.test.tsx b/user-interface/src/lib/components/Header.test.tsx index f4c3080fe..cb55fbbb0 100644 --- a/user-interface/src/lib/components/Header.test.tsx +++ b/user-interface/src/lib/components/Header.test.tsx @@ -15,7 +15,6 @@ describe('Header', () => { }); vi.spyOn(FeatureFlags, 'default').mockReturnValue({ 'transfer-orders-enabled': true, - 'case-search-enabled': true, }); beforeEach(() => { From f7c951da4a3dac26288eca234ee7fb8381e520f5 Mon Sep 17 00:00:00 2001 From: James Brooks <12275865+jamesobrooks@users.noreply.github.com> Date: Tue, 4 Mar 2025 10:52:42 -0600 Subject: [PATCH 09/10] Fix cases-runtime-state test and logging Jira ticket: CAMS-421 Co-authored-by: Brian Posey <15091170+btposey@users.noreply.github.com> --- .../dataflows/cases-runtime-state.test.ts | 20 +++++++++++++++++-- .../dataflows/cases-runtime-state.ts | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/backend/lib/use-cases/dataflows/cases-runtime-state.test.ts b/backend/lib/use-cases/dataflows/cases-runtime-state.test.ts index a48dfc63f..7d7731025 100644 --- a/backend/lib/use-cases/dataflows/cases-runtime-state.test.ts +++ b/backend/lib/use-cases/dataflows/cases-runtime-state.test.ts @@ -15,27 +15,43 @@ describe('storeRuntimeState tests', () => { jest.restoreAllMocks(); }); - test('should persist a new sync state', async () => { + test('should persist a new sync state and log', async () => { jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(undefined); const upsertSpy = jest .spyOn(MockMongoRepository.prototype, 'upsert') .mockResolvedValue(undefined); const lastSyncDate = new Date().toISOString(); + const errorLogSpy = jest.spyOn(context.logger, 'camsError'); + const infoLogSpy = jest.spyOn(context.logger, 'info'); await CasesRuntimeState.storeRuntimeState(context, lastSyncDate); expect(upsertSpy).toHaveBeenCalledWith({ documentType: 'CASES_SYNC_STATE', lastSyncDate, }); + expect(errorLogSpy).not.toHaveBeenCalled(); + expect(infoLogSpy).toHaveBeenCalledWith( + expect.any(String), + expect.stringContaining('Wrote runtime state: '), + expect.anything(), + ); }); - test('should throw CamsError', async () => { + test('should log CamsError', async () => { const original: CasesSyncState = { documentType: 'CASES_SYNC_STATE', lastSyncDate: '1000', }; jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(original); jest.spyOn(MockMongoRepository.prototype, 'upsert').mockRejectedValue(new Error('some error')); + const errorLogSpy = jest.spyOn(context.logger, 'camsError'); + const infoLogSpy = jest.spyOn(context.logger, 'info'); await expect(CasesRuntimeState.storeRuntimeState(context, '1001')).resolves.toBeUndefined(); + expect(errorLogSpy).toHaveBeenCalled(); + expect(infoLogSpy).not.toHaveBeenCalledWith( + expect.any(String), + expect.stringContaining('Wrote runtime state: '), + expect.anything(), + ); }); }); diff --git a/backend/lib/use-cases/dataflows/cases-runtime-state.ts b/backend/lib/use-cases/dataflows/cases-runtime-state.ts index 5926c7199..cd5fc7ef7 100644 --- a/backend/lib/use-cases/dataflows/cases-runtime-state.ts +++ b/backend/lib/use-cases/dataflows/cases-runtime-state.ts @@ -13,7 +13,7 @@ async function storeRuntimeState(context: ApplicationContext, lastSyncDate: stri lastSyncDate, }; await runtimeStateRepo.upsert(newSyncState); - context.logger.info(MODULE_NAME, `Wrote runtime state: ${newSyncState}.`); + context.logger.info(MODULE_NAME, `Wrote runtime state: `, newSyncState); } catch (originalError) { const error = getCamsError( originalError, From b8d07d47248c6957fd25317115d66d161804bfc5 Mon Sep 17 00:00:00 2001 From: Brian Posey <15091170+btposey@users.noreply.github.com> Date: Tue, 4 Mar 2025 12:01:25 -0500 Subject: [PATCH 10/10] Make lastSyncDate required Jira ticket: CAMS-421 Co-authored-by: James Brooks <12275865+jamesobrooks@users.noreply.github.com> Co-authored-by: Brian Posey <15091170+btposey@users.noreply.github.com> --- backend/lib/use-cases/dataflows/sync-cases.ts | 5 ----- backend/lib/use-cases/gateways.types.ts | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/backend/lib/use-cases/dataflows/sync-cases.ts b/backend/lib/use-cases/dataflows/sync-cases.ts index f1d8967a2..b26110c6a 100644 --- a/backend/lib/use-cases/dataflows/sync-cases.ts +++ b/backend/lib/use-cases/dataflows/sync-cases.ts @@ -4,7 +4,6 @@ import Factory, { getCasesGateway } from '../../factory'; import { getCamsError } from '../../common-errors/error-utilities'; import { CasesSyncState } from '../gateways.types'; import { randomUUID } from 'node:crypto'; -import { CamsError } from '../../common-errors/cams-error'; const MODULE_NAME = 'SYNC-CASES-USE-CASE'; @@ -25,10 +24,6 @@ async function getCaseIds(context: ApplicationContext, lastSyncDate?: string) { } const casesGateway = getCasesGateway(context); - const start = syncState.lastSyncDate; - if (!start) { - throw new CamsError(MODULE_NAME); - } const caseIds = await casesGateway.getUpdatedCaseIds(context, syncState.lastSyncDate); const events: CaseSyncEvent[] = caseIds.map((caseId) => { diff --git a/backend/lib/use-cases/gateways.types.ts b/backend/lib/use-cases/gateways.types.ts index c4870b491..673feb8f2 100644 --- a/backend/lib/use-cases/gateways.types.ts +++ b/backend/lib/use-cases/gateways.types.ts @@ -177,7 +177,7 @@ export type OrderSyncState = RuntimeState & { export type CasesSyncState = RuntimeState & { documentType: 'CASES_SYNC_STATE'; - lastSyncDate?: string; + lastSyncDate: string; }; export type OfficeStaffSyncState = RuntimeState & {