Merge pull request #1240 from US-Trustee-Program/CAMS-421-update-sync

CAMS-421 Upgrade case sync to use last sync date

backend/function-apps/dataflows/import/migrate-cases.ts

@@ -17,6 +17,7 @@ import ExportAndLoadCase from '../../../lib/use-cases/dataflows/export-and-load-
 import { isNotFoundError } from '../../../lib/common-errors/not-found-error';
 import ApplicationContextCreator from '../../azure/application-context-creator';
 import { UnknownError } from '../../../lib/common-errors/unknown-error';
+import { getTodaysIsoDate } from '../../../../common/src/date-helper';
 
 const MODULE_NAME = 'MIGRATE-CASES';
 const PAGE_SIZE = 100;
@@ -236,7 +237,7 @@ async function getCaseIdsToMigrate(
  */
 async function storeRuntimeState(invocationContext: InvocationContext) {
   const appContext = await ContextCreator.getApplicationContext({ invocationContext });
-  return CasesRuntimeState.storeRuntimeState(appContext);
+  return CasesRuntimeState.storeRuntimeState(appContext, getTodaysIsoDate());
 }
 
 export function setupMigrateCases() {

backend/function-apps/dataflows/import/sync-cases.ts

@@ -49,7 +49,10 @@ const TIMER_TRIGGER = buildFunctionName(MODULE_NAME, 'timerTrigger');
 async function handleStart(startMessage: StartMessage, invocationContext: InvocationContext) {
   try {
     const context = await ContextCreator.getApplicationContext({ invocationContext });
-    const { events, lastTxId } = await SyncCases.getCaseIds(context, startMessage['lastTxId']);
+    const { events, lastSyncDate } = await SyncCases.getCaseIds(
+      context,
+      startMessage['lastSyncDate'],
+    );
 
     if (!events.length) return;
 
@@ -64,7 +67,7 @@ async function handleStart(startMessage: StartMessage, invocationContext: Invoca
     }
     invocationContext.extraOutputs.set(PAGE, pages);
 
-    await CasesRuntimeState.storeRuntimeState(context, lastTxId);
+    await CasesRuntimeState.storeRuntimeState(context, lastSyncDate);
   } catch (originalError) {
     invocationContext.extraOutputs.set(
       DLQ,

backend/lib/adapters/gateways/cases.local.gateway.ts

@@ -1,8 +1,4 @@
-import {
-  CasesInterface,
-  CasesSyncMeta,
-  TransactionIdRangeForDate,
-} from '../../use-cases/cases/cases.interface';
+import { CasesInterface, TransactionIdRangeForDate } from '../../use-cases/cases/cases.interface';
 import { ApplicationContext } from '../types/basic';
 import { GatewayHelper } from './gateway-helper';
 import { getMonthDayYearStringFromDate } from '../utils/date-helper';
@@ -124,10 +120,10 @@ export class CasesLocalGateway implements CasesInterface {
     }
   }
 
-  public async getCaseIdsAndMaxTxIdToSync(
+  public async getUpdatedCaseIds(
     _applicationContext: ApplicationContext,
-    _lastTxId: string,
-  ): Promise<CasesSyncMeta> {
+    _start: string,
+  ): Promise<string[]> {
     throw new Error('Not implemented');
   }
 

backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.test.ts

@@ -757,56 +757,6 @@ describe('Test DXTR Gateway', () => {
     });
   });
 
-  describe('getCaseIdsAndMaxTxIdToSync tests', () => {
-    test('should return array of case ids and a max tx id', async () => {
-      const mockRecords = [
-        { caseId: MockData.getCaseBasics().caseId, maxTxId: 10 },
-        { caseId: MockData.getCaseBasics().caseId, maxTxId: 9 },
-        { caseId: MockData.getCaseBasics().caseId, maxTxId: 8 },
-        { caseId: MockData.getCaseBasics().caseId, maxTxId: 7 },
-      ];
-      const mockResults: QueryResults = {
-        success: true,
-        results: {
-          recordset: mockRecords,
-        },
-        message: '',
-      };
-      const expectedReturn = {
-        caseIds: mockRecords.map((rec) => rec.caseId),
-        lastTxId: '10',
-      };
-
-      querySpy.mockImplementationOnce(async () => {
-        return Promise.resolve(mockResults);
-      });
-
-      const result = await testCasesDxtrGateway.getCaseIdsAndMaxTxIdToSync(applicationContext, '0');
-      expect(result).toEqual(expectedReturn);
-    });
-
-    test('should return an empty array and the existing max tx id', async () => {
-      const mockResults: QueryResults = {
-        success: true,
-        results: {
-          recordset: [],
-        },
-        message: '',
-      };
-      const expectedReturn = {
-        caseIds: [],
-        lastTxId: '0',
-      };
-
-      querySpy.mockImplementationOnce(async () => {
-        return Promise.resolve(mockResults);
-      });
-
-      const result = await testCasesDxtrGateway.getCaseIdsAndMaxTxIdToSync(applicationContext, '0');
-      expect(result).toEqual(expectedReturn);
-    });
-  });
-
   describe('findTransactionIdRangeForDate', () => {
     const dateRangeMock = (_context, _config, query: string, params: DbTableFieldSpec[]) => {
       const mockTxMap = new Map<number, string>([
@@ -898,4 +848,44 @@ describe('Test DXTR Gateway', () => {
       },
     );
   });
+
+  describe('getUpdatedCaseIds', () => {
+    test('should return a list of updated case ids', async () => {
+      const recordset = MockData.buildArray(MockData.randomCaseId, 100).map((caseId) => {
+        return { caseId };
+      });
+
+      const executeResults: QueryResults = {
+        success: true,
+        results: {
+          recordset,
+        },
+        message: '',
+      };
+
+      const expectedReturn = recordset.map((record) => record.caseId);
+
+      querySpy.mockImplementationOnce(async () => {
+        return executeResults;
+      });
+
+      const startDate = new Date().toISOString();
+      const actual = await testCasesDxtrGateway.getUpdatedCaseIds(applicationContext, startDate);
+      expect(actual).toEqual(expectedReturn);
+    });
+
+    test('should return an empty array', async () => {
+      const mockResults: QueryResults = {
+        success: true,
+        results: {
+          recordset: [],
+        },
+        message: '',
+      };
+
+      querySpy.mockReturnValue(mockResults);
+      const actual = await testCasesDxtrGateway.getUpdatedCaseIds(applicationContext, 'foo');
+      expect(actual).toEqual([]);
+    });
+  });
 });

backend/lib/adapters/gateways/dxtr/cases.dxtr.gateway.ts

@@ -1,7 +1,6 @@
 import * as mssql from 'mssql';
 import {
   CasesInterface,
-  CasesSyncMeta,
   TransactionIdRangeForDate,
 } from '../../../use-cases/cases/cases.interface';
 import { ApplicationContext } from '../../types/basic';
@@ -30,6 +29,7 @@ const orderToTransferCode = 'CTO';
 const NOT_FOUND = -1;
 
 type RawCaseIdAndMaxId = { caseId: string; maxTxId: number };
+type CaseIdRecord = { caseId: string };
 
 export function getCaseIdParts(caseId: string) {
   const parts = caseId.split('-');
@@ -472,58 +472,45 @@ export default class CasesDxtrGateway implements CasesInterface {
     return bCase;
   }
 
-  async getCaseIdsAndMaxTxIdToSync(
-    context: ApplicationContext,
-    lastTxId: string,
-  ): Promise<CasesSyncMeta> {
-    const input: DbTableFieldSpec[] = [];
-
-    input.push({
-      name: 'txId',
-      type: mssql.BigInt,
-      value: parseInt(lastTxId),
+  /**
+   * getUpdatedCaseIds
+   *
+   * Gets the case ids for all cases with LAST_UPDATE_DATE values greater than the provided date.
+   * 2025-02-23 06:35:30.453
+   *
+   * @param {string} start The date and time to begin checking for LAST_UPDATE_DATE values.
+   * @returns {string[]} A list of case ids for updated cases.
+   */
+  async getUpdatedCaseIds(context: ApplicationContext, start: string): Promise<string[]> {
+    const params: DbTableFieldSpec[] = [];
+    params.push({
+      name: 'start',
+      type: mssql.DateTime,
+      value: start,
     });
 
     const query = `
-      SELECT
-        CONCAT(CS_DIV.CS_DIV_ACMS, '-', C.CASE_ID) AS caseId,
-        MAX(T.TX_ID) as maxTxId
-      FROM AO_TX T
-      JOIN AO_CS C ON C.CS_CASEID = T.CS_CASEID AND C.COURT_ID = T.COURT_ID
+      SELECT CONCAT(CS_DIV.CS_DIV_ACMS, '-', C.CASE_ID) AS caseId
+      FROM AO_CS C
       JOIN AO_CS_DIV AS CS_DIV ON C.CS_DIV = CS_DIV.CS_DIV
-      WHERE T.TX_ID > @txId
-      GROUP BY CS_DIV.CS_DIV_ACMS, C.CASE_ID
-      ORDER BY MAX(T.TX_ID) DESC
+      WHERE C.LAST_UPDATE_DATE > @start
     `;
 
     const queryResult: QueryResults = await executeQuery(
       context,
       context.config.dxtrDbConfig,
       query,
-      input,
+      params,
     );
 
-    const results = handleQueryResult<RawCaseIdAndMaxId[]>(
+    const results = handleQueryResult<CaseIdRecord[]>(
       context,
       queryResult,
       MODULE_NAME,
-      this.caseIdsAndMaxTxIdCallback,
+      this.getUpdatedCaseIdsCallback,
     );
 
-    let meta: CasesSyncMeta;
-    if (results.length) {
-      meta = {
-        caseIds: results.map((bCase) => bCase.caseId),
-        lastTxId: results[0].maxTxId.toString(),
-      };
-    } else {
-      meta = {
-        caseIds: [],
-        lastTxId,
-      };
-    }
-
-    return meta;
+    return results.map((record) => record.caseId);
   }
 
   private async queryCase(
@@ -902,4 +889,10 @@ export default class CasesDxtrGateway implements CasesInterface {
 
     return (queryResult.results as mssql.IResult<RawCaseIdAndMaxId[]>).recordset;
   }
+
+  getUpdatedCaseIdsCallback(applicationContext: ApplicationContext, queryResult: QueryResults) {
+    applicationContext.logger.debug(MODULE_NAME, `Results received from DXTR`);
+
+    return (queryResult.results as mssql.IResult<CaseIdRecord[]>).recordset;
+  }
 }

backend/lib/use-cases/cases/cases.interface.d.ts

@@ -26,10 +26,7 @@ export interface CasesInterface {
 
   getSuggestedCases(applicationContext: ApplicationContext, caseId: string): Promise<CaseSummary[]>;
 
-  getCaseIdsAndMaxTxIdToSync(
-    applicationContext: ApplicationContext,
-    lastTxId: string,
-  ): Promise<CasesSyncMeta>;
+  getUpdatedCaseIds(applicationContext: ApplicationContext, start: string): Promise<string[]>;
 
   findTransactionIdRangeForDate(
     context: ApplicationContext,

backend/lib/use-cases/dataflows/cases-runtime-state.test.ts

@@ -1,7 +1,6 @@
 import { ApplicationContext } from '../../adapters/types/basic';
 import { createMockApplicationContext } from '../../testing/testing-utilities';
 import { MockMongoRepository } from '../../testing/mock-gateways/mock-mongo.repository';
-import { CasesLocalGateway } from '../../adapters/gateways/cases.local.gateway';
 import { CasesSyncState } from '../gateways.types';
 import CasesRuntimeState from './cases-runtime-state';
 
@@ -16,85 +15,43 @@ describe('storeRuntimeState tests', () => {
     jest.restoreAllMocks();
   });
 
-  test('should persist a new sync state when transaction id is not provided', async () => {
+  test('should persist a new sync state and log', async () => {
     jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(undefined);
     const upsertSpy = jest
       .spyOn(MockMongoRepository.prototype, 'upsert')
       .mockResolvedValue(undefined);
-    const txId = '1001';
-    jest.spyOn(CasesLocalGateway.prototype, 'findMaxTransactionId').mockResolvedValue(txId);
 
-    await CasesRuntimeState.storeRuntimeState(context);
-    expect(upsertSpy).toHaveBeenCalledWith({ documentType: 'CASES_SYNC_STATE', txId });
-  });
-
-  test('should persist a new sync state', async () => {
-    jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(undefined);
-    const upsertSpy = jest
-      .spyOn(MockMongoRepository.prototype, 'upsert')
-      .mockResolvedValue(undefined);
-
-    const txId = '1001';
-    await CasesRuntimeState.storeRuntimeState(context, txId);
+    const lastSyncDate = new Date().toISOString();
+    const errorLogSpy = jest.spyOn(context.logger, 'camsError');
+    const infoLogSpy = jest.spyOn(context.logger, 'info');
+    await CasesRuntimeState.storeRuntimeState(context, lastSyncDate);
     expect(upsertSpy).toHaveBeenCalledWith({
       documentType: 'CASES_SYNC_STATE',
-      txId,
+      lastSyncDate,
     });
+    expect(errorLogSpy).not.toHaveBeenCalled();
+    expect(infoLogSpy).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.stringContaining('Wrote runtime state: '),
+      expect.anything(),
+    );
   });
 
-  test('should persist a higher transaction id', async () => {
-    const original: CasesSyncState = {
-      documentType: 'CASES_SYNC_STATE',
-      txId: '1000',
-    };
-    jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(original);
-    const upsertSpy = jest
-      .spyOn(MockMongoRepository.prototype, 'upsert')
-      .mockResolvedValue(undefined);
-
-    const txId = '1001';
-    await CasesRuntimeState.storeRuntimeState(context, txId);
-    expect(upsertSpy).toHaveBeenCalledWith({ ...original, txId });
-  });
-
-  test('should not persist a lower transaction id', async () => {
+  test('should log CamsError', async () => {
     const original: CasesSyncState = {
       documentType: 'CASES_SYNC_STATE',
-      txId: '1000',
-    };
-    jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(original);
-    const upsertSpy = jest
-      .spyOn(MockMongoRepository.prototype, 'upsert')
-      .mockRejectedValue(new Error('this should not be called'));
-
-    const txId = '1';
-    await CasesRuntimeState.storeRuntimeState(context, txId);
-    expect(upsertSpy).not.toHaveBeenCalled();
-  });
-
-  test('should throw CamsError', async () => {
-    const original: CasesSyncState = {
-      documentType: 'CASES_SYNC_STATE',
-      txId: '1000',
+      lastSyncDate: '1000',
     };
     jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(original);
     jest.spyOn(MockMongoRepository.prototype, 'upsert').mockRejectedValue(new Error('some error'));
+    const errorLogSpy = jest.spyOn(context.logger, 'camsError');
+    const infoLogSpy = jest.spyOn(context.logger, 'info');
     await expect(CasesRuntimeState.storeRuntimeState(context, '1001')).resolves.toBeUndefined();
-  });
-
-  test('should throw error if gateway throws', async () => {
-    jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(undefined);
-    jest
-      .spyOn(CasesLocalGateway.prototype, 'findMaxTransactionId')
-      .mockRejectedValue(new Error('some error'));
-
-    await expect(CasesRuntimeState.storeRuntimeState(context)).resolves.toBeUndefined();
-  });
-
-  test('should throw error if max transaction id cannot be determined', async () => {
-    jest.spyOn(MockMongoRepository.prototype, 'read').mockResolvedValue(undefined);
-    jest.spyOn(CasesLocalGateway.prototype, 'findMaxTransactionId').mockResolvedValue(undefined);
-
-    await expect(CasesRuntimeState.storeRuntimeState(context)).resolves.toBeUndefined();
+    expect(errorLogSpy).toHaveBeenCalled();
+    expect(infoLogSpy).not.toHaveBeenCalledWith(
+      expect.any(String),
+      expect.stringContaining('Wrote runtime state: '),
+      expect.anything(),
+    );
   });
 });