10372 refactor

docs/dynamodb-access-patterns.csv

@@ -1,14 +1,14 @@
-Access Scenario,PK,SK,GS1PK,GSI2PK
+Access Scenario,PK,SK,GS1PK,gsiUserBox,gsiSectionBox
 the case deadline object,case-deadline|${DEADLINE_ID},case-deadline|${DEADLINE_ID},
 associate a deadline with a case (mapping record),case|${CASE_ID},case-deadline|${DEADLINE_ID},
 a case,case|${CASE_ID},case|${CASE_ID},leadCase|${LEAD_CASE_ID}
 associate an irs practitioner onto a case,case|${CASE_ID},irsPractitioner|${USER_ID},leadCase|${LEAD_CASE_ID}
 associate a private practitioner onto a case,case|${CASE_ID},privatePractitioner|${USER_ID},leadCase|${LEAD_CASE_ID}
 associate docket entry on a case,case|${CASE_ID},docket-entry|${DOCKET_ENTRY_ID},
 add correspondence to a case,case|${CASE_ID},correspondence|${CORRESPONDENCE_ID},
-associate message onto a case,case|${CASE_ID},message|${MESSAGE_ID},message|${MESSAGE_ID}
+associate message onto a case,case|${CASE_ID},message|${MESSAGE_ID},message|${MESSAGE_ID},assigneeId|${ASSIGNEE_ID},section|${SECTION}
 add a hearing to a case,case|${CASE_ID},hearing|${TRIAL_SESSION_ID},
-a work item on a case,case|${CASE_ID},work-item|${WORK_ITEM_ID},work-item|${WORK_ITEM_ID},assigneeId|${ASSIGNEE_ID}
+a work item on a case,case|${CASE_ID},work-item|${WORK_ITEM_ID},work-item|${WORK_ITEM_ID},assigneeId|${inbox|inProgress}|${ASSIGNEE_ID},section|${inbox|inProgress}|${SECTION}
 docket number generator counter,docketNumberCounter-${YEAR},docketNumberCounter-${YEAR},
 how we store the list of trial sessions eligble for a case,eligible-for-trial-case-catalog,LasVegasNevada-H-B-20190816132910-107-19,eligible-for-trial-case-catalog|${CASE_ID}
 associate practitioner by bar number for lookup,privatePractitioner|${BAR_NUMBER},user|${userId},
@@ -26,4 +26,4 @@ associate user with pending case,user|${USER_ID},pending-case|${CASE_ID},
 save web socket connection associated with the user,user|${USER_ID},connection|${SOCKET_ID},connection
 create a user,user|${USER_ID},user|${USER_ID},
 save work items for a user inbox,user|${USER_ID},work-item|${WORK_ITEM_ID},work-item|${WORK_ITEM_ID}
-a work item entry,work-item|${WORK_ITEM_ID},work-item|${WORK_ITEM_ID},work-item|${WORK_ITEM_ID}
+a work item entry,work-item|${WORK_ITEM_ID},work-item|${WORK_ITEM_ID},work-item|${WORK_ITEM_ID}

docs/zenhub.md

@@ -1,9 +1,9 @@
 # Zenhub
-Zenhub is the organizational tool that we use to track stories, bugs, and Devex/Opex work. It integrates with Github Issues on the Flexion fork of the EFMCS repo
+Zenhub is the organizational tool that we use to track stories, bugs, and Devex/Opex work. It integrates with Github Issues on the Flexion fork of the EFCMS repo
 
 ## Access
 To gain write access to our Zenhub board, you must:
 - Have a Github account linked to an email you are comfortable using 
-- Be added as a collaborator on the Flexion fork of the EFMCS repo, by any team member with admin status
+- Be added as a collaborator on the Flexion fork of the EFCMS repo, by any team member with admin status
 - Be invited to the Zenhub board using the _email_ linked to the aforementioned Github account
 - Be given a seat as a licensed member on the Zenhub board

scripts/run-once-scripts/add-gsis-to-messages.ts

@@ -0,0 +1,173 @@
+// usage: npx ts-node --transpile-only scripts/run-once-scripts/add-gsis-to-messages.ts
+
+import { Message } from '../../shared/src/business/entities/Message';
+import { createApplicationContext } from '../../web-api/src/applicationContext';
+import { calculateTimeToLive } from '../../web-api/src/persistence/dynamo/calculateTimeToLive';
+import { TDynamoRecord } from '../../web-api/src/persistence/dynamo/dynamoTypes';
+import {
+  formatResults,
+  search,
+} from '../../web-api/src/persistence/elasticsearch/searchClient';
+import { exit } from 'process';
+import { get } from 'lodash';
+import { updateRecords } from 'scripts/run-once-scripts/scriptHelper';
+
+const applyMessageChanges = ({ items }) => {
+  const itemsAfter: TDynamoRecord[] = [];
+  for (const item of items) {
+    if (!item.completedAt) {
+      const ttl = calculateTimeToLive({
+        numDays: 8,
+        timestamp: item.createdAt!,
+      });
+
+      // add outbox records
+      itemsAfter.push({
+        ...item,
+        gsi1pk: undefined,
+        pk: `message|outbox|user|${item.fromUserId}`,
+        sk: item.createdAt,
+        ttl: ttl.expirationTimestamp,
+      });
+      itemsAfter.push({
+        ...item,
+        gsi1pk: undefined,
+        pk: `message|outbox|section|${item.fromSection}`,
+        sk: item.createdAt,
+        ttl: ttl.expirationTimestamp,
+      });
+
+      // add global secondary indexes
+      item.gsiUserBox = item.toUserId
+        ? `assigneeId|${item.toUserId}`
+        : undefined;
+      item.gsiSectionBox = item.toSection
+        ? `section|${item.toSection}`
+        : undefined;
+      itemsAfter.push(item);
+    } else {
+      const ttl = calculateTimeToLive({
+        numDays: 8,
+        timestamp: item.completedAt!,
+      });
+
+      // add completed box records
+      itemsAfter.push({
+        ...item,
+        gsi1pk: undefined,
+        pk: `message|completed|user|${item.completedByUserId}`,
+        sk: item.completedAt!,
+        ttl: ttl.expirationTimestamp,
+      });
+      itemsAfter.push({
+        ...item,
+        gsi1pk: undefined,
+        pk: `message|completed|section|${item.completedBySection}`,
+        sk: item.completedAt!,
+        ttl: ttl.expirationTimestamp,
+      });
+
+      // completed message does not get global secondary indexes
+      itemsAfter.push({
+        ...item,
+        gsiSectionBox: undefined,
+        gsiUserBox: undefined,
+      });
+    }
+  }
+  return itemsAfter;
+};
+
+(async function () {
+  const dryRun = process.argv.slice(2)[0] || true;
+
+  const applicationContext = createApplicationContext({});
+  const searchParameters = {
+    body: {
+      query: {
+        bool: {
+          must: [{ term: { 'entityName.S': 'Message' } }],
+        },
+      },
+    },
+    index: 'efcms-message',
+    size: 10000,
+  };
+
+  const index = searchParameters.index || '';
+  const query = searchParameters.body?.query || {};
+  const size = searchParameters.size;
+
+  let countQ;
+  try {
+    countQ = await applicationContext.getSearchClient().count({
+      body: {
+        query,
+      },
+      index,
+    });
+  } catch (searchError) {
+    applicationContext.logger.error(searchError);
+    throw new Error('Search client encountered an error.');
+  }
+
+  const expected = get(countQ, 'body.count', 0);
+
+  let i = 0;
+  let search_after = [0];
+  while (i < expected) {
+    let searchResults = [];
+    const chunk = await applicationContext.getSearchClient().search({
+      _source: [],
+      body: {
+        query,
+        search_after,
+        sort: [{ 'createdAt.S': 'asc' }],
+      },
+      index,
+      size,
+    });
+    const hits = get(chunk, 'body.hits.hits', []);
+
+    if (hits.length > 0) {
+      searchResults = searchResults.concat(hits);
+      search_after = hits[hits.length - 1].sort;
+    }
+
+    i += size; // this avoids an endless loop if expected is somehow greater than the sum of all hits
+
+    let { results } = formatResults({
+      hits: {
+        hits: searchResults,
+        total: {
+          value: searchResults.length,
+        },
+      },
+    });
+
+    try {
+      results = results.map(message => {
+        delete message._score;
+        delete message.sort;
+        new Message(message, {
+          applicationContext,
+        }).validate();
+        return message;
+      });
+    } catch (e) {
+      console.log('Error migrating message records:', e);
+      exit(); //???
+    }
+
+    const migratedItems = applyMessageChanges({ items: results });
+
+    if (dryRun != true) {
+      try {
+        await updateRecords(applicationContext, migratedItems);
+      } catch (e) {
+        console.log('Error writing migrated message records to dynamo:', e);
+        exit(); //???
+      }
+    }
+  }
+})();

scripts/run-once-scripts/add-gsis-to-work-items.ts

@@ -0,0 +1,148 @@
+// usage: npx ts-node --transpile-only scripts/run-once-scripts/add-gsis-to-work-items.ts
+
+import { createApplicationContext } from '../../web-api/src/applicationContext';
+import { scan } from '../../web-api/src/persistence/dynamodbClientService';
+import { TDynamoRecord } from '../../web-api/src/persistence/dynamo/dynamoTypes';
+import { updateRecords } from 'scripts/run-once-scripts/scriptHelper';
+import { get } from 'lodash';
+import { formatResults } from '@web-api/persistence/elasticsearch/searchClient';
+import { WorkItem } from '@shared/business/entities/WorkItem';
+import { exit } from 'process';
+
+// const findWorkItems = async ({ applicationContext }) => {
+//   const result = await scan({
+//     ExpressionAttributeValues: {
+//       ':pk_prefix': 'case|',
+//       ':sk_prefix': 'work-item|',
+//     },
+//     FilterExpression:
+//       'begins_with(pk, :pk_prefix) AND begins_with(sk, :sk_prefix)',
+//     applicationContext,
+//   });
+//   return result;
+// };
+
+const migrateItems = items => {
+  for (const item of items) {
+    if (!item.completedAt && (item.inProgress || item.caseIsInProgress)) {
+      item.gsiUserBox = item.assigneeId
+        ? `assigneeId|inProgress|${item.assigneeId}`
+        : undefined;
+      item.gsiSectionBox = item.section
+        ? `section|inProgress|${item.section}`
+        : undefined;
+    } else if (!item.completedAt) {
+      item.gsiUserBox = item.assigneeId
+        ? `assigneeId|inbox|${item.assigneeId}`
+        : undefined;
+      item.gsiSectionBox = item.section
+        ? `section|inbox|${item.section}`
+        : undefined;
+    }
+    item.gsi2pk = undefined;
+  }
+
+  return items as unknown as TDynamoRecord[];
+};
+
+(async () => {
+  const dryRun = process.argv.slice(2)[0] || true;
+
+  const applicationContext = createApplicationContext({});
+  const searchParameters = {
+    body: {
+      query: {
+        bool: {
+          must: [
+            {
+              exists: {
+                field: 'section.S',
+              },
+            },
+          ],
+        },
+      },
+    },
+    index: 'efcms-work-item',
+    size: 10000,
+  };
+
+  const index = searchParameters.index || '';
+  const query = searchParameters.body?.query || {};
+  const size = searchParameters.size;
+
+  let countQ;
+  try {
+    countQ = await applicationContext.getSearchClient().count({
+      body: {
+        query,
+      },
+      index,
+    });
+
+    console.log('countQ', countQ);
+  } catch (searchError) {
+    applicationContext.logger.error(searchError);
+    throw new Error('Search client encountered an error.');
+  }
+
+  const expected = get(countQ, 'body.count', 0);
+
+  let i = 0;
+  let search_after = [0];
+  while (i < expected) {
+    let searchResults = [];
+    const chunk = await applicationContext.getSearchClient().search({
+      _source: [],
+      body: {
+        query,
+        search_after,
+        // this is the only indexed date field, can we use it to sort?
+        sort: [{ 'completedAt.S': 'asc' }],
+      },
+      index,
+      size,
+    });
+    const hits = get(chunk, 'body.hits.hits', []);
+
+    if (hits.length > 0) {
+      searchResults = searchResults.concat(hits);
+      search_after = hits[hits.length - 1].sort;
+    }
+
+    i += size; // this avoids an endless loop if expected is somehow greater than the sum of all hits
+
+    let { results } = formatResults({
+      hits: {
+        hits: searchResults,
+        total: {
+          value: searchResults.length,
+        },
+      },
+    });
+
+    try {
+      results = results.map(workItem => {
+        delete workItem._score;
+        delete workItem.sort;
+        new WorkItem(workItem, {
+          applicationContext,
+        }).validate();
+        return workItem;
+      });
+    } catch (e) {
+      console.log('Error migrating message records:', e);
+      exit(); //???
+    }
+
+    const migratedItems = migrateItems(results);
+    if (dryRun != true) {
+      try {
+        await updateRecords(applicationContext, migratedItems);
+      } catch (e) {
+        console.log('Error writing migrated workItem records to dynamo:', e);
+        exit(); //???
+      }
+    }
+  }
+})();