Merge pull request #238 from adhocteam/data-validation-script

Add data validation script

package.json

@@ -33,6 +33,7 @@
     "cucumber:ci": "cross-env TTA_SMART_HUB_URI=http://localhost:3000 yarn cucumber",
     "db:bootstrap:admin:local": "./node_modules/.bin/babel-node ./src/tools/bootstrapAdminCLI.js",
     "db:bootstrap:admin": "node ./build/server/tools/bootstrapAdminCLI.js",
+    "db:validation": "node ./build/server/tools/dataValidationCLI.js",
     "db:migrate": "node_modules/.bin/sequelize db:migrate",
     "db:migrate:ci": "cross-env POSTGRES_USERNAME=postgres POSTGRES_DB=ttasmarthub node_modules/.bin/sequelize db:migrate",
     "db:migrate:prod": "node_modules/.bin/sequelize db:migrate --options-path .production.sequelizerc",

src/seeders/20201211172017-grants.js

@@ -4,72 +4,84 @@ const grants = [
     number: '14CH1234',
     regionId: 14,
     granteeId: 1,
+    status: 'Active',
   },
   {
     id: 2,
     number: '14CH10000',
     regionId: 14,
     granteeId: 2,
+    status: 'Active',
   },
   {
     id: 3,
     number: '14CH00001',
     regionId: 14,
     granteeId: 3,
+    status: 'Active',
   },
   {
     id: 4,
     number: '14CH00002',
     regionId: 14,
     granteeId: 4,
+    status: 'Active',
   },
   {
     id: 5,
     number: '14CH00003',
     regionId: 14,
     granteeId: 4,
+    status: 'Active',
   },
   {
     id: 6,
     number: '09CH011111',
     regionId: 9,
     granteeId: 5,
+    status: 'Active',
   },
   {
     id: 7,
     number: '09CH022222',
     regionId: 9,
     granteeId: 6,
+    status: 'Active',
   },
   {
     id: 8,
     number: '09CH033333',
     regionId: 9,
     granteeId: 7,
+    status: 'Active',
   },
   {
     id: 9,
     number: '09HP044444',
     regionId: 9,
     granteeId: 8,
+    status: 'Active',
   },
   {
     id: 10,
     number: '01HP044444',
     regionId: 1,
     granteeId: 9,
+    status: 'Active',
   },
   {
     id: 11,
     number: '01HP022222',
     regionId: 1,
     granteeId: 10,
+    status: 'Inactive',
   },
   {
     id: 12,
     number: '09HP01111',
     regionId: 1,
     granteeId: 11,
+    status: 'Active',
   },
 ];
 

src/services/accessValidation.test.js

@@ -12,12 +12,7 @@ const {
   SITE_ACCESS, ADMIN, READ_REPORTS, READ_WRITE_REPORTS,
 } = SCOPES;
 
-jest.mock('../logger', () => ({
-  auditLogger: {
-    error: jest.fn(),
-    info: jest.fn(),
-  },
-}));
+jest.mock('../logger');
 
 const mockUser = {
   id: 47,

src/tools/bootstrapAdmin.test.js

@@ -2,8 +2,8 @@ import bootstrapAdmin, { ADMIN_USERNAME } from './bootstrapAdmin';
 import db, { User, Permission } from '../models';
 
 describe('Bootstrap the first Admin user', () => {
-  afterAll(() => {
-    db.sequelize.close();
+  afterAll(async () => {
+    await db.sequelize.close();
   });
 
   describe('when user already exists', () => {

src/tools/dataValidation.js

@@ -0,0 +1,54 @@
+import { QueryTypes } from 'sequelize';
+import { sequelize } from '../models';
+import { auditLogger } from '../logger';
+
+const runSelectQuery = (query) => (
+  sequelize.query(query, { type: QueryTypes.SELECT })
+);
+
+const countAndLastUpdated = async (tableName) => {
+  const updatedAtQuery = `SELECT "updatedAt" FROM "${tableName}" ORDER BY "updatedAt" DESC LIMIT 1`;
+  const [results] = await runSelectQuery(updatedAtQuery);
+  let updatedAt = '';
+  if (results) { updatedAt = results.updatedAt; }
+  const countQuery = `SELECT count(*) FROM "${tableName}"`;
+  const [{ count }] = await runSelectQuery(countQuery);
+  return {
+    updatedAt,
+    count,
+  };
+};
+
+const dataValidation = async () => {
+  let query;
+  let results;
+  const tableNames = [
+    'ActivityReports',
+    'Files',
+    'Goals',
+    'Objectives',
+    'NextSteps',
+    'Grantees',
+    'Grants',
+    'Users',
+  ];
+  const tableChecks = tableNames.map(async (table) => {
+    const { updatedAt, count } = await countAndLastUpdated(table);
+    auditLogger.info(`${table} has ${count} records, last updated at: ${updatedAt}`);
+  });
+  await Promise.allSettled(tableChecks);
+
+  query = 'SELECT "regionId", "status", count(*) FROM "Grants" GROUP BY "regionId", "status" ORDER BY "regionId", "status"';
+  results = await runSelectQuery(query);
+  auditLogger.info(`Grants data counts: ${JSON.stringify(results, null, 2)}`);
+
+  query = 'SELECT "regionId", "status", count(*) FROM "ActivityReports" GROUP BY "regionId", "status" ORDER BY "regionId", "status"';
+  results = await runSelectQuery(query);
+  auditLogger.info(`ActivityReports data counts: ${JSON.stringify(results, null, 2)}`);
+};
+
+export {
+  runSelectQuery,
+  countAndLastUpdated,
+};
+export default dataValidation;

src/tools/dataValidation.test.js

@@ -0,0 +1,45 @@
+import dataValidation, { countAndLastUpdated, runSelectQuery } from './dataValidation';
+import { sequelize } from '../models';
+import { auditLogger } from '../logger';
+import { DECIMAL_BASE } from '../constants';
+
+jest.mock('../logger');
+
+describe('dataValidation', () => {
+  afterAll(async () => {
+    await sequelize.close();
+  });
+
+  describe('run basic query', () => {
+    it('should return the data in an object', async () => {
+      const query = 'SELECT "regionId", "status", count(*) FROM "Grants" GROUP BY "regionId", "status" ORDER BY "regionId", "status"';
+      const [
+        { regionId: firstRowRegion, status: firstRowStatus, count: firstRowCount },
+        { regionId: secondRowRegion, status: secondRowStatus, count: secondRowCount },
+      ] = await runSelectQuery(query);
+
+      expect(firstRowRegion).toBe(1);
+      expect(firstRowStatus).toBe('Active');
+      expect(firstRowCount).toBe('2');
+      expect(secondRowRegion).toBe(1);
+      expect(secondRowStatus).toBe('Inactive');
+      expect(secondRowCount).toBe('1');
+    });
+  });
+
+  describe('run count and last updated', () => {
+    it('should return the count and last updated value for the given table', async () => {
+      const {
+        updatedAt,
+        count,
+      } = await countAndLastUpdated('Grants');
+      expect(parseInt(count, DECIMAL_BASE)).toBeGreaterThan(0);
+      expect(updatedAt).not.toBe('');
+    });
+  });
+
+  it('should log results to the auditLogger', async () => {
+    await dataValidation();
+    expect(auditLogger.info).toHaveBeenCalledTimes(10);
+  });
+});

src/tools/dataValidationCLI.js

@@ -0,0 +1,14 @@
+import dataValidation from './dataValidation';
+import { auditLogger } from '../logger';
+
+/**
+ * dataValidationCLI runs basic queries against the DB to verify that the db state
+ * is as we expect following import or restore operations.
+ *
+ * To run: `cf run-task tta-smarthub-prod --command "yarn db:validation"`
+ */
+
+dataValidation().catch((e) => {
+  auditLogger.error(e);
+  return process.exit(1);
+});