Feature: add base and coremis end-to-end tests

.github/workflows/ci.yaml

@@ -22,7 +22,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: SonarCloud Scan
-        uses: SonarSource/sonarcloud-github-action@master
+        uses: SonarSource/sonarqube-scan-action@1a6d90ebcb0e6a6b1d87e37ba693fe453195ae25
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
@@ -44,6 +44,8 @@ jobs:
           echo 'DEMO_DATASET=true' >> .env
           echo 'BE_TAG=develop' >> .env
           echo 'FE_TAG=develop' >> .env
+          echo 'DOMAIN=localhost' >> .env
+          echo 'HOSTS=localhost' >> .env
           cp .env.openSearch.example .env.openSearch
 
           docker compose up -d
@@ -66,7 +68,7 @@ jobs:
           exit 1
 
       - name: Cypress run
-        uses: cypress-io/github-action@v6
+        uses: cypress-io/github-action@b8ba51a856ba5f4c15cf39007636d4ab04f23e3c
         with:
           record: false
           parallel: false

.gitignore

@@ -7,3 +7,5 @@ data/*
 openimis-dist_dkr.code-workspace
 node_modules/
 cypress/screenshots/
+cypress/downloads/
+cypress/fixtures/tmp_individuals.csv

README.md

@@ -8,3 +8,105 @@ This repository provides configuration files for deploying openIMIS via Docker.
 
 Detailed instructions are in our Wiki: [Installation Guide](https://openimis.atlassian.net/wiki/spaces/OP/pages/963182705/MO1.1+Install+the+modular+openIMIS+using+Docker)
 
+In case of troubles, please contact our service desk via our [ticketing platform](https://openimis.atlassian.net/servicedesk/customer).
+
+## Prerequisite
+
+* Docker installed
+
+## Fast lane
+
+You can use the script `deploy_openimis.sh` to initialise all components (uses PostgreSQL DB).
+
+## First startup
+
+* Copy `.env.example` to `.env` and make the necessary adjustments.
+* Choose a database default system to use. The default is PostgreSQL (`DB_DEFAULT=postgresql`, `DB_PORT=5432`), but you can also use MSSQL (`DB_DEFAULT=mssql`, `DB_PORT=1433`, `ACCEPT_EULA=Y`). 
+* Uncomment the line `DEMO_DATASET=true` in `.env` to initialise the database with the DEMO dataset. If you leave it commented, an empty openIMIS database will be created.
+
+## OpenFN/Lightning setup 
+
+If the implementation involves managing the social protection workflow/import, then OpenFN/Lightning must be set up with the following steps:
+
+* Copy `.env.lightning.example` to `.env.lightning` and make the necessary adjustments.
+* Create the `lightning_dev` database in the database container.
+* Build the container: `docker compose -f docker-compose.yml -f docker-compose.lightning.yml build lightning`.
+* Run migrations: `docker compose -f docker-compose.yml -f docker-compose.lightning.yml run --rm lightning mix ecto.migrate`.
+* Set up the IMIS demo: `docker compose -f docker-compose.yml -f docker-compose.lightning.yml run --rm lightning ./imisSetup.sh`.
+* Start the service: `docker compose -f docker-compose.yml -f docker-compose.lightning.yml up lightning`.
+
+## OpenSearch/OpenSearch Dashboards setup 
+
+Both OpenSearch and OpenSearch Dashboards are not by default enabled in dockerized instance. To make them work, it's required to: 
+
+  * Copy `.env.openSearch.example` to `.env.openSearch` and make adjustments
+  * Run container build `docker compose -f docker-compose.yml -f docker-compose.openSearch.yml build opensearch opensearch-dashboards nginx`
+  * Run service `docker compose -f docker-compose.yml -f docker-compose.openSearch.yml up opensearch opensearch-dashboards nginx`
+
+This build provides also additional nginx proxy server in order to handle OpenSearch Dashboard application on frontend level. 
+
+To run on a dockerized instance of openIMIS (database, backend, and frontend), including OpenSearch, please follow the steps below:
+
+  * Add a value for the OPENSEARCH_BASIC_TOKEN in the environment (`.env`) file. This should be based on the admin and password credentials for OpenSearch.
+  * In the `.env` file in openimis-fe_js, use the following environment variable: `ENV OPENSEARCH_PROXY_ROOT="opensearch"`.
+  * Run the backend and frontend services.
+
+## Run openIMIS with Docker
+
+You can run the docker compose commands from within `openimis-dist_dkr` folder.
+
+### Pull new images
+
+To pull new images or images update `docker-compose pull` 
+
+### Start / Stop
+
+* To start or restart all docker containers: `docker-compose start` 
+* To stop all docker containers: `docker-compose stop`
+
+## Create LetsEncrypt certificates
+
+Use the certbot docker compose file. 
+
+`export DOMAIN [domain_name]`
+
+### Dry run 
+
+`docker-compose run --rm --entrypoint "  certbot certonly --webroot -w /var/www/certbot  --staging  --register-unsafely-without-email  -d  ${DOMAIN}    --rsa-key-size 2048     --agree-tos     --force-renewal" certbot`
+
+### Actual setup
+
+`docker-compose run --rm --entrypoint "  certbot certonly --webroot -w /var/www/certbot    --register-unsafely-without-email  -d  ${DOMAIN}    --rsa-key-size 2048     --agree-tos     --force-renewal" certbot`
+
+
+## Run integration tests
+
+Integration tests live in the `/cypress` folder of this repo.
+
+### Requirements
+
+Ensure npm is installed, then install the required dependencies:
+
+`npm install`
+
+### Run e2e tests against local docker containers
+
+Make sure all expected containers are running: `docker ps`;
+if not, start them with `docker compose start`. Then run tests:
+
+Headless: `npx cypress run`
+Headed: `npx cypress open` or `npm run cy:open`
+
+### Run e2e tests against any local or remote url
+
+This can be useful for local development or verifying a staging deployment,
+for example, if the target host is localhost:3000,
+pass it into the corresponding test command with `-- --config "baseUrl=http://localhost:3000"`:
+
+Additionally, if you are using social protection specific language pack,
+e.g. benefit plan would be called programme, you can pass in 
+`--env useSocialProtectionLanguagePack=true`
+
+- Headless: `npx cypress run --config "baseUrl=http://localhost:3000" --env useSocialProtectionLanguagePack=true`
+- Headed: `npx cypress open --config "baseUrl=http://localhost:3000" --env useSocialProtectionLanguagePack=true`
+

cypress.config.js

@@ -7,8 +7,8 @@
 const payload = {
   "query": "{ moduleConfigurations { module, config, controls { field, usage } } }"
 }
-const timeout = 4 * 60 * 1000 // 4 minutes
-const interval = 10000 // Check every 10 seconds
+const timeoutMinutes = 10
+const retryIntervalSeconds = 15
 
 function waitForServerToStart(url) {
   console.log('Waiting for API server to start...')
@@ -30,11 +30,13 @@
           }
         })
         .catch(error => {
-          if (Date.now() - startTime >= timeout) {
-            return reject(new Error('Timed out waiting for the server to start'))
+          if (Date.now() - startTime >= timeoutMinutes * 60 * 1000) {
+            return reject(
+              new Error(`Timed out waiting for the server to start: ${error.message}`)
+            )
           } else {
-            console.log(`Retrying in ${interval / 1000} seconds...`)
-            return setTimeout(checkServer, interval)
+            console.log(`${error.message}. Retrying in ${retryIntervalSeconds} seconds...`)
+            return setTimeout(checkServer, retryIntervalSeconds * 1000)
           }
         })
     }
@@ -44,36 +46,83 @@
 }
 
 module.exports = defineConfig({
+  viewportWidth: 1280,
+  viewportHeight: 670,
   e2e: {
     projectId: "q6gc25", // Cypress Cloud, needed for recording
-    baseUrl: 'http://localhost/front',
-    defaultCommandTimeout: 10000,
-    taskTimeout: 300000,
+    baseUrl: 'http://localhost',
+    defaultCommandTimeout: 15000,
+    taskTimeout: timeoutMinutes * 60 * 1000 + 10,
+    downloadsFolder: 'cypress/downloads',
     setupNodeEvents(on, config) {
       on('task', {
-          checkSetup() {
-            return fs.existsSync(serverFlagPath)
-          },
-          completeSetup() {
-            const rootUrl = config.baseUrl.replace("/front", "") 
-            const url = `${rootUrl}/api/graphql`
-            return waitForServerToStart(url)
-              .then(() => {
-                console.log('Server is ready!')
-                fs.writeFileSync(serverFlagPath, new Date().toString())
-                return null
-              })
-              .catch(error => {
-                console.error('Failed to start server:', error)
-                return null
-              })
-          },
-          removeSetupFile() {
-            if (fs.existsSync(serverFlagPath)) {
-              fs.unlinkSync(serverFlagPath)
+        checkSetup() {
+          return fs.existsSync(serverFlagPath)
+        },
+        completeSetup() {
+          const url = `${config.baseUrl}/api/graphql`
+          return waitForServerToStart(url)
+            .then(() => {
+              console.log('Server is ready!')
+              fs.writeFileSync(serverFlagPath, new Date().toString())
+              return null
+            })
+            .catch(error => {
+              console.error('Failed to start server:')
+              console.error(error.stack);
+              return reject(error);
+            })
+        },
+        removeSetupFile() {
+          if (fs.existsSync(serverFlagPath)) {
+            fs.unlinkSync(serverFlagPath)
+          }
+          return null
+        },
+        updateCSV({ numIndividuals } = {}) {
+          const fixturePath = path.join(config.fixturesFolder, 'individuals.csv');
+          const tmpPath = path.join(config.fixturesFolder, 'tmp_individuals.csv');
+          const csv = fs.readFileSync(fixturePath, 'utf8');
+
+          const lines = csv.trim().split('\n');
+          const header = lines[0];
+          const rows = lines.slice(1);
+          const headerCols = header.split(',');
+          const groupCodeIdx = headerCols.indexOf('group_code');
+
+          // map old group_code → new group_code
+          const groupMap = {};
+
+          // Subset or duplicate rows
+          let adjustedRows = [];
+          if (numIndividuals && numIndividuals > 0) {
+            if (numIndividuals <= rows.length) {
+              adjustedRows = rows.slice(0, numIndividuals);
+            } else {
+              const times = Math.floor(numIndividuals / rows.length);
+              const remainder = numIndividuals % rows.length;
+              adjustedRows = Array(times).fill(rows).flat().concat(rows.slice(0, remainder));
             }
-            return null
+          } else {
+            adjustedRows = rows;
           }
+
+          const newLines = [header];
+          for (const line of adjustedRows) {
+            const row = line.split(',');
+            const oldCode = row[groupCodeIdx];
+
+            if (!groupMap[oldCode]) {
+              groupMap[oldCode] = Math.random().toString(36).substring(2, 8).toUpperCase();
+            }
+
+            row[groupCodeIdx] = groupMap[oldCode];
+            newLines.push(row.join(','));
+          }
+
+          fs.writeFileSync(tmpPath, newLines.join('\n'));
+          return null;
+        }
       })
     },
   },