Skip to content

Commit

Permalink
feat(cypress): execute cypress tests in parallel (#6225)
Browse files Browse the repository at this point in the history
  • Loading branch information
pixincreate authored Oct 22, 2024
1 parent f3a869e commit f247978
Show file tree
Hide file tree
Showing 4 changed files with 234 additions and 57 deletions.
31 changes: 6 additions & 25 deletions .github/workflows/cypress-tests-runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ concurrency:
env:
CARGO_INCREMENTAL: 0
CARGO_NET_RETRY: 10
CONNECTORS: stripe
PAYMENTS_CONNECTORS: "stripe"
PAYOUTS_CONNECTORS: "wise"
RUST_BACKTRACE: short
RUSTUP_MAX_RETRIES: 10
RUN_TESTS: ${{ ((github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name)) || (github.event_name == 'merge_group')}}
Expand Down Expand Up @@ -68,7 +69,7 @@ jobs:
CONNECTOR_AUTH_PASSPHRASE: ${{ secrets.CONNECTOR_AUTH_PASSPHRASE }}
CONNECTOR_CREDS_S3_BUCKET_URI: ${{ secrets.CONNECTOR_CREDS_S3_BUCKET_URI}}
DESTINATION_FILE_NAME: "creds.json.gpg"
S3_SOURCE_FILE_NAME: "f64157fe-a8f7-43a8-a268-b17e9a8c305f.json.gpg"
S3_SOURCE_FILE_NAME: "5a3f7679-445e-4621-86c5-39bd8d26b7c5.json.gpg"
shell: bash
run: |
mkdir -p ".github/secrets" ".github/test"
Expand Down Expand Up @@ -186,29 +187,10 @@ jobs:
if: ${{ env.RUN_TESTS == 'true' }}
env:
CYPRESS_BASEURL: "http://localhost:8080"
ROUTER__SERVER__WORKERS: 4
shell: bash -leuo pipefail {0}
run: |
cd cypress-tests
RED='\033[0;31m'
RESET='\033[0m'
failed_connectors=()
for connector in $(echo "${CONNECTORS}" | tr "," "\n"); do
echo "${connector}"
for service in "payments" "payouts"; do
if ! ROUTER__SERVER__WORKERS=4 CYPRESS_CONNECTOR="${connector}" npm run cypress:"${service}"; then
failed_connectors+=("${connector}-${service}")
fi
done
done
if [ ${#failed_connectors[@]} -gt 0 ]; then
echo -e "${RED}One or more connectors failed to run:${RESET}"
printf '%s\n' "${failed_connectors[@]}"
exit 1
fi
. scripts/execute_cypress.sh --parallel 3
kill "${{ env.PID }}"
Expand All @@ -218,6 +200,5 @@ jobs:
with:
name: cypress-test-results
path: |
cypress-tests/cypress/reports/*.json
cypress-tests/cypress/reports/*.html
cypress-tests/cypress/reports/
retention-days: 1
29 changes: 4 additions & 25 deletions cypress-tests/cypress.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ const path = require("path");
let globalState;
// Fetch from environment variable
const connectorId = process.env.CYPRESS_CONNECTOR || "service";
const screenshotsFolderName = `screenshots/${connectorId}`;
const reportName = process.env.REPORT_NAME || `${connectorId}_report`;

module.exports = defineConfig({
Expand All @@ -25,36 +26,12 @@ module.exports = defineConfig({
return null;
},
});
on("after:screenshot", (details) => {
// Full path to the screenshot file
const screenshotPath = details.path;

// Extract filename without extension
const name = path.basename(
screenshotPath,
path.extname(screenshotPath)
);

// Define a new name with a connectorId
const newName = `[${connectorId}] ${name}.png`;
const newPath = path.join(path.dirname(screenshotPath), newName);

return fs
.rename(screenshotPath, newPath)
.then(() => {
console.log("Screenshot renamed successfully");
return { path: newPath };
})
.catch((err) => {
console.error("Failed to rename screenshot:", err);
});
});
},
experimentalRunAllSpecs: true,

reporter: "cypress-mochawesome-reporter",
reporterOptions: {
reportDir: "cypress/reports",
reportDir: `cypress/reports/${connectorId}`,
reportFilename: reportName,
reportPageTitle: `[${connectorId}] Cypress test report`,
embeddedScreenshots: true,
Expand All @@ -66,4 +43,6 @@ module.exports = defineConfig({
chromeWebSecurity: false,
defaultCommandTimeout: 10000,
pageLoadTimeout: 20000,

screenshotsFolder: screenshotsFolderName,
});
43 changes: 36 additions & 7 deletions cypress-tests/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,16 @@ To run test cases, follow these steps:
2. Install Cypress and its dependencies to `cypress-tests` directory by running the following command:

```shell
npm install
npm ci
```

3. Set environment variables for cypress
3. Insert data to `cards_info` table in `hyperswitch_db`

```shell
psql --host=localhost --port=5432 --username=db_user --dbname=hyperswitch_db --command "\copy cards_info FROM '.github/data/cards_info.csv' DELIMITER ',' CSV HEADER;"
```

4. Set environment variables for cypress

```shell
export CYPRESS_CONNECTOR="connector_id"
Expand All @@ -40,7 +46,7 @@ To run test cases, follow these steps:
export CYPRESS_CONNECTOR_AUTH_FILE_PATH="path/to/creds.json"
```

4. Run Cypress test cases
5. Run Cypress test cases

To run the tests in interactive mode run the following command

Expand Down Expand Up @@ -72,6 +78,32 @@ To run test cases, follow these steps:
npm run cypress:routing
```

In order to run cypress tests against multiple connectors at a time:

1. Set up `.env` file that exports necessary info:

```env
export DEBUG=cypress:cli
export CYPRESS_ADMINAPIKEY='admin_api_key'
export CYPRESS_BASEURL='base_url'
export CYPRESS_CONNECTOR_AUTH_FILE_PATH="path/to/creds.json"
export PAYMENTS_CONNECTORS="payment_connector_1 payment_connector_2 payment_connector_3 payment_connector_4"
export PAYOUTS_CONNECTORS="payout_connector_1 payout_connector_2 payout_connector_3"
export PAYMENT_METHOD_LIST=""
export ROUTING=""
```

2. In terminal, execute:

```shell
source .env
scripts/execute_cypress.sh
```

Optionally, `--parallel <jobs (integer)>` can be passed to run cypress tests in parallel. By default, when `parallel` command is passed, it will be run in batches of `5`.

> [!NOTE]
> To learn about how creds file should be structured, refer to the [example.creds.json](#example-credsjson) section below.

Expand Down Expand Up @@ -157,10 +189,7 @@ Cypress.Commands.add("listMandateCallTest", (globalState) => {
if (xRequestId) {
cy.task("cli_log", "x-request-id ->> " + xRequestId);
} else {
cy.task(
"cli_log",
"x-request-id is not available in the response headers"
);
cy.task("cli_log", "x-request-id is not available in the response headers");
}
expect(response.headers["content-type"]).to.include("application/json");
console.log(response.body);
Expand Down
188 changes: 188 additions & 0 deletions scripts/execute_cypress.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
#! /usr/bin/env bash

set -euo pipefail

# Initialize tmp_file globally
tmp_file=""

# Define arrays for services, etc.
# Read service arrays from environment variables
read -r -a payments <<< "${PAYMENTS_CONNECTORS[@]:-}"
read -r -a payouts <<< "${PAYOUTS_CONNECTORS[@]:-}"
read -r -a payment_method_list <<< "${PAYMENT_METHOD_LIST[@]:-}"
read -r -a routing <<< "${ROUTING[@]:-}"

# Define arrays
connector_map=()
failed_connectors=()

# Define an associative array to map environment variables to service names
declare -A services=(
["PAYMENTS_CONNECTORS"]="payments"
["PAYOUTS_CONNECTORS"]="payouts"
["PAYMENT_METHOD_LIST"]="payment_method_list"
["ROUTING"]="routing"
)

# Function to print messages in color
function print_color() {
# Input params
local color="$1"
local message="$2"

# Define colors
local reset='\033[0m'
local red='\033[0;31m'
local green='\033[0;32m'
local yellow='\033[0;33m'

# Use indirect reference to get the color value
echo -e "${!color}${message}${reset}"
}
export -f print_color

# Function to check if a command exists
function command_exists() {
command -v "$1" > /dev/null 2>&1
}

# Function to read service arrays from environment variables
function read_service_arrays() {
# Loop through the associative array and check if each service is exported
for var in "${!services[@]}"; do
if [[ -n "${!var+x}" ]]; then
connector_map+=("${services[$var]}")
else
print_color "yellow" "Environment variable ${var} is not set. Skipping..."
fi
done
}

# Function to execute Cypress tests
function execute_test() {
if [[ $# -lt 3 ]]; then
print_color "red" "ERROR: Insufficient arguments provided to execute_test."
exit 1
fi

local connector="$1"
local service="$2"
local tmp_file="$3"

print_color "yellow" "Executing tests for ${service} with connector ${connector}..."

export REPORT_NAME="${service}_${connector}_report"

if ! CYPRESS_CONNECTOR="$connector" npm run "cypress:$service"; then
echo "${service}-${connector}" >> "${tmp_file}"
fi
}
export -f execute_test

# Function to run tests
function run_tests() {
local jobs="${1:-1}"
tmp_file=$(mktemp)

# Ensure temporary file is removed on script exit
trap 'cleanup' EXIT

for service in "${connector_map[@]}"; do
declare -n connectors="$service"

if [[ ${#connectors[@]} -eq 0 ]]; then
# Service-level test (e.g., payment-method-list or routing)
[[ $service == "payment_method_list" ]] && service="payment-method-list"

echo "Running ${service} tests without connectors..."
export REPORT_NAME="${service}_report"

if ! npm run "cypress:${service}"; then
echo "${service}" >> "${tmp_file}"
fi
else
# Connector-specific tests (e.g., payments or payouts)
print_color "yellow" "Running tests for service: '${service}' with connectors: [${connectors[*]}] in batches of ${jobs}..."

# Execute tests in parallel
printf '%s\n' "${connectors[@]}" | parallel --jobs "${jobs}" execute_test {} "${service}" "${tmp_file}"
fi
done

# Collect failed connectors
if [[ -s "${tmp_file}" ]]; then
failed_connectors=($(< "${tmp_file}"))
print_color "red" "One or more connectors failed to run:"
printf '%s\n' "${failed_connectors[@]}"
exit 1
else
print_color "green" "Cypress tests execution successful!"
fi
}

# Function to check and install dependencies
function check_dependencies() {
# parallel and npm are mandatory dependencies. exit the script if not found.
local dependencies=("parallel" "npm")

for cmd in "${dependencies[@]}"; do
if ! command_exists "$cmd"; then
print_color "red" "ERROR: ${cmd^} is not installed!"
exit 1
else
print_color "green" "${cmd^} is installed already!"

if [[ ${cmd} == "npm" ]]; then
npm ci || {
print_color "red" "Command \`npm ci\` failed!"
exit 1
}
fi
fi
done
}

# Cleanup function to handle exit
function cleanup() {
print_color "yellow" "Cleaning up..."
if [[ -d "cypress-tests" ]]; then
cd -
fi

if [[ -n "${tmp_file}" && -f "${tmp_file}" ]]; then
rm -f "${tmp_file}"
fi
}

# Main function
function main() {
local command="${1:-}"
local jobs="${2:-5}"

# Ensure script runs from 'cypress-tests' directory
if [[ "$(basename "$PWD")" != "cypress-tests" ]]; then
print_color "yellow" "Changing directory to 'cypress-tests'..."
cd cypress-tests || {
print_color "red" "ERROR: Directory 'cypress-tests' not found!"
exit 1
}
fi

check_dependencies
read_service_arrays

case "$command" in
--parallel | -p)
print_color "yellow" "WARNING: Running Cypress tests in parallel is more resource-intensive!"
# At present, parallel execution is restricted to not run out of memory
# But can be scaled up by passing the value as an argument
run_tests "$jobs"
;;
*)
run_tests 1
;;
esac
}

# Execute the main function with passed arguments
main "$@"

0 comments on commit f247978

Please sign in to comment.