Merge pull request #61 from traveltime-dev/2022-12_automated-tests

automated tests

Author: olivierdalang

.env.example

@@ -0,0 +1,8 @@
+COMPOSE_FILE=testing/docker-compose.yml
+
+# QGIS image to test against
+QGIS_IMAGE=opengisch/qgis:3.28-jammy
+
+# Get these from https://igeolise.3scale.net/admin
+API_APP_ID=
+API_KEY=

.github/workflows/tests.yml

@@ -0,0 +1,34 @@
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+  schedule:
+    - cron:  '0 6 * * 1'  # weekly check
+
+jobs:
+  tests:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 10
+
+    strategy:
+      fail-fast: false
+      matrix:
+        qgis_image:
+          - opengisch/qgis:3.28-jammy
+          - opengisch/qgis:3.22-focal
+
+    env:
+      QGIS_IMAGE: ${{ matrix.qgis_image }}
+      API_APP_ID: ${{ secrets.API_APP_ID }}
+      API_KEY: ${{ secrets.API_KEY }}
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Copy .env
+        run: cp .env.example .env
+
+      - name: Run tests (${{ matrix.qgis_image }})
+        run: docker-compose run tests

.gitignore

@@ -1,2 +1,3 @@
 __pycache__
 .vscode
+.env

.pre-commit-config.yaml

@@ -3,7 +3,7 @@
 repos:
   # Fix end of files
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
+    rev: v4.4.0
     hooks:
       - id: trailing-whitespace
         exclude: .*\.md
@@ -14,7 +14,7 @@ repos:
 
   # Remove unused imports/variables
   - repo: https://github.com/myint/autoflake
-    rev: v1.4
+    rev: v2.0.0
     hooks:
       - id: autoflake
         args:
@@ -24,13 +24,13 @@ repos:
 
   # Sort imports
   - repo: https://github.com/pycqa/isort
-    rev: "5.7.0"
+    rev: "v5.11.3"
     hooks:
       - id: isort
         args: ["--profile", "black"]
 
   # Black formatting
   - repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 22.12.0
     hooks:
       - id: black

README.md

@@ -1,5 +1,7 @@
 # ![Travel Time Platform](travel_time_platform_plugin/resources/TravelTime_logo_horizontal.svg)
 
+[![.github/workflows/tests.yml](https://github.com/traveltime-dev/traveltime-platform-qgis-plugin/actions/workflows/tests.yml/badge.svg)](https://github.com/traveltime-dev/traveltime-platform-qgis-plugin/actions/workflows/tests.yml)
+
 This is the repository for the TravelTime platform plugin for QGIS.
 
 This plugin intends to make the TravelTime platform API available for ues in QGIS through processing algorithms.
@@ -16,6 +18,28 @@ The documentation is available here : https://qgis.traveltimeplatform.com/
 
 Please report any issue to the bug tracker.
 
+## Contribute
+
+### Tests
+
+Run the integrations tests with docker
+
+```bash
+# Copy the default config, then edit with your own API key
+cp .env.example .env
+
+# Run tests (headless)
+docker-compose run tests
+
+# Run tests (with visual feedback, on Windows only works under WSL)
+docker-compose run tests-gui
+```
+
+These tests are run automatically in Github actions. The actions require two secrets to be set: `API_APP_ID` and `API_KEY`.
+
+Alternatively, you can also run the tests from the QGIS desktop.
+
+
 ## Code style
 
 Formatting is done with `pre-commit`. Please run `pip install pre-commit` and `pre-commit install` prior to contributing.

testing/QGIS3.ini

@@ -0,0 +1,2 @@
+[PythonPlugins]
+travel_time_platform_plugin=true

testing/artifacts/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

testing/docker-compose.yml

@@ -0,0 +1,37 @@
+version: "3"
+
+services:
+
+  # Runs tests (headless)
+  tests:
+    image: ${QGIS_IMAGE:?}
+    volumes:
+      # mount the plugin
+      - ../travel_time_platform_plugin:/root/.local/share/QGIS/QGIS3/profiles/default/python/plugins/travel_time_platform_plugin
+      # include default config (enabling the plugin)
+      - ../testing/QGIS3.ini:/root/.local/share/QGIS/QGIS3/profiles/default/QGIS/QGIS3.ini
+      # include initialisation script (test launcher in QGIS)
+      - ../testing/startup.py:/root/.local/share/QGIS/QGIS3/startup.py
+      # include entrypoint script (test launcher in QGIS)
+      - ../testing/runner.py:/runner.py
+      # mount artifacts folder
+      - ../testing/artifacts:/tmp/ttp_tests
+    environment:
+      TEST_MODE: HEADLESS
+      API_APP_ID: ${API_APP_ID:?}
+      API_KEY: ${API_KEY:?}
+    init: true # not sure why, but this is necessary for xvfb (see https://stackoverflow.com/a/72017110)
+    entrypoint: xvfb-run
+    command: python3 /runner.py
+
+  # Runs tests (with visual feedback)
+  tests-gui:
+    extends: tests
+    volumes:
+      # mount the x11 socket to forward display to host
+      - /tmp/.X11-unix:/tmp/.X11-unix
+    environment:
+      TEST_MODE: HEADFULL
+      DISPLAY: :0
+    init: false
+    entrypoint: ""

testing/runner.py

@@ -0,0 +1,28 @@
+"""
+This script should be used to wrap calls to QGIS. It will check the output and set exit code
+accordingly to test results, working around the issue about QGIS crashing on exitting from python.
+"""
+
+import sys
+from subprocess import PIPE, STDOUT, Popen
+
+print("Starting qgis...")
+
+# Run QGIS in a subprocess
+process = Popen("qgis", stdout=PIPE, stderr=STDOUT, encoding="utf-8")
+
+# Print output in realtime, keep full output in string
+full_output = ""
+while True:
+    output = process.stdout.readline()
+    if output.strip():
+        print(output.strip(), flush=True)
+    full_output += output
+    if output == "" and process.poll() is not None:
+        break
+
+# Set exit code
+if "__SUCCESS__" in full_output and "__FAILURE__" not in full_output:
+    sys.exit(0)
+else:
+    sys.exit(1)

testing/startup.py

@@ -0,0 +1,48 @@
+"""
+This script starts the test suite from within QGIS.
+
+It should be mounted to ~/.local/share/QGIS/QGIS3/startup.py to run the tests on QGIS startup.
+"""
+
+import sys
+
+from qgis.PyQt.QtCore import qDebug
+from qgis.utils import iface
+
+# TODO: proper tests discovery
+from travel_time_platform_plugin.tests import run_suite, system_info
+
+# Forward pyqgis output to console
+sys.stdout.write = lambda text: qDebug(text.encode("ascii", "replace").strip())
+sys.stderr.write = lambda text: qDebug(text.encode("ascii", "replace").strip())
+
+print("Waiting for initialisation...")
+
+
+def run_tests():
+
+    print("Starting tests...")
+
+    # Show output
+    print(system_info())
+
+    # Maximize the window
+    iface.mainWindow().showMaximized()
+    iface.mainWindow().activateWindow()
+
+    # Run the tests
+    tests = run_suite(stream=sys.stdout)
+
+    # To workaround missing exit code (see below), so we print the result value and check for it in the runner
+    if tests.wasSuccessful():
+        print("__SUCCESS__")
+    else:
+        print("__FAILURE__")
+
+    # Exit code here is lost, since this crashes QGIS with segfault
+    print("notice: following `QGIS died on signal 11` can be ignored.")
+    sys.exit(0 if tests.wasSuccessful() else 1)
+
+
+# Start tests only once init is complete
+iface.initializationCompleted.connect(run_tests)