2991: E2E tests, workflow and bench control (v0.1)

.github/workflows/smoke_tests.yml

@@ -0,0 +1,123 @@
+name: 'Smoke tests'
+
+on:
+  push:
+    branches:
+      - "release*"
+
+env:
+  TARGETS: f7
+  DEFAULT_TARGET: f7
+  FBT_TOOLCHAIN_PATH: /opt
+
+jobs:
+  run_smoke_on_bench:
+    runs-on: [self-hosted, FlipperZeroIntegrationTest]
+    steps:
+      - name: 'Wipe workspace'
+        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
+
+      - name: Checkout code
+        timeout-minutes: 0.5
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: 'Install submodules'
+        timeout-minutes: 0.5
+        run:
+          git submodule update --init --recursive --force
+
+      - name: 'Get flipper from device manager (mock)'
+        id: device
+        timeout-minutes: 0.5
+        run: |
+          echo "flipper=/dev/ttyACM1" >> $GITHUB_OUTPUT
+          echo "stlink=49FF75065182575513201587" >> $GITHUB_OUTPUT
+
+      - name: 'Flash firmware and upload all applications'
+        id: flashing
+        if: success()
+        timeout-minutes: 7
+        run: |                               
+          ./fbt flash_usb_full PORT=${{steps.device.outputs.flipper}} FORCE=1
+
+      - name: 'Wait for flipper'
+        id: await_flipper
+        if: steps.flashing.outcome == 'success'
+        timeout-minutes: 3
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
+
+      - name: 'Run smoke tests'
+        id: run_smoke
+        if: steps.await_flipper.outcome == 'success'
+        timeout-minutes: 15
+        run: |
+          cd scripts/flippigator
+          docker buildx build -t pytest-runner .
+          docker run --rm --device ${{steps.device.outputs.flipper}}:/dev/ttyACM0 pytest-runner
+
+      - name: 'Generate allure reports'
+        id: generate_allure_reports
+        if: steps.run_smoke.outcome == 'success'
+        timeout-minutes: 0.5
+        run: |
+          pwd
+
+      - name: 'Upload allure reports'
+        id: upload_allure_reports
+        if: steps.generate_allure_reports.outcome == 'success'
+        timeout-minutes: 0.5
+        run: |
+          pwd
+
+      - name: 'Wait for flipper and format ext after tests'
+        id: format_ext
+        if: always()
+        timeout-minutes: 3
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
+          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext
+          python3 scripts/power.py -p ${{steps.device.outputs.flipper}} reboot
+          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
+
+      - name: 'Get last release tag if flipper not responding'
+        id: release_tag
+        if: steps.format_ext.outcome == 'failure'
+        run: |
+          echo "tag=$(git tag -l --sort=-version:refname | grep -v "rc\|RC" | head -1)" >> $GITHUB_OUTPUT
+
+      - name: 'Wipe workspace'
+        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
+
+      - name: 'Checkout latest release'
+        uses: actions/checkout@v3
+        if: steps.format_ext.outcome == 'failure'
+        with:
+          fetch-depth: 1
+          ref: ${{ steps.release_tag.outputs.tag }}
+
+      - name: 'Flash last release'
+        id: fbt_flash_with_serial
+        if: steps.format_ext.outcome == 'failure'
+        run: |
+          ./fbt flash OPENOCD_ADAPTER_SERIAL=${{steps.device.outputs.stlink}} FORCE=1
+
+      - name: 'Wait for flipper and format ext'
+        if: steps.format_ext.outcome == 'failure'
+        timeout-minutes: 3
+        run: |
+          source scripts/toolchain/fbtenv.sh
+          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
+          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext
+          python3 scripts/power.py -p ${{steps.device.outputs.flipper}} reboot
+          python3 scripts/testing/await_flipper.py ${{steps.device.outputs.flipper}}
+
+      - name: 'Check GDB output if we failed to flash'
+        if: steps.fbt_flash_with_serial.outcome == 'failure'
+        run: |
+          ./fbt gdb_trace_all OPENOCD_ADAPTER_SERIAL=2A0906016415303030303032 FIRMWARE_APP_SET=unit_tests FORCE=1

.github/workflows/unit_tests.yml

@@ -1,6 +1,12 @@
 name: 'Unit tests'
 
 on:
+  push:
+    branches:
+      - dev
+      - "release*"
+    tags:
+      - '*'
   pull_request:
 
 env:

.github/workflows/updater_test.yml

@@ -1,6 +1,12 @@
 name: 'Updater test'
 
 on:
+  push:
+    branches:
+      - dev
+      - "release*"
+    tags:
+      - '*'
   pull_request:
 
 env:

.gitignore

@@ -61,6 +61,14 @@ openocd.log
 PVS-Studio.log
 *.PVS-Studio.*
 
+# Test framework
+flipperzero-firmware/scripts/flippigator/flipperzero_protobuf_py
+scripts/flippigator/img/failed.bmp
+scripts/flippigator/img/recognitionFailPic.bmp
+scripts/flippigator/img/saved1.jpg
+scripts/flippigator/img/saved2.png
+scripts/flippigator/img/saved3.bmp
+
 .gdbinit
 
 /fbt_options_local.py

.gitmodules

@@ -26,6 +26,12 @@
 [submodule "lib/cxxheaderparser"]
 	path = lib/cxxheaderparser
 	url = https://github.com/robotpy/cxxheaderparser.git
+[submodule "applications/external/dap_link/lib/free-dap"]
+	path = applications/external/dap_link/lib/free-dap
+	url = https://github.com/ataradov/free-dap.git
+[submodule "scripts/flippigator/flipperzero_protobuf_py"]
+	path = scripts/flippigator/flipperzero_protobuf_py
+	url = https://github.com/flipperdevices/flipperzero_protobuf_py.git
 [submodule "lib/heatshrink"]
 	path = lib/heatshrink
 	url = https://github.com/flipperdevices/heatshrink.git

scripts/flippigator/Dockerfile

@@ -0,0 +1,10 @@
+# syntax=docker/dockerfile-upstream:master-labs
+FROM python:3.10 as builder
+
+COPY . /build
+
+WORKDIR /build
+
+RUN pip3 install -r requirements-headless.txt
+
+CMD pytest -v --gui=0 --port=/dev/ttyACM0 -m smoke

scripts/flippigator/README.md

@@ -0,0 +1,47 @@
+# flipperzero-e2e-py
+```
+           .-._   _ _ _ _ _ _ _ _
+.-''-.__.-'00  '-' ' ' ' ' ' ' ' '-.
+'.___ '    .   .--_'-' '-' '-' _'-' '._
+ V: V 'vv-'   '_   '.       .'  _..' '.'.
+   '=.____.=_.--'   :_.__.__:_   '.   : :
+           (((____.-'        '-.  /   : :
+                             (((-'\ .' /
+                           _____..'  .'
+                          '-._____.-'
+```
+Internal tool for hardware e2e testing of flipperzero on pytest.
+
+
+### Running the tests
+To run use the following command:
+```
+pytest
+```
+Additionally, you can specify port:
+```
+pytest --p COM3
+```
+Or you can run tests for specific module:
+``` 
+pytest --m nfc
+```
+Or you can search for a test or a test class:
+``` 
+pytest -k TestNfc
+```
+You can also write allure report:
+```
+pytest -p COM3 --alluredir=allure-report
+```
+And finally you can serve allure report if you have allure installed:
+```
+allure serve allure-report
+```
+
+### Writing tests
+All tests should be placed in testing folder, specified by the module
+in case of changes, write task ID into docstring of the test class, so we can know what made you do the changes.
+
+### Running tests in parallel
+To run tests in parallel, you need to install pytest-xdist and we need to finish writing device manager :)

scripts/flippigator/async_protopy/clients/application_client.py

@@ -0,0 +1,138 @@
+from async_protopy.commands.application_commands import (
+    ApplicationButtonReleaseRequestCommand,
+    ApplicationDataExchangeRequestCommand,
+    ApplicationButtonPressRequestCommand,
+    ApplicationLockStatusRequestCommand,
+    ApplicationLoadFileRequestCommand,
+    ApplicationGetErrorRequestCommand,
+    ApplicationStartRequestCommand,
+    ApplicationExitRequestCommand,
+)
+from async_protopy.clients.base_client import FlipperBaseProtoClient
+from async_protopy.flipperzero_protobuf_compiled import flipper_pb2
+
+
+class FlipperApplicationProtoClient(FlipperBaseProtoClient):
+    """The application flipper protobuf client.
+
+    Inherits from FlipperBaseProtoClient.
+    """
+    async def lock_status_request(self, wait_for_response: bool = True) -> flipper_pb2.Main:
+        """Requests the application lock status.
+
+        Args:
+            wait_for_response: bool
+                A flag used to wait for the response data or return after the command is sent (default is True)
+
+        Returns:
+            A request response message.
+        """
+        return await self.request(
+            ApplicationLockStatusRequestCommand(), wait_for_response=wait_for_response, to_validate=True
+        )
+
+    async def start_request(self, name: str, args: str, wait_for_response: bool = True) -> flipper_pb2.Main:
+        """Requests the application start/run command.
+
+        Args:
+            name: str
+                A name of the application.
+            args:
+                A path to the application.  # TODO: check it out
+            wait_for_response: bool
+                A flag used to wait for the response data or return after the command is sent (default is True)
+
+        Returns:
+            A request response message.
+        """
+        return await self.request(
+            ApplicationStartRequestCommand(name=name, args=args), wait_for_response=wait_for_response, to_validate=True
+        )
+
+    async def exit_request(self, wait_for_response: bool = True) -> flipper_pb2.Main:
+        """Requests the exit command to the application.
+
+        Args:
+            wait_for_response: bool
+                A flag used to wait for the response data or return after the command is sent (default is True)
+
+        Returns:
+            A request response message.
+        """
+        return await self.request(
+            ApplicationExitRequestCommand(), wait_for_response=wait_for_response, to_validate=True
+        )
+
+    async def load_file_request(self, path: str, wait_for_response: bool = True) -> flipper_pb2.Main:
+        """Requests the load file command to the application.
+
+        Args:
+            path: str  # TODO: get to know
+            wait_for_response: bool
+                A flag used to wait for the response data or return after the command is sent (default is True)
+
+        Returns:
+            A request response message.
+        """
+        return await self.request(
+            ApplicationLoadFileRequestCommand(path=path), wait_for_response=wait_for_response, to_validate=True
+        )
+
+    async def button_press_request(self, args: str, wait_for_response: bool = True) -> flipper_pb2.Main:
+        """Requests the button press command to the application.
+
+        Args:
+            wait_for_response: bool
+                A flag used to wait for the response data or return after the command is sent (default is True)
+
+        Returns:
+            A request response message.
+        """
+        return await self.request(
+            ApplicationButtonPressRequestCommand(args=args), wait_for_response=wait_for_response, to_validate=True
+        )
+
+    async def button_release_request(self, wait_for_response: bool = True) -> flipper_pb2.Main:
+        """Requests the button release command to the application.
+
+        Args:
+            wait_for_response: bool
+                A flag used to wait for the response data or return after the command is sent (default is True)
+
+        Returns:
+            A request response message.
+        """
+        return await self.request(
+            ApplicationButtonReleaseRequestCommand(), wait_for_response=wait_for_response, to_validate=True
+        )
+
+    async def get_error_request(self, wait_for_response: bool = True) -> flipper_pb2.Main:
+        """Requests the get error command to the application.
+
+        Args:
+            wait_for_response: bool
+                A flag used to wait for the response data or return after the command is sent (default is True)
+
+        Returns:
+            A request response message.
+        """
+        return await self.request(
+            ApplicationGetErrorRequestCommand(), wait_for_response=wait_for_response, to_validate=True
+        )
+
+    async def data_exchange_request(self, data: bytes, wait_for_response: bool = True) -> flipper_pb2.Main:
+        """Requests the data exchange command to the application.
+
+        Args:
+            wait_for_response: bool
+                A flag used to wait for the response data or return after the command is sent (default is True)
+
+        Returns:
+            A request response message.
+        """
+        return await self.request(
+            ApplicationDataExchangeRequestCommand(data=data), wait_for_response=wait_for_response, to_validate=True
+        )
+
+    async def data_exchange_receive(self) -> flipper_pb2.Main:
+        raise NotImplementedError()  # TODO: implement it