diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index d37337452a9..309dd7ebd4b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -5,54 +5,39 @@ on: env: TARGETS: f7 DEFAULT_TARGET: f7 - FBT_TOOLCHAIN_PATH: /opt + FBT_TOOLCHAIN_PATH: /opt/ FBT_GIT_SUBMODULE_SHALLOW: 1 jobs: run_units_on_bench: - runs-on: [self-hosted, FlipperZeroUnitTest] + runs-on: [ self-hosted, FlipperZeroTest ] steps: - - name: 'Wipe workspace' - run: find ./ -mount -maxdepth 1 -exec rm -rf {} \; - - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 1 ref: ${{ github.event.pull_request.head.sha }} - - name: 'Get flipper from device manager (mock)' - id: device - run: | - echo "flipper=auto" >> $GITHUB_OUTPUT - - name: 'Flash unit tests firmware' id: flashing if: success() - timeout-minutes: 10 - run: | - ./fbt resources firmware_latest flash SWD_TRANSPORT_SERIAL=2A0906016415303030303032 LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 - - - name: 'Wait for flipper and format ext' - id: format_ext - if: steps.flashing.outcome == 'success' - timeout-minutes: 5 + timeout-minutes: 20 run: | source scripts/toolchain/fbtenv.sh - python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=120 await_flipper - python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext + ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 + - name: 'Copy assets and unit data, reboot and wait for flipper' id: copy - if: steps.format_ext.outcome == 'success' + if: steps.flashing.outcome == 'success' timeout-minutes: 7 run: | source scripts/toolchain/fbtenv.sh - python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=15 await_flipper - rm -rf build/latest/resources/dolphin - python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} -f send build/latest/resources /ext - python3 scripts/power.py -p ${{steps.device.outputs.flipper}} reboot - python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=15 await_flipper + python3 scripts/testops.py -t=15 await_flipper + python3 scripts/storage.py -f send build/latest/resources /ext + python3 scripts/storage.py -f send /region_data /ext/.int/.region_data + python3 scripts/power.py reboot + python3 scripts/testops.py -t=30 await_flipper - name: 'Run units and validate results' id: run_units @@ -60,9 +45,16 @@ jobs: timeout-minutes: 7 run: | source scripts/toolchain/fbtenv.sh - python3 scripts/testops.py run_units -p ${{steps.device.outputs.flipper}} + python3 scripts/testops.py run_units + + - name: 'Upload test results' + if: failure() && steps.flashing.outcome == 'success' && steps.run_units.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: unit-tests_output + path: unit_tests*.txt - name: 'Check GDB output' if: failure() && steps.flashing.outcome == 'success' run: | - ./fbt gdb_trace_all SWD_TRANSPORT_SERIAL=2A0906016415303030303032 LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 + ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1 diff --git a/.github/workflows/updater_test.yml b/.github/workflows/updater_test.yml index dbe5df883d6..59cc6e716be 100644 --- a/.github/workflows/updater_test.yml +++ b/.github/workflows/updater_test.yml @@ -1,39 +1,31 @@ name: 'Updater test' on: pull_request: + env: TARGETS: f7 DEFAULT_TARGET: f7 - FBT_TOOLCHAIN_PATH: /opt + FBT_TOOLCHAIN_PATH: /opt/ FBT_GIT_SUBMODULE_SHALLOW: 1 jobs: test_updater_on_bench: - runs-on: [self-hosted, FlipperZeroUpdaterTest] + runs-on: [self-hosted, FlipperZeroTest ] steps: - - name: 'Wipe workspace' - run: find ./ -mount -maxdepth 1 -exec rm -rf {} \; - - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 1 - submodules: false ref: ${{ github.event.pull_request.head.sha }} - - name: 'Get flipper from device manager (mock)' - id: device - run: | - echo "flipper=auto" >> $GITHUB_OUTPUT - echo "stlink=0F020D026415303030303032" >> $GITHUB_OUTPUT - - name: 'Flashing target firmware' id: first_full_flash - timeout-minutes: 10 + timeout-minutes: 20 run: | source scripts/toolchain/fbtenv.sh - ./fbt flash_usb_full PORT=${{steps.device.outputs.flipper}} FORCE=1 - python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=180 await_flipper + python3 scripts/testops.py -t=180 await_flipper + ./fbt flash_usb_full FORCE=1 + - name: 'Validating updater' id: second_full_flash @@ -41,33 +33,7 @@ jobs: if: success() run: | source scripts/toolchain/fbtenv.sh - ./fbt flash_usb PORT=${{steps.device.outputs.flipper}} FORCE=1 - python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=180 await_flipper - - - name: 'Get last release tag' - id: release_tag - if: failure() - run: | - echo "tag=$(git tag -l --sort=-version:refname | grep -v "rc\|RC" | head -1)" >> $GITHUB_OUTPUT - - - name: 'Wipe workspace' - run: find ./ -mount -maxdepth 1 -exec rm -rf {} \; + python3 scripts/testops.py -t=180 await_flipper + ./fbt flash_usb FORCE=1 + python3 scripts/testops.py -t=180 await_flipper - - name: 'Checkout latest release' - uses: actions/checkout@v4 - if: failure() - with: - fetch-depth: 1 - ref: ${{ steps.release_tag.outputs.tag }} - - - name: 'Flash last release' - if: failure() - run: | - ./fbt flash SWD_TRANSPORT_SERIAL=${{steps.device.outputs.stlink}} FORCE=1 - - - name: 'Wait for flipper and format ext' - if: failure() - run: | - source scripts/toolchain/fbtenv.sh - python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=180 await_flipper - python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext diff --git a/scripts/testops.py b/scripts/testops.py index 4ae10c7f4ae..3100a9b7fe7 100644 --- a/scripts/testops.py +++ b/scripts/testops.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 - import re -import sys import time +from datetime import datetime from typing import Optional from flipper.app import App @@ -11,7 +10,10 @@ class Main(App): - # this is basic use without sub-commands, simply to reboot flipper / power it off, not meant as a full CLI wrapper + def __init__(self, no_exit=False): + super().__init__(no_exit) + self.test_results = None + def init(self): self.parser.add_argument("-p", "--port", help="CDC Port", default="auto") self.parser.add_argument( @@ -67,64 +69,108 @@ def run_units(self): self.logger.info("Running unit tests") flipper.send("unit_tests" + "\r") self.logger.info("Waiting for unit tests to complete") - data = flipper.read.until(">: ") - self.logger.info("Parsing result") - - lines = data.decode().split("\r\n") - - tests_re = r"Failed tests: \d{0,}" - time_re = r"Consumed: \d{0,}" - leak_re = r"Leaked: \d{0,}" - status_re = r"Status: \w{3,}" - - tests_pattern = re.compile(tests_re) - time_pattern = re.compile(time_re) - leak_pattern = re.compile(leak_re) - status_pattern = re.compile(status_re) tests, elapsed_time, leak, status = None, None, None, None total = 0 - - for line in lines: - self.logger.info(line) - if "()" in line: - total += 1 - - if not tests: - tests = re.match(tests_pattern, line) - if not elapsed_time: - elapsed_time = re.match(time_pattern, line) - if not leak: - leak = re.match(leak_pattern, line) - if not status: - status = re.match(status_pattern, line) - - if None in (tests, elapsed_time, leak, status): - self.logger.error( - f"Failed to parse output: {tests} {elapsed_time} {leak} {status}" + all_required_found = False + + full_output = [] + + tests_pattern = re.compile(r"Failed tests: \d{0,}") + time_pattern = re.compile(r"Consumed: \d{0,}") + leak_pattern = re.compile(r"Leaked: \d{0,}") + status_pattern = re.compile(r"Status: \w{3,}") + + try: + while not all_required_found: + try: + line = flipper.read.until("\r\n", cut_eol=True).decode() + self.logger.info(line) + if "command not found," in line: + self.logger.error(f"Command not found: {line}") + return 1 + + if "()" in line: + total += 1 + self.logger.debug(f"Test completed: {line}") + + if not tests: + tests = tests_pattern.match(line) + if not elapsed_time: + elapsed_time = time_pattern.match(line) + if not leak: + leak = leak_pattern.match(line) + if not status: + status = status_pattern.match(line) + + pattern = re.compile( + r"(\[-]|\[\\]|\[\|]|\[/-]|\[[^\]]*\]|\x1b\[\d+D)" + ) + line_to_append = pattern.sub("", line) + pattern = re.compile(r"\[3D[^\]]*") + line_to_append = pattern.sub("", line_to_append) + line_to_append = f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')} {line_to_append}" + + full_output.append(line_to_append) + + if tests and elapsed_time and leak and status: + all_required_found = True + try: + remaining = flipper.read.until(">: ", cut_eol=True).decode() + if remaining.strip(): + full_output.append(remaining) + except: + pass + break + + except Exception as e: + self.logger.error(f"Error reading output: {e}") + raise + + if None in (tests, elapsed_time, leak, status): + raise RuntimeError( + f"Failed to parse output: {tests} {elapsed_time} {leak} {status}" + ) + + leak = int(re.findall(r"[- ]\d+", leak.group(0))[0]) + status = re.findall(r"\w+", status.group(0))[1] + tests = int(re.findall(r"\d+", tests.group(0))[0]) + elapsed_time = int(re.findall(r"\d+", elapsed_time.group(0))[0]) + + test_results = { + "full_output": "\n".join(full_output), + "total_tests": total, + "failed_tests": tests, + "elapsed_time_ms": elapsed_time, + "memory_leak_bytes": leak, + "status": status, + } + + self.test_results = test_results + + output_file = "unit_tests_output.txt" + with open(output_file, "w") as f: + f.write(test_results["full_output"]) + + print( + f"::notice:: Total tests: {total} Failed tests: {tests} Status: {status} Elapsed time: {elapsed_time / 1000} s Memory leak: {leak} bytes" ) - sys.exit(1) - - leak = int(re.findall(r"[- ]\d+", leak.group(0))[0]) - status = re.findall(r"\w+", status.group(0))[1] - tests = int(re.findall(r"\d+", tests.group(0))[0]) - elapsed_time = int(re.findall(r"\d+", elapsed_time.group(0))[0]) - - if tests > 0 or status != "PASSED": - self.logger.error(f"Got {tests} failed tests.") - self.logger.error(f"Leaked (not failing on this stat): {leak}") - self.logger.error(f"Status: {status}") - self.logger.error(f"Time: {elapsed_time/1000} seconds") - flipper.stop() - return 1 - self.logger.info(f"Leaked (not failing on this stat): {leak}") - self.logger.info( - f"Tests ran successfully! Time elapsed {elapsed_time/1000} seconds. Passed {total} tests." - ) + if tests > 0 or status != "PASSED": + self.logger.error(f"Got {tests} failed tests.") + self.logger.error(f"Leaked (not failing on this stat): {leak}") + self.logger.error(f"Status: {status}") + self.logger.error(f"Time: {elapsed_time / 1000} seconds") + return 1 - flipper.stop() - return 0 + self.logger.info(f"Leaked (not failing on this stat): {leak}") + self.logger.info( + f"Tests ran successfully! Time elapsed {elapsed_time / 1000} seconds. Passed {total} tests." + ) + return 0 + + finally: + flipper.stop() if __name__ == "__main__":