Move updater and unit tests to dockerized runner (#4028)

* extended unit_tests and changed to dockerized runner * added branch to run units * fixing unit-tests-output * online output * command not found fix * added stm logging * cleaned output * Updated updater test to work on dockerized runner * Test run for changed actions * small refactor of run_unit_tests * Final test of jobs * Checked * On-failure actions runs only on fail * Set action trigger to pull request * Bumped timeout a little * Removed extra steps * Removed stm monitor, as it's now part of docker-runner * fix: testops without stm_monitoring * fix: timeout extended Co-authored-by: あく <[email protected]>
flipperdevices · Dec 19, 2024 · 8dd5e64 · 8dd5e64
1 parent a7b3a13
commit 8dd5e64
Show file tree

Hide file tree

Showing 3 changed files with 131 additions and 127 deletions.
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -5,64 +5,56 @@ on:
 env:
   TARGETS: f7
   DEFAULT_TARGET: f7
-  FBT_TOOLCHAIN_PATH: /opt
+  FBT_TOOLCHAIN_PATH: /opt/
   FBT_GIT_SUBMODULE_SHALLOW: 1
 
 jobs:
   run_units_on_bench:
-    runs-on: [self-hosted, FlipperZeroUnitTest]
+    runs-on: [ self-hosted, FlipperZeroTest ]
     steps:
-      - name: 'Wipe workspace'
-        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
-
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: 'Get flipper from device manager (mock)'
-        id: device
-        run: |
-          echo "flipper=auto" >> $GITHUB_OUTPUT
-
       - name: 'Flash unit tests firmware'
         id: flashing
         if: success()
-        timeout-minutes: 10
-        run: |
-          ./fbt resources firmware_latest flash SWD_TRANSPORT_SERIAL=2A0906016415303030303032 LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1
-
-      - name: 'Wait for flipper and format ext'
-        id: format_ext
-        if: steps.flashing.outcome == 'success'
-        timeout-minutes: 5
+        timeout-minutes: 20
         run: |
           source scripts/toolchain/fbtenv.sh
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=120 await_flipper 
-          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext
+          ./fbt resources firmware_latest flash LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1
+
 
       - name: 'Copy assets and unit data, reboot and wait for flipper'
         id: copy
-        if: steps.format_ext.outcome == 'success'
+        if: steps.flashing.outcome == 'success'
         timeout-minutes: 7
         run: |
           source scripts/toolchain/fbtenv.sh
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=15 await_flipper 
-          rm -rf build/latest/resources/dolphin
-          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} -f send build/latest/resources /ext
-          python3 scripts/power.py -p ${{steps.device.outputs.flipper}} reboot
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=15 await_flipper 
+          python3 scripts/testops.py -t=15 await_flipper
+          python3 scripts/storage.py -f send build/latest/resources /ext
+          python3 scripts/storage.py -f send /region_data /ext/.int/.region_data
+          python3 scripts/power.py reboot
+          python3 scripts/testops.py -t=30 await_flipper
 
       - name: 'Run units and validate results'
         id: run_units
         if: steps.copy.outcome == 'success'
         timeout-minutes: 7
         run: |
           source scripts/toolchain/fbtenv.sh
-          python3 scripts/testops.py run_units -p ${{steps.device.outputs.flipper}}
+          python3 scripts/testops.py run_units
+
+      - name: 'Upload test results'
+        if: failure() && steps.flashing.outcome == 'success' && steps.run_units.outcome != 'skipped'
+        uses: actions/upload-artifact@v4
+        with:
+          name: unit-tests_output
+          path: unit_tests*.txt
 
       - name: 'Check GDB output'
         if: failure() && steps.flashing.outcome == 'success'
         run: |
-          ./fbt gdb_trace_all SWD_TRANSPORT_SERIAL=2A0906016415303030303032 LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1
+          ./fbt gdb_trace_all LIB_DEBUG=1 FIRMWARE_APP_SET=unit_tests FORCE=1
diff --git a/.github/workflows/updater_test.yml b/.github/workflows/updater_test.yml
@@ -1,73 +1,39 @@
 name: 'Updater test'
 on:
   pull_request:
+
 env:
   TARGETS: f7
   DEFAULT_TARGET: f7
-  FBT_TOOLCHAIN_PATH: /opt
+  FBT_TOOLCHAIN_PATH: /opt/
   FBT_GIT_SUBMODULE_SHALLOW: 1
 
 jobs:
   test_updater_on_bench:
-    runs-on: [self-hosted, FlipperZeroUpdaterTest]
+    runs-on: [self-hosted, FlipperZeroTest ]
     steps:
-      - name: 'Wipe workspace'
-        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
-
       - name: Checkout code
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
-          submodules: false
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: 'Get flipper from device manager (mock)'
-        id: device
-        run: |
-          echo "flipper=auto" >> $GITHUB_OUTPUT
-          echo "stlink=0F020D026415303030303032" >> $GITHUB_OUTPUT
-
       - name: 'Flashing target firmware'
         id: first_full_flash
-        timeout-minutes: 10
+        timeout-minutes: 20
         run: |
           source scripts/toolchain/fbtenv.sh
-          ./fbt flash_usb_full PORT=${{steps.device.outputs.flipper}} FORCE=1
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=180 await_flipper 
+          python3 scripts/testops.py -t=180 await_flipper
+          ./fbt flash_usb_full FORCE=1
+          
 
       - name: 'Validating updater'
         id: second_full_flash
         timeout-minutes: 10
         if: success()
         run: |
           source scripts/toolchain/fbtenv.sh
-          ./fbt flash_usb PORT=${{steps.device.outputs.flipper}} FORCE=1
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=180 await_flipper 
-
-      - name: 'Get last release tag'
-        id: release_tag
-        if: failure()
-        run: |
-          echo "tag=$(git tag -l --sort=-version:refname | grep -v "rc\|RC" | head -1)" >> $GITHUB_OUTPUT
-
-      - name: 'Wipe workspace'
-        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
+          python3 scripts/testops.py -t=180 await_flipper
+          ./fbt flash_usb FORCE=1
+          python3 scripts/testops.py -t=180 await_flipper
 
-      - name: 'Checkout latest release'
-        uses: actions/checkout@v4
-        if: failure()
-        with:
-          fetch-depth: 1
-          ref: ${{ steps.release_tag.outputs.tag }}
-
-      - name: 'Flash last release'
-        if: failure()
-        run: |
-          ./fbt flash SWD_TRANSPORT_SERIAL=${{steps.device.outputs.stlink}} FORCE=1
-
-      - name: 'Wait for flipper and format ext'
-        if: failure()
-        run: |
-          source scripts/toolchain/fbtenv.sh
-          python3 scripts/testops.py -p=${{steps.device.outputs.flipper}} -t=180 await_flipper 
-          python3 scripts/storage.py -p ${{steps.device.outputs.flipper}} format_ext
diff --git a/scripts/testops.py b/scripts/testops.py
@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
-
 import re
-import sys
 import time
+from datetime import datetime
 from typing import Optional
 
 from flipper.app import App
@@ -11,7 +10,10 @@
 
 
 class Main(App):
-    # this is basic use without sub-commands, simply to reboot flipper / power it off, not meant as a full CLI wrapper
+    def __init__(self, no_exit=False):
+        super().__init__(no_exit)
+        self.test_results = None
+
     def init(self):
         self.parser.add_argument("-p", "--port", help="CDC Port", default="auto")
         self.parser.add_argument(
@@ -67,64 +69,108 @@ def run_units(self):
         self.logger.info("Running unit tests")
         flipper.send("unit_tests" + "\r")
         self.logger.info("Waiting for unit tests to complete")
-        data = flipper.read.until(">: ")
-        self.logger.info("Parsing result")
-
-        lines = data.decode().split("\r\n")
-
-        tests_re = r"Failed tests: \d{0,}"
-        time_re = r"Consumed: \d{0,}"
-        leak_re = r"Leaked: \d{0,}"
-        status_re = r"Status: \w{3,}"
-
-        tests_pattern = re.compile(tests_re)
-        time_pattern = re.compile(time_re)
-        leak_pattern = re.compile(leak_re)
-        status_pattern = re.compile(status_re)
 
         tests, elapsed_time, leak, status = None, None, None, None
         total = 0
-
-        for line in lines:
-            self.logger.info(line)
-            if "()" in line:
-                total += 1
-
-            if not tests:
-                tests = re.match(tests_pattern, line)
-            if not elapsed_time:
-                elapsed_time = re.match(time_pattern, line)
-            if not leak:
-                leak = re.match(leak_pattern, line)
-            if not status:
-                status = re.match(status_pattern, line)
-
-        if None in (tests, elapsed_time, leak, status):
-            self.logger.error(
-                f"Failed to parse output: {tests} {elapsed_time} {leak} {status}"
+        all_required_found = False
+
+        full_output = []
+
+        tests_pattern = re.compile(r"Failed tests: \d{0,}")
+        time_pattern = re.compile(r"Consumed: \d{0,}")
+        leak_pattern = re.compile(r"Leaked: \d{0,}")
+        status_pattern = re.compile(r"Status: \w{3,}")
+
+        try:
+            while not all_required_found:
+                try:
+                    line = flipper.read.until("\r\n", cut_eol=True).decode()
+                    self.logger.info(line)
+                    if "command not found," in line:
+                        self.logger.error(f"Command not found: {line}")
+                        return 1
+
+                    if "()" in line:
+                        total += 1
+                        self.logger.debug(f"Test completed: {line}")
+
+                    if not tests:
+                        tests = tests_pattern.match(line)
+                    if not elapsed_time:
+                        elapsed_time = time_pattern.match(line)
+                    if not leak:
+                        leak = leak_pattern.match(line)
+                    if not status:
+                        status = status_pattern.match(line)
+
+                    pattern = re.compile(
+                        r"(\[-]|\[\\]|\[\|]|\[/-]|\[[^\]]*\]|\x1b\[\d+D)"
+                    )
+                    line_to_append = pattern.sub("", line)
+                    pattern = re.compile(r"\[3D[^\]]*")
+                    line_to_append = pattern.sub("", line_to_append)
+                    line_to_append = f"{datetime.now().strftime('%Y-%m-%d %H:%M:%S,%f')} {line_to_append}"
+
+                    full_output.append(line_to_append)
+
+                    if tests and elapsed_time and leak and status:
+                        all_required_found = True
+                        try:
+                            remaining = flipper.read.until(">: ", cut_eol=True).decode()
+                            if remaining.strip():
+                                full_output.append(remaining)
+                        except:
+                            pass
+                        break
+
+                except Exception as e:
+                    self.logger.error(f"Error reading output: {e}")
+                    raise
+
+            if None in (tests, elapsed_time, leak, status):
+                raise RuntimeError(
+                    f"Failed to parse output: {tests} {elapsed_time} {leak} {status}"
+                )
+
+            leak = int(re.findall(r"[- ]\d+", leak.group(0))[0])
+            status = re.findall(r"\w+", status.group(0))[1]
+            tests = int(re.findall(r"\d+", tests.group(0))[0])
+            elapsed_time = int(re.findall(r"\d+", elapsed_time.group(0))[0])
+
+            test_results = {
+                "full_output": "\n".join(full_output),
+                "total_tests": total,
+                "failed_tests": tests,
+                "elapsed_time_ms": elapsed_time,
+                "memory_leak_bytes": leak,
+                "status": status,
+            }
+
+            self.test_results = test_results
+
+            output_file = "unit_tests_output.txt"
+            with open(output_file, "w") as f:
+                f.write(test_results["full_output"])
+
+            print(
+                f"::notice:: Total tests: {total} Failed tests: {tests} Status: {status} Elapsed time: {elapsed_time / 1000} s Memory leak: {leak} bytes"
             )
-            sys.exit(1)
-
-        leak = int(re.findall(r"[- ]\d+", leak.group(0))[0])
-        status = re.findall(r"\w+", status.group(0))[1]
-        tests = int(re.findall(r"\d+", tests.group(0))[0])
-        elapsed_time = int(re.findall(r"\d+", elapsed_time.group(0))[0])
-
-        if tests > 0 or status != "PASSED":
-            self.logger.error(f"Got {tests} failed tests.")
-            self.logger.error(f"Leaked (not failing on this stat): {leak}")
-            self.logger.error(f"Status: {status}")
-            self.logger.error(f"Time: {elapsed_time/1000} seconds")
-            flipper.stop()
-            return 1
 
-        self.logger.info(f"Leaked (not failing on this stat): {leak}")
-        self.logger.info(
-            f"Tests ran successfully! Time elapsed {elapsed_time/1000} seconds. Passed {total} tests."
-        )
+            if tests > 0 or status != "PASSED":
+                self.logger.error(f"Got {tests} failed tests.")
+                self.logger.error(f"Leaked (not failing on this stat): {leak}")
+                self.logger.error(f"Status: {status}")
+                self.logger.error(f"Time: {elapsed_time / 1000} seconds")
+                return 1
 
-        flipper.stop()
-        return 0
+            self.logger.info(f"Leaked (not failing on this stat): {leak}")
+            self.logger.info(
+                f"Tests ran successfully! Time elapsed {elapsed_time / 1000} seconds. Passed {total} tests."
+            )
+            return 0
+
+        finally:
+            flipper.stop()
 
 
 if __name__ == "__main__":