MicroPython: Upgrade to latest build setup.

Borrows the build setup from Badger OS which-

* Uses a uf2-manifest and dir2uf2 to append examples in a user-visible filesystem
* Uses MICROPY_BOARD_DIR to specify an out-of-tree board def
* Uses ccache and caches MicroPython (compiled tools)
* Builds against upstream MicroPython

.github/workflows/micropython.yml

@@ -1,4 +1,4 @@
-name: MicroPython
+name: MicroPython Firmware
 
 on:
   push:
@@ -7,96 +7,171 @@ on:
     types: [created]
 
 env:
-  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
-  MICROPYTHON_VERSION: v1.19
-  BUILD_TYPE: Release
-  BOARD_TYPE: PIMORONI_PICOSYSTEM
+  MICROPYTHON_VERSION: 668a7bd28a49980b239fd7666684885382526988
 
 jobs:
-  build:
-    name: ${{matrix.name}}
-    strategy:
-      matrix:
-        include:
-          - os: ubuntu-20.04
-            name: MicroPython Build (Linux)
-            cache-key: linux
-            apt-packages: clang-tidy gcc-arm-none-eabi libnewlib-arm-none-eabi libstdc++-arm-none-eabi-newlib
-
-    runs-on: ${{matrix.os}}
-
+  deps:
+    runs-on: ubuntu-20.04
+    name: Dependencies
     steps:
+    - name: Workspace Cache
+      id: cache
+      uses: actions/cache@v3
+      with:
+        path: ${{runner.workspace}}
+        key: workspace-micropython-${{env.MICROPYTHON_VERSION}}
+        restore-keys: |
+          workspace-micropython-${{env.MICROPYTHON_VERSION}}
+
     # Check out MicroPython
     - name: Checkout MicroPython
-      uses: actions/checkout@v2
+      if: steps.cache.outputs.cache-hit != 'true'
+      uses: actions/checkout@v3
       with:
-        repository: pimoroni/micropython
-        ref: picosystem-nohax-rebase
+        repository: micropython/micropython
+        ref: ${{env.MICROPYTHON_VERSION}}
         submodules: false  # MicroPython submodules are hideously broken
         path: micropython
 
-    - uses: actions/checkout@v2
-      with:
-        submodules: true
-        path: picosystem-${{ github.sha }}
-
-    # HACK: Copy Python module files
-    - name: Copy modules
-      run: |
-        cp -r picosystem-${GITHUB_SHA}/micropython/modules_py/* micropython/ports/rp2/modules/
-        cp -r picosystem-${GITHUB_SHA}/micropython/examples/picosystem/colour.py micropython/ports/rp2/modules/
-        cp -r picosystem-${GITHUB_SHA}/micropython/examples/picosystem/shapes.py micropython/ports/rp2/modules/
-        cp -r picosystem-${GITHUB_SHA}/micropython/examples/picosystem/text.py micropython/ports/rp2/modules/
-        cp -r picosystem-${GITHUB_SHA}/micropython/examples/picosystem/music.py micropython/ports/rp2/modules/
-        cp -r picosystem-${GITHUB_SHA}/micropython/examples/picosystem/audio.py micropython/ports/rp2/modules/
-        cp -r picosystem-${GITHUB_SHA}/micropython/examples/picosystem/launcher.py micropython/ports/rp2/modules/
-
-    # Linux deps
-    - name: Install deps
-      if: runner.os == 'Linux'
-      run: |
-        sudo apt update && sudo apt install ${{matrix.apt-packages}}
-
     - name: Fetch base MicroPython submodules
+      if: steps.cache.outputs.cache-hit != 'true'
       shell: bash
       working-directory: micropython
       run: git submodule update --init
 
     - name: Fetch Pico SDK submodules
+      if: steps.cache.outputs.cache-hit != 'true'
       shell: bash
       working-directory: micropython/lib/pico-sdk
       run: git submodule update --init
 
     - name: Build mpy-cross
+      if: steps.cache.outputs.cache-hit != 'true'
       shell: bash
       working-directory: micropython/mpy-cross
       run: make
 
+  build:
+    needs: deps
+    name: Build ${{matrix.name}}
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        include:
+          - name: PicoSystem
+            shortname: picosystem
+            board: PIMORONI_PICOSYSTEM
+            # 0x10000000 + (16*1024*1024) - (15*1024*1024)  ( 1MB firmware + 15MB filesystem )
+            dir2uf2_fs_start: 269484032
+            dir2uf2_fs_size: 7340032
+
+    env:
+      RELEASE_FILE: pimoroni-${{matrix.shortname}}-${{github.event.release.tag_name || github.sha}}-micropython.uf2
+      RELEASE_FILE_WITH_OS: pimoroni-${{matrix.shortname}}-${{github.event.release.tag_name || github.sha}}-micropython-with-examples.uf2
+      BOARD_DIR: "$GITHUB_WORKSPACE/picosystem/micropython/${{matrix.board}}"
+      EXAMPLES_DIR: "$GITHUB_WORKSPACE/picosystem/micropython/examples/picosystem/"
+      C_MODULES_DIR: "$GITHUB_WORKSPACE/picosystem/micropython/modules/"
+
+    steps:
+    - name: Compiler Cache
+      uses: actions/cache@v3
+      with:
+        path: /home/runner/.ccache
+        key: ccache-micropython-${{matrix.shortname}}-${{github.ref}}-${{github.sha}}
+        restore-keys: |
+          ccache-micropython-${{matrix.shortname}}-${{github.ref}}
+          ccache-micropython-${{matrix.shortname}}-
+
+    - name: Workspace Cache
+      uses: actions/cache@v3
+      with:
+        path: ${{runner.workspace}}
+        key: workspace-micropython-${{env.MICROPYTHON_VERSION}}
+        restore-keys: |
+          workspace-micropython-${{env.MICROPYTHON_VERSION}}
+
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+        path: picosystem
+
+    # Check out Pimoroni Pico
+    - uses: actions/checkout@v3
+      with:
+        repository: pimoroni/pimoroni-pico
+        submodules: true
+        path: pimoroni-pico
+
+    # Check out dir2u2f
+    - uses: actions/checkout@v3
+      with:
+        repository: gadgetoid/dir2uf2
+        ref: v0.0.1
+        path: dir2uf2
+
+    # Install apt packages
+    - name: Install CCache & Compiler
+      shell: bash
+      run:
+        sudo apt update && sudo apt install ccache gcc-arm-none-eabi
+
+    # Build firmware
+    - name: Configure MicroPython
+      shell: bash
+      working-directory: micropython/ports/rp2
+      run: |
+        cmake -S . -B build -DPICO_BUILD_DOCS=0 -DUSER_C_MODULES=${{env.C_MODULES_DIR}}/micropython.cmake -DMICROPY_BOARD_DIR=${{env.BOARD_DIR}} -DMICROPY_BOARD= -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+
     - name: Build MicroPython
       shell: bash
       working-directory: micropython/ports/rp2
-      run: make USER_C_MODULES=../../../picosystem-${GITHUB_SHA}/micropython/modules/micropython.cmake BOARD=${BOARD_TYPE} -j2
+      run: |
+        ccache --zero-stats || true
+        cmake --build build -j 2
+        ccache --show-stats || true
 
-    - name: Rename .uf2 for artifact & release
+    - name: Rename .uf2 for artifact
       shell: bash
-      working-directory: micropython/ports/rp2/build-${{env.BOARD_TYPE}}
+      working-directory: micropython/ports/rp2/build
       run: |
-        cp firmware.uf2 ${{github.event.repository.name}}-${{github.sha}}-micropython-${{env.MICROPYTHON_VERSION}}.uf2
-        cp firmware.uf2 ${{github.event.repository.name}}-${{github.event.release.tag_name}}-micropython-${{env.MICROPYTHON_VERSION}}.uf2
+        cp firmware.uf2 ${{env.RELEASE_FILE}}
+
+    - name: Append Filesystem
+      shell: bash
+      run: |
+        python3 -m pip install littlefs-python
+        ./dir2uf2/dir2uf2 --fs-start ${{matrix.dir2uf2_fs_start}} --fs-size ${{matrix.dir2uf2_fs_size}} --append-to micropython/ports/rp2/build/${{env.RELEASE_FILE}} --manifest ${{env.BOARD_DIR}}/uf2-manifest.txt --filename with-examples.uf2 ${{env.EXAMPLES_DIR}}/
 
     - name: Store .uf2 as artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
-        name: ${{github.event.repository.name}}-${{github.sha}}-micropython-${{env.MICROPYTHON_VERSION}}.uf2
-        path: micropython/ports/rp2/build-${{env.BOARD_TYPE}}/${{github.event.repository.name}}-${{github.sha}}-micropython-${{env.MICROPYTHON_VERSION}}.uf2
+        name: ${{env.RELEASE_FILE}}
+        path: micropython/ports/rp2/build/${{env.RELEASE_FILE}}
+
+    - name: Store .uf2 + Examples as artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{env.RELEASE_FILE_WITH_OS}}
+        path: ${{env.RELEASE_FILE_WITH_OS}}
 
     - name: Upload .uf2
       if: github.event_name == 'release'
       uses: actions/upload-release-asset@v1
       env:
         GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
       with:
-        asset_path: micropython/ports/rp2/build-${{env.BOARD_TYPE}}/firmware.uf2
+        asset_path: micropython/ports/rp2/build/${{env.RELEASE_FILE}}
+        upload_url: ${{github.event.release.upload_url}}
+        asset_name: ${{env.RELEASE_FILE}}
+        asset_content_type: application/octet-stream
+
+    - name: Upload .uf2 + Examples
+      if: github.event_name == 'release'
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+      with:
+        asset_path: ${{env.RELEASE_FILE_WITH_OS}}
         upload_url: ${{github.event.release.upload_url}}
-        asset_name: ${{github.event.repository.name}}-${{github.event.release.tag_name}}-micropython-${{env.MICROPYTHON_VERSION}}.uf2
+        asset_name: ${{env.RELEASE_FILE_WITH_OS}}
         asset_content_type: application/octet-stream

micropython/PIMORONI_PICOSYSTEM/manifest.py

@@ -0,0 +1,7 @@
+freeze("$(PORT_DIR)/modules/")
+freeze("$(BOARD_DIR)/../modules_py/")
+include("$(MPY_DIR)/extmod/uasyncio")
+require("onewire")
+require("ds18x20")
+require("dht")
+require("neopixel")

micropython/PIMORONI_PICOSYSTEM/mpconfigboard.cmake

@@ -0,0 +1,5 @@
+# cmake file for Pimoroni PicoSystem
+set(MICROPY_BOARD PIMORONI_PICOSYSTEM)
+
+# Board specific version of the frozen manifest
+set(MICROPY_FROZEN_MANIFEST ${CMAKE_CURRENT_LIST_DIR}/manifest.py)

micropython/PIMORONI_PICOSYSTEM/mpconfigboard.h

@@ -0,0 +1,2 @@
+#define MICROPY_HW_BOARD_NAME          "Pimoroni PicoSystem"
+#define MICROPY_HW_FLASH_STORAGE_BYTES (15 * 1024 * 1024)

micropython/PIMORONI_PICOSYSTEM/uf2-manifest.txt

@@ -0,0 +1 @@
+*.py

micropython/modules/picosystem/micropython.cmake

@@ -1,8 +1,6 @@
-set(MOD_NAME picosystem)
-string(TOUPPER ${MOD_NAME} MOD_NAME_UPPER)
-add_library(usermod_${MOD_NAME} INTERFACE)
+add_library(usermod_picosystem INTERFACE)
 
-target_sources(usermod_${MOD_NAME} INTERFACE
+target_sources(usermod_picosystem INTERFACE
     ${CMAKE_CURRENT_LIST_DIR}/picosystem.c
     ${CMAKE_CURRENT_LIST_DIR}/picosystem.cpp
     ${CMAKE_CURRENT_LIST_DIR}/voice.cpp
@@ -27,16 +25,20 @@ target_sources(usermod_${MOD_NAME} INTERFACE
 pico_generate_pio_header(usermod_picosystem ${CMAKE_CURRENT_LIST_DIR}/../../../libraries/screen.pio)
 pico_generate_pio_header(usermod_picosystem ${CMAKE_CURRENT_LIST_DIR}/../../../libraries/screen_double.pio)
 
-target_include_directories(usermod_${MOD_NAME} INTERFACE
+target_include_directories(usermod_picosystem INTERFACE
     ${CMAKE_CURRENT_LIST_DIR}
 )
 
-target_compile_definitions(usermod_${MOD_NAME} INTERFACE
-    MODULE_${MOD_NAME_UPPER}_ENABLED=1
+target_compile_definitions(usermod_picosystem INTERFACE
+    MODULE_PICOSYSTEM_ENABLED=1
     PIXEL_DOUBLE=1
 )
 
-target_link_libraries(usermod INTERFACE usermod_${MOD_NAME})
+target_link_libraries(usermod_picosystem INTERFACE
+    hardware_vreg
+)
+
+target_link_libraries(usermod INTERFACE usermod_picosystem)
 
 set_source_files_properties(
     ${CMAKE_CURRENT_LIST_DIR}/picosystem.c