Skip to content

Update python Docker tag to v3.12 #1982

Update python Docker tag to v3.12

Update python Docker tag to v3.12 #1982

name: CI
on:
push:
branches:
- main
tags:
- "*"
pull_request:
workflow_dispatch:
jobs:
lint:
name: Linting/style checks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: '.python-version'
check-latest: true
- name: Upgrade `pip` and install `wheel`
run: python -m pip install --upgrade pip wheel
- name: Install pre-commit
run: pip install pre-commit
- name: Cache pre-commit hooks
uses: actions/cache@v3
with:
path: ~/.cache/pre-commit/
key: pre-commit-cache|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}|${{ hashFiles('setup.cfg') }}
- name: Run pre-commit checks
run: pre-commit run --all-files --verbose
type-check:
name: Type-checking
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: '.python-version'
check-latest: true
- name: Upgrade `pip` and install `wheel`
run: python -m pip install --upgrade pip wheel
- name: Install tox
run: pip install tox
- name: Run type checking
run: tox -e type
# check-logic:
# name: Validate logic
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# - name: Checkout rando-shuffler repo
# uses: actions/checkout@v3
# with:
# repository: minishmaker/rando-shuffler
# path: rando-shuffler
# - name: Get cache key
# id: shuffler-cache-key
# run: echo "key=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
# shell: bash
# working-directory: rando-shuffler
# - uses: actions/cache@v3
# id: rando-shuffler-cache
# with:
# path: rando-shuffler
# key: ${{ steps.shuffler-cache-key.outputs.key }}
# - name: Compile rando-shuffler
# if: steps.rando-shuffler-cache.outputs.cache-hit != 'true'
# run: cargo build
# working-directory: rando-shuffler
# env:
# CARGO_TERM_COLOR: always
# - name: Make it executable
# run: chmod +x rando-shuffler/target/debug/rando_shuffler
# - name: Run the MCR shuffler
# run: |
# shopt -s globstar
# find rando-shuffler/ -name "*.logic" -type f -delete # remove test files
# file_count=0
# failed=0;
# for file in **/*.logic; do
# echo "$file... ";
# status=0
# tmpfile=$(mktemp /tmp/output.XXXXXX) # create temporary file to store output
# rando-shuffler/target/debug/rando_shuffler "$file" >> $tmpfile 2>&1 || status=1
# if [[ $status != 0 ]]; then
# failed=$((failed + 1));
# echo -e "\033[0;31mFAILED\033[0m"
# cat $tmpfile # print output of rando-shuffler if errors occured
# else
# echo -e "\033[0;32mPASSED\033[0m"
# fi
# echo "-------------------------------"
# echo ""
# file_count=$((file_count + 1))
# done;
# echo "RESULTS: $((file_count - failed)) / $file_count passed"
# if [[ $failed != 0 ]]; then
# exit 1;
# fi
check-aux-data:
name: Validate aux data
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: '.python-version'
check-latest: true
- name: Upgrade `pip` and install `wheel`
run: python -m pip install --upgrade pip wheel
- name: Install dependencies
run: pip install -e .
- name: Check if there are changes to pydantic models that aren't reflected in JSON schema
run: |
python ph_rando/shuffler/aux_models.py
git diff --exit-code
if [ $? -ne 0 ]
then
exit 1
fi
- name: Run JSON Schema validation on aux data
uses: walbo/[email protected]
with:
files: ph_rando/shuffler/logic/**/*.json
- name: Run pydantic validation on aux data
run: |
python <<HEREDOC
import json, sys
from pathlib import Path
from pydantic import ValidationError
from ph_rando.shuffler.aux_models import Area
failed = False
for file in (Path.cwd() / 'ph_rando' / 'shuffler' / 'logic').rglob('*.json'):
with open(file) as f:
try:
Area(**json.load(f))
except ValidationError as e:
failed = True
print(file.relative_to(Path.cwd()), file=sys.stderr, end='\n')
print(e, file=sys.stderr, end='\n\n\n')
if failed:
exit(1)
HEREDOC
check-randomizer-settings:
name: Validate randomizer settings
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: '.python-version'
check-latest: true
- name: Upgrade `pip` and install `wheel`
run: python -m pip install --upgrade pip wheel
- name: Install dependencies
run: pip install -e .
- name: Check if there are changes to pydantic models that aren't reflected in JSON schema
run: |
python ph_rando/settings.py
git diff --exit-code
if [ $? -ne 0 ]
then
exit 1
fi
- name: Run pydantic validation on settings json
run: |
python <<HEREDOC
import json, sys
from pathlib import Path
from pydantic import ValidationError
from ph_rando.settings import Settings
failed = False
with open(Path.cwd() / 'ph_rando' / 'settings.json') as f:
try:
Settings(**json.load(f))
except ValidationError as e:
failed = True
print(file.relative_to(Path.cwd()), file=sys.stderr, end='\n')
print(e, file=sys.stderr, end='\n\n\n')
if failed:
exit(1)
HEREDOC
run-shuffler:
name: Run the shuffler against logic and aux data
needs:
- check-aux-data
# - check-logic
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: '.python-version'
check-latest: true
- name: Upgrade `pip` and install `wheel`
run: python -m pip install --upgrade pip wheel
- name: Install dependencies
run: pip install -e .
- name: Run the shuffler
run: ph_rando_shuffler -o -- --spoiler-log spoiler.txt
- name: Print spoiler log
run: cat spoiler.txt
- name: Save spoiler log as an artifact
uses: actions/upload-artifact@v3
with:
name: spoiler.txt
path: spoiler.txt
tests:
name: Run tests
needs:
- check-aux-data
# - check-logic
strategy:
fail-fast: false
matrix:
os: [ubuntu, macos, windows]
defaults:
run:
shell: bash
runs-on: ${{ matrix.os }}-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: '.python-version'
check-latest: true
- name: Upgrade `pip` and install `wheel`
run: python -m pip install --upgrade pip wheel
- name: Install tox
run: pip install tox
- name: Run tests
run: tox -e test
generate-base-patch:
name: Build base patch [${{ matrix.rom_variant }}]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- rom_variant: US
google_drive_url_secret: PH_US_GOOGLE_DRIVE_ID
- rom_variant: US (D-Pad Patch)
google_drive_url_secret: PH_US_DPAD_GOOGLE_DRIVE_ID
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: '.python-version'
check-latest: true
- name: Upgrade `pip` and install `wheel`
run: python -m pip install --upgrade pip wheel
- name: Fetch PH rom
run: |
pip install gdown
gdown ${{ secrets[matrix.google_drive_url_secret] }}
working-directory: base
- name: Get filename of rom
id: rom-filename
run: echo "rom_filename=$(ls *.nds)" >> $GITHUB_OUTPUT
working-directory: base
- name: Get checksum of ROM
id: rom-checksum
run: echo "checksum=$(sha256sum '${{ steps.rom-filename.outputs.rom_filename }}' | awk '{ print $1 }')" >> $GITHUB_OUTPUT
working-directory: base
- name: Get current commit hash
id: commit-hash
run: echo "commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Build base patch
uses: docker/build-push-action@v5
with:
context: base/
push: false
# cache-from: type=gha
# cache-to: type=gha,mode=max
build-args: |
PH_ROM_PATH=${{ steps.rom-filename.outputs.rom_filename }}
VERSION_STRING=${{ steps.commit-hash.outputs.commit_hash }}
outputs: |
base/out
- name: Save base patch as an artifact
uses: actions/upload-artifact@v3
with:
name: ${{ steps.rom-checksum.outputs.checksum }}.bps
path: base/out/${{ steps.rom-checksum.outputs.checksum }}.bps
- name: Cleanup
if: always()
run: rm -rf *.nds
generate-base-patch-test-jobs:
name: Generate base patch test jobs
runs-on: ubuntu-latest
outputs:
should-run-tests: ${{ steps.filter.outputs.base-patches }}
job-matrix: ${{ steps.set-test-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v4
- name: Check if base patch tests should be run
uses: dorny/paths-filter@v2
id: filter
with:
filters: |
base-patches:
- 'base/**'
- 'tests/desmume/**'
- '.github/workflows/**'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: '.python-version'
check-latest: true
- name: Upgrade `pip` and install `wheel`
run: python -m pip install --upgrade pip wheel
- name: Install dependencies
run: |
pip install -e .[test]
- name: Get list of tests to run in separate jobs
id: set-test-matrix
run: |
python .github/generate_desmume_test_jobs.py
echo "matrix=$(cat .github/matrix.json)" >> $GITHUB_OUTPUT
test-desmume:
name: Run base patch tests [${{ matrix.module }}] (${{ matrix.rom_variant }})
needs:
- generate-base-patch
- generate-base-patch-test-jobs
# Only run these tests if base patches or the tests themselves are modified, or if a release is happening, or on push to main.
if: "startsWith(github.ref, 'refs/tags/') || needs.generate-base-patch-test-jobs.outputs.should-run-tests == 'true'"
# Windows is used to run the desmume-based tests because py-desmume has issues running on headless ubuntu
runs-on: windows-latest
defaults:
run:
shell: bash
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.generate-base-patch-test-jobs.outputs.job-matrix) }}
steps:
- uses: actions/checkout@v4
with:
# Also retrieve previous commit so we can check if there are relevant changes
fetch-depth: 2
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: '.python-version'
check-latest: true
- name: Upgrade `pip` and install `wheel`
run: python -m pip install --upgrade pip wheel
- name: Fetch PH rom
run: |
pip install gdown
gdown ${{ secrets[matrix.google_drive_url_secret] }}
- name: Get filename of rom
id: rom-filename
run: echo "filename=$(ls *.nds)" >> $GITHUB_OUTPUT
- name: Get checksum of ROM
id: rom-checksum
run: echo "checksum=$(sha256sum '${{ steps.rom-filename.outputs.filename }}' | awk '{ print $1 }')" >> $GITHUB_OUTPUT
- name: Download base patch from previous workflow
uses: actions/download-artifact@v3
with:
name: ${{ steps.rom-checksum.outputs.checksum }}.bps
- name: Move base patch to directory expected by patcher
run: |
mkdir -p base/out
mv ${{ steps.rom-checksum.outputs.checksum }}.bps base/out/${{ steps.rom-checksum.outputs.checksum }}.bps
- name: Install tox
run: pip install tox
- name: Run tests
run: PY_DESMUME_BATTERY_DIR="$pythonLocation" tox -e test-desmume -- -k "${{ matrix.module }}"
env:
PH_ROM_PATH: ${{ steps.rom-filename.outputs.filename }}
PY_DESMUME_VIDEO_RECORDING_DIR: recordings/
- name: Save recordings of failed tests as artifacts
if: failure()
uses: actions/upload-artifact@v3
with:
name: test_recordings_${{ matrix.module }}_${{ matrix.rom_variant }}
path: recordings/
- name: Cleanup
if: always()
run: rm -rf *.nds
build:
name: Build randomizer
needs:
- generate-base-patch
strategy:
fail-fast: false
matrix:
os: [ubuntu, macos, windows]
defaults:
run:
shell: bash
runs-on: ${{ matrix.os }}-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version-file: '.python-version'
check-latest: true
- name: Upgrade `pip` and install `wheel`
run: python -m pip install --upgrade pip wheel
- name: Install randomizer dependencies
run: pip install -e .
- name: Install pyinstaller
run: pip install -U pyinstaller
- name: Download base patches
uses: actions/download-artifact@v3
- name: Move all patch files into the same directory so pyinstaller can find them
run: |
mkdir -p base/out
mv ./**/*.bps base/out
- name: Build randomizer executable
run: pyinstaller ph_rando.spec
- name: Get current commit hash
id: commit-hash
run: echo "hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
- uses: actions/upload-artifact@v3
with:
name: ph-randomizer_${{ steps.commit-hash.outputs.hash }}_${{ matrix.os }}
path: dist/
run-executable:
name: Run built randomizer executable [${{ matrix.os }} / ${{ matrix.rom_variant }}]
needs: [build]
runs-on: ${{ matrix.os }}-latest
timeout-minutes: 10
defaults:
run:
shell: bash
strategy:
fail-fast: false
matrix:
os: [ubuntu, macos, windows]
rom_variant: [US, US (D-Pad Patch)]
include:
- rom_variant: US
google_drive_url_secret: PH_US_GOOGLE_DRIVE_ID
- rom_variant: US (D-Pad Patch)
google_drive_url_secret: PH_US_DPAD_GOOGLE_DRIVE_ID
steps:
- uses: actions/checkout@v4
# Note: intentionally use an older version of Python here;
# The pyinstaller build should not depend on a specific Python
# version being installed (or, Python being installed at all
# for that matter).
- name: Get two versions of Python behind min required
id: python-version
run: |
PYTHON_VERSION=$(cat .python-version)
echo "python-version="3.$(echo "${PYTHON_VERSION:2} 2" | awk '{print $1-$2}')"" >> $GITHUB_OUTPUT
rm -rf *
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ steps.python-version.outputs.python-version }}
- name: Upgrade `pip` and install `wheel`
run: python -m pip install --upgrade pip wheel
- name: Fetch PH rom
run: |
pip install gdown
gdown ${{ secrets[matrix.google_drive_url_secret] }}
- name: Get filename of rom
id: rom-filename
run: echo "rom_filename=$(ls *.nds)" >> $GITHUB_OUTPUT
- name: Get current commit hash
id: commit-hash
run: echo "hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
- name: Download built executable from previous job
uses: actions/download-artifact@v3
with:
name: ph-randomizer_${{ steps.commit-hash.outputs.hash }}_${{ matrix.os }}
- name: Run executable (no GUI)
run: |
chmod +x ./ph_rando
./ph_rando --no-gui --input-rom-path="${{ steps.rom-filename.outputs.rom_filename }}" --output-rom-path="out.nds"
- name: Cleanup
if: always()
run: rm -rf *.nds
release:
name: Publish a release
# Require all tests to pass before a release is possible
needs:
- build
- lint
- type-check
- tests
# - check-logic
- check-aux-data
- run-shuffler
- test-desmume
- run-executable
# Only create a release if a tag was pushed
if: "startsWith(github.ref, 'refs/tags/')"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Get current commit hash
id: commit-hash
run: echo "hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
- name: Get version string
id: version-string
run: echo "version=$(git describe --tags)" >> $GITHUB_OUTPUT
- name: Download built executables
uses: actions/download-artifact@v3
- name: Replace commit hash with version string
run: |
mv ph-randomizer_${{ steps.commit-hash.outputs.hash }}_ubuntu ph-randomizer_${{ steps.version-string.outputs.version }}_ubuntu
mv ph-randomizer_${{ steps.commit-hash.outputs.hash }}_macos ph-randomizer_${{ steps.version-string.outputs.version }}_macos
mv ph-randomizer_${{ steps.commit-hash.outputs.hash }}_windows ph-randomizer_${{ steps.version-string.outputs.version }}_windows
- name: Zip up executables
run: |
zip -r -j ph-randomizer_${{ steps.version-string.outputs.version }}_linux.zip ph-randomizer_${{ steps.version-string.outputs.version }}_ubuntu
zip -r -j ph-randomizer_${{ steps.version-string.outputs.version }}_macos.zip ph-randomizer_${{ steps.version-string.outputs.version }}_macos
zip -r -j ph-randomizer_${{ steps.version-string.outputs.version }}_windows.zip ph-randomizer_${{ steps.version-string.outputs.version }}_windows
- name: Create a GitHub Release
uses: softprops/action-gh-release@v1
with:
files: |
ph-randomizer_${{ steps.version-string.outputs.version }}_linux.zip
ph-randomizer_${{ steps.version-string.outputs.version }}_macos.zip
ph-randomizer_${{ steps.version-string.outputs.version }}_windows.zip