core_tests.yml

name: Core Testing

on:
  push:
    branches:
      - '*'

  pull_request:
    branches:
      - '*'

env:
  CACHE_NUMBER: 2  # increase to reset cache manually

jobs:
  foundation:

    strategy:
      matrix:
        python-version: ["3.10"]
    defaults:
      run:
        shell: bash -l {0}
    name: linux-64-py${{ matrix.python-version }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Miniforge
        uses: conda-incubator/setup-miniconda@v3
        with:
            miniforge-version: latest
            activate-environment: asim-test
            python-version: ${{ matrix.python-version }}

      - name: Set cache date for year and month
        run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

      - uses: actions/cache@v4
        with:
          path: /usr/share/miniconda3/envs/asim-test
          key: linux-64-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
        id: cache

      - name: Update environment
        run: |
          conda env update -n asim-test -f conda-environments/github-actions-tests.yml
        if: steps.cache.outputs.cache-hit != 'true'

      - name: Install activitysim
        # installing without dependencies is faster, we trust that all needed dependencies
        # are in the conda environment defined above.  Also, this avoids pip getting
        # confused and reinstalling tables (pytables).
        run: |
          python -m pip install -e . --no-deps

      - name: Conda checkup
        run: |
          conda info -a
          conda list

      - name: Lint with Black
        run: |
          # stop the build if there are problems
          black --check --diff .

      - name: Test activitysim.core
        run: |
          python -m pytest --pyargs activitysim.core

      - name: Test activitysim.abm.models
        run: |
          python -m pytest --pyargs activitysim.abm.models

      - name: Test activitysim.abm.test
        run: |
          python -m pytest --pyargs activitysim.abm.test

      - name: Test activitysim.cli
        run: |
          python -m pytest --pyargs activitysim.cli

      - name: Test activitysim.examples.test
        run: |
          python -m pytest --pyargs activitysim.examples.test


  cross-platform:
    # also test foundation cross platforms, but do not require a successful
    # completion before starting regional model tests
    needs: foundation
    strategy:
      matrix:
        include:
          - os: macos-latest
            label: macOS
            python-version: "3.10"

          - os: windows-latest
            label: win-64
            python-version: "3.10"

    defaults:
      run:
        shell: bash -l {0}

    name: ${{ matrix.label }}
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - name: Setup Miniforge
        uses: conda-incubator/setup-miniconda@v3
        with:
            miniforge-version: latest
            activate-environment: asim-test
            python-version: ${{ matrix.python-version }}

      - name: Set cache date for year and month
        run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

      - uses: actions/cache@v4
        with:
          path: ${{ env.CONDA }}/envs
          key: ${{ matrix.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
        id: cache

      - name: Update environment
        run: |
          conda env update -n asim-test -f conda-environments/github-actions-tests.yml
        if: steps.cache.outputs.cache-hit != 'true'

      - name: Install activitysim
        # installing without dependencies is faster, we trust that all needed dependencies
        # are in the conda environment defined above.  Also, this avoids pip getting
        # confused and reinstalling tables (pytables).
        run: |
          python -m pip install -e . --no-deps

      - name: Conda checkup
        run: |
          conda info -a
          conda list

      - name: Lint with Black
        run: |
          # stop the build if there are problems
          black --check --diff .

      - name: Test activitysim.core
        run: |
          python -m pytest --pyargs activitysim.core

      - name: Test activitysim.abm.models
        run: |
          python -m pytest --pyargs activitysim.abm.models

      - name: Test activitysim.abm.test
        run: |
          python -m pytest --pyargs activitysim.abm.test

      - name: Test activitysim.cli
        run: |
          python -m pytest --pyargs activitysim.cli


  builtin_regional_models:
    needs: foundation
    env:
      python-version: "3.10"
      label: linux-64
    strategy:
      matrix:
        region:
          - prototype_mtc
          - placeholder_psrc
          - prototype_marin
          - prototype_mtc_extended
          - placeholder_sandag
          - prototype_sandag_xborder
          - production_semcog
          - prototype_mwcog
          - placeholder_multiple_zone
      fail-fast: false
    defaults:
      run:
        shell: bash -l {0}
    name: ${{ matrix.region }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Miniforge
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-version: latest
          activate-environment: asim-test
          python-version: ${{ env.python-version }}

      - name: Set cache date for year and month
        run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

      - uses: actions/cache@v4
        with:
          path: ${{ env.CONDA }}/envs
          key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
        id: cache

      - name: Update environment
        run: |
          conda env update -n asim-test -f conda-environments/github-actions-tests.yml
        if: steps.cache.outputs.cache-hit != 'true'

      - name: Install activitysim
        # installing without dependencies is faster, we trust that all needed dependencies
        # are in the conda environment defined above.  Also, this avoids pip getting
        # confused and reinstalling tables (pytables).
        run: |
          python -m pip install -e . --no-deps

      - name: Conda checkup
        run: |
          conda info -a
          conda list

      # TODO: Cache sharrow compiled flows?  The contents of __pycache__ appear to
      #       be ignored, so this is not working as expected right now
      #
      #  - name: Define Inputs
      #    run: echo "REGION_DEFS=activitysim/examples/${{ matrix.region }}/** " >> $GITHUB_ENV
      #
      #  - name: Get a random number
      #    run: echo "RANDOM_SUFFIX=${RANDOM}${RANDOM}" >> $GITHUB_ENV
      #
      #  - uses: actions/cache@v3
      #    # store the regional model's cache directory in github actions cache
      #    # this will (almost) never hit on primary key due to the random number
      #    # but will pull the most recent cache from restore-keys... and then
      #    # update the cache with additional compiled flows as needed.
      #    # Hoping this will result in fewer re-compiles on tests and faster
      #    # testing overall
      #    with:
      #      path: activitysim/examples/${{ matrix.region }}/test/output/cache
      #      key: ${{ matrix.region }}-${{ env.label }}-${{ hashFiles(env.REGION_DEFS) }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}-${{ env.RANDOM_SUFFIX }}
      #      restore-keys: |
      #        ${{ matrix.region }}-${{ env.label }}-${{ hashFiles(env.REGION_DEFS) }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
      #    id: cache-example

      - name: Test ${{ matrix.region }}
        run: |
          python -m pytest activitysim/examples/${{ matrix.region }}/test --durations=0

  external_regional_models:
    needs: foundation
    env:
      python-version: "3.10"
      label: linux-64
    strategy:
      matrix:
        include:
          - region: Standard 1-Zone Example (MTC)
            region-org: ActivitySim
            region-repo: activitysim-prototype-mtc
            region-branch: extended
          - region: Standard 2-Zone Example (SANDAG)
            region-org: ActivitySim
            region-repo: sandag-abm3-example
            region-branch: main
      fail-fast: false
    defaults:
      run:
        shell: bash -l {0}
    name: ${{ matrix.region }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout ActivitySim
        uses: actions/checkout@v4

      - name: Setup Miniforge
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-version: latest
          activate-environment: asim-test
          python-version: ${{ env.python-version }}

      - name: Set cache date for year and month
        run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

      - uses: actions/cache@v4
        with:
          path: |
            ${{ env.CONDA }}/envs
            ~/.cache/ActivitySim
          key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
        id: cache

      - name: Update environment
        run: |
          conda env update -n asim-test -f conda-environments/github-actions-tests.yml
        if: steps.cache.outputs.cache-hit != 'true'

      - name: Install activitysim
        # installing without dependencies is faster, we trust that all needed dependencies
        # are in the conda environment defined above.  Also, this avoids pip getting
        # confused and reinstalling tables (pytables).
        run: |
          python -m pip install . --no-deps

      - name: Conda checkup
        run: |
          conda info -a
          conda list

      - name: Checkout Example
        uses: actions/checkout@v4
        with:
          repository: '${{ matrix.region-org }}/${{ matrix.region-repo }}'
          ref: '${{ matrix.region-branch }}'
          path: '${{ matrix.region-repo }}'

      - name: Test ${{ matrix.region }}
        run: |
          cd ${{ matrix.region-repo }}/test
          python -m pytest .

  random_seed_generation:
    needs: foundation
    env:
      python-version: "3.10"
      label: linux-64
    defaults:
      run:
        shell: bash -l {0}
    name: random_seed_generation_test
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Setup Miniforge
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-version: latest
          activate-environment: asim-test
          python-version: ${{ env.python-version }}

      - name: Set cache date for year and month
        run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

      - uses: actions/cache@v4
        with:
          path: ${{ env.CONDA }}/envs
          key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
        id: cache

      - name: Update environment
        run: |
          conda env update -n asim-test -f conda-environments/github-actions-tests.yml
        if: steps.cache.outputs.cache-hit != 'true'

      - name: Install activitysim
        # installing without dependencies is faster, we trust that all needed dependencies
        # are in the conda environment defined above.  Also, this avoids pip getting
        # confused and reinstalling tables (pytables).
        run: |
          python -m pip install -e . --no-deps

      - name: Conda checkup
        run: |
          conda info -a
          conda list

      - name: Test Random Seed Generation
        run: |
          python -m pytest test/random_seed/test_random_seed.py --durations=0

  estimation_mode:
    needs: foundation
    env:
      python-version: "3.10"
      label: linux-64
    defaults:
      run:
        shell: bash -l {0}
    name: Estimation Mode Unit Tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Miniforge
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-version: latest
          activate-environment: asim-test
          python-version: ${{ env.python-version }}

      - name: Set cache date for year and month
        run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

      - uses: actions/cache@v4
        with:
          path: ${{ env.CONDA }}/envs
          key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
        id: cache

      - name: Update environment
        run: |
          conda env update -n asim-test -f conda-environments/github-actions-tests.yml
        if: steps.cache.outputs.cache-hit != 'true'

      - name: Install Larch v6
        run: python -m pip install larch6

      - name: Install activitysim
        # installing without dependencies is faster, we trust that all needed dependencies
        # are in the conda environment defined above.  Also, this avoids pip getting
        # confused and reinstalling tables (pytables).
        run: |
          python -m pip install -e . --no-deps

      - name: Conda checkup
        run: |
          conda info -a
          conda list

      - name: Test Estimation Mode
        run: |
          python -m pytest activitysim/estimation/test/test_larch_estimation.py --durations=0

  estimation_notebooks:
    needs: foundation
    env:
      python-version: "3.10"
      label: linux-64
    defaults:
      run:
        shell: bash -l {0}
    name: Estimation Notebooks Test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Miniforge
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-version: latest
          activate-environment: asim-test
          python-version: ${{ env.python-version }}

      - name: Set cache date for year and month
        run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

      - uses: actions/cache@v4
        with:
          path: ${{ env.CONDA }}/envs
          key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
        id: cache

      - name: Update environment
        run: |
          conda env update -n asim-test -f conda-environments/github-actions-tests.yml
        if: steps.cache.outputs.cache-hit != 'true'

      - name: Install Graphviz
        uses: ts-graphviz/setup-graphviz@v2

      - name: Install Larch v6
        run: python -m pip install larch6 "pandas<2" pydot

      - name: Install activitysim
        # installing without dependencies is faster, we trust that all needed dependencies
        # are in the conda environment defined above.  Also, this avoids pip getting
        # confused and reinstalling tables (pytables).
        run: |
          python -m pip install -e . --no-deps

      - name: Conda checkup
        run: |
          conda info -a
          conda list

      - name: Create Estimation Data
        run: |
          python activitysim/examples/example_estimation/notebooks/est_mode_setup.py --household_sample_size 5000

      - name: Test Estimation Notebooks
        run: |
          python -m pytest activitysim/examples/example_estimation/notebooks/*.ipynb \
            --nbmake-timeout=3000 \
            --ignore=activitysim/examples/example_estimation/notebooks/01_estimation_mode.ipynb

  estimation_edb_creation:
    needs: foundation
    env:
      python-version: "3.10"
      label: linux-64
    defaults:
      run:
        shell: bash -l {0}
    name: estimation_edb_creation_test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Miniforge
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-version: latest
          activate-environment: asim-test
          python-version: ${{ env.python-version }}

      - name: Set cache date for year and month
        run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

      - uses: actions/cache@v4
        with:
          path: ${{ env.CONDA }}/envs
          key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
        id: cache

      - name: Update environment
        run: |
          conda env update -n asim-test -f conda-environments/github-actions-tests.yml
        if: steps.cache.outputs.cache-hit != 'true'

      - name: Install activitysim
        # installing without dependencies is faster, we trust that all needed dependencies
        # are in the conda environment defined above.  Also, this avoids pip getting
        # confused and reinstalling tables (pytables).
        run: |
          python -m pip install -e . --no-deps

      - name: Conda checkup
        run: |
          conda info -a
          conda list

      - name: Test Estimation EDB Creation
        run: |
          python -m pytest activitysim/estimation/test/test_edb_creation/test_edb_formation.py --durations=0

  develop-docbuild:
    needs: foundation
    if: github.ref_name == 'main'
    name: develop-docbuild
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0  # get all tags, lets setuptools_scm do its thing
      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Install dependencies
        uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-version: latest
          environment-file: conda-environments/docbuild.yml
          python-version: "3.10"
          activate-environment: docbuild
          auto-activate-base: false
          auto-update-conda: false
      - name: Install activitysim
        run: |
          python -m pip install .
      - name: Conda checkup
        run: |
          conda info -a
          conda list
          echo ${{ github.repository }}
          echo ${{ github.ref_name }}
      - name: localize version switcher
        run: |
          python .github/workflows/localize-base-urls.py docs/_static/switcher.json
          git update-index --assume-unchanged docs/_static/switcher.json
          cat docs/_static/switcher.json
      - name: Build the docs
        run: |
          cd docs
          make clean
          make html
      - name: Push to GitHub Pages
        uses: peaceiris/actions-gh-pages@v3.8.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          # Token is created automatically by Github Actions, no other config needed
          publish_dir: ./docs/_build/html
          destination_dir: develop