Skip to content

Commit

Permalink
Changes to support sharrow on 2-zone model (#867)
Browse files Browse the repository at this point in the history 
* test external regional model examples

* cache buster

* optional variable doc

* fix conda cache dirs

* trip_destination alts preprocessor

* non_hh_veh cat, drop unused cols for alts

* blacken

* adding missed alts columns used in xborder model

* remove unneeded addition to categorical

* clearer time logging

* bump required numba to 0.57 for np.nan_to_num

* sharrow docs

* use compute_setting in sharrow debugging

* fix comment

* debug helper values

* dtype compute fixes

* land_use_columns_orig

* fix and test orig_land_use with explicit chunking

* repair

* add missing test result file

* omx_ignore_patterns

* revert change to drop size terms

* creating separate sample and simulate preprocessors

* bugfix

* skim_dataset loading without dask

* require sharrow 2.9

* wait to close open files

* require sharrow 2.9.1

* landuse index sort before sharrow recode check

* decode time periods

* use original tazs where possible

* update numba in envs to 0.57

* no fastmath in tour mode choice

* sharrow cache by version

* include sharrow setting in log by defualt

* use dask if required

* store_skims_in_shm setting

---------

Co-authored-by: David Hensle <[email protected]>
Co-authored-by: David Hensle <[email protected]>
  • Loading branch information
@jpn-- @dhensle
3 people authored Jul 3, 2024
1 parent bd48d3d commit 28a0ad0
Show file tree
Hide file tree
Showing 19 changed files with 386 additions and 208 deletions.
161 changes: 46 additions & 115 deletions .github/workflows/core_tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ on:
- '*'

env:
CACHE_NUMBER: 0 # increase to reset cache manually
CACHE_NUMBER: 2 # increase to reset cache manually

jobs:
foundation:
Expand All @@ -24,10 +24,10 @@ jobs:
name: linux-64-py${{ matrix.python-version }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
Expand All @@ -38,7 +38,7 @@ jobs:
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

- uses: actions/cache@v3
- uses: actions/cache@v4
with:
path: /usr/share/miniconda3/envs/asim-test
key: linux-64-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
Expand All @@ -47,19 +47,6 @@ jobs:
- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=2.6.1" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'

- name: Install activitysim
Expand Down Expand Up @@ -109,12 +96,10 @@ jobs:
include:
- os: macos-latest
label: macOS
prefix: /Users/runner/miniconda3/envs/asim-test
python-version: "3.10"

- os: windows-latest
label: win-64
prefix: C:\Miniconda3\envs\asim-test
python-version: "3.10"

defaults:
Expand All @@ -124,10 +109,10 @@ jobs:
name: ${{ matrix.label }}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
Expand All @@ -138,28 +123,15 @@ jobs:
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

- uses: actions/cache@v3
- uses: actions/cache@v4
with:
path: ${{ matrix.prefix }}
path: ${{ env.CONDA }}/envs
key: ${{ matrix.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
id: cache

- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=2.6.1" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'

- name: Install activitysim
Expand Down Expand Up @@ -199,7 +171,6 @@ jobs:
builtin_regional_models:
needs: foundation
env:
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
python-version: "3.10"
label: linux-64
strategy:
Expand All @@ -221,10 +192,10 @@ jobs:
name: ${{ matrix.region }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
Expand All @@ -235,28 +206,15 @@ jobs:
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

- uses: actions/cache@v3
- uses: actions/cache@v4
with:
path: ${{ env.mamba-env-prefix }}
path: ${{ env.CONDA }}/envs
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
id: cache

- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=2.6.1" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'

- name: Install activitysim
Expand Down Expand Up @@ -301,25 +259,31 @@ jobs:
external_regional_models:
needs: foundation
env:
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
python-version: "3.10"
label: linux-64
strategy:
matrix:
region:
- prototype_mtc
- prototype_psrc_in_development
include:
- region: Standard 1-Zone Example (MTC)
region-org: ActivitySim
region-repo: activitysim-prototype-mtc
region-branch: extended
- region: Standard 2-Zone Example (SANDAG)
region-org: ActivitySim
region-repo: sandag-abm3-example
region-branch: main
fail-fast: false
defaults:
run:
shell: bash -l {0}
name: ${{ matrix.region }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Checkout ActivitySim
uses: actions/checkout@v4

- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
Expand All @@ -330,52 +294,46 @@ jobs:
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

- uses: actions/cache@v3
- uses: actions/cache@v4
with:
path: |
${{ env.mamba-env-prefix }}
${{ env.CONDA }}/envs
~/.cache/ActivitySim
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
id: cache

- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=2.6.1" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'

- name: Install activitysim
# installing without dependencies is faster, we trust that all needed dependencies
# are in the conda environment defined above. Also, this avoids pip getting
# confused and reinstalling tables (pytables).
run: |
python -m pip install -e . --no-deps
python -m pip install . --no-deps
- name: Conda checkup
run: |
mamba info -a
mamba list
- name: Checkout Example
uses: actions/checkout@v4
with:
repository: '${{ matrix.region-org }}/${{ matrix.region-repo }}'
ref: '${{ matrix.region-branch }}'
path: '${{ matrix.region-repo }}'

- name: Test ${{ matrix.region }}
run: |
python -m activitysim test ${{ matrix.region }}
cd ${{ matrix.region-repo }}/test
python -m pytest .
random_seed_generation:
needs: foundation
env:
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
python-version: "3.10"
label: linux-64
defaults:
Expand All @@ -385,10 +343,10 @@ jobs:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
Expand All @@ -399,28 +357,15 @@ jobs:
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

- uses: actions/cache@v3
- uses: actions/cache@v4
with:
path: ${{ env.mamba-env-prefix }}
path: ${{ env.CONDA }}/envs
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
id: cache

- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=2.6.1" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'

- name: Install activitysim
Expand All @@ -442,7 +387,6 @@ jobs:
estimation_mode:
needs: foundation
env:
mamba-env-prefix: /usr/share/miniconda3/envs/asim-test
python-version: "3.10"
label: linux-64
defaults:
Expand All @@ -451,10 +395,10 @@ jobs:
name: estimation_mode_test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
Expand All @@ -465,28 +409,15 @@ jobs:
- name: Set cache date for year and month
run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV

- uses: actions/cache@v3
- uses: actions/cache@v4
with:
path: ${{ env.mamba-env-prefix }}
path: ${{ env.CONDA }}/envs
key: ${{ env.label }}-conda-${{ hashFiles('conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
id: cache

- name: Update environment
run: |
mamba env update -n asim-test -f conda-environments/github-actions-tests.yml
mamba install --yes \
"psutil=5.9.5" \
"pydantic=2.6.1" \
"pypyr=5.8.0" \
"pytables=3.6.1" \
"pytest-cov" \
"pytest-regressions=2.5.0" \
"scikit-learn=1.2.2" \
"sharrow>=2.6.0" \
"simwrapper=1.8.5" \
"xarray=2023.2.0" \
"zarr=2.14.2" \
"zstandard=0.21.0"
if: steps.cache.outputs.cache-hit != 'true'

- name: Install Larch
Expand Down Expand Up @@ -517,15 +448,15 @@ jobs:
run:
shell: bash -l {0}
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0 # get all tags, lets setuptools_scm do its thing
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Install dependencies
uses: conda-incubator/setup-miniconda@v2
uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Mambaforge
miniforge-version: latest
Expand Down
Loading

0 comments on commit 28a0ad0

Please sign in to comment.