Nightly CLI Execution Tests (Published + BMDB) #476
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly CLI Execution Tests (Published + BMDB) | |
| on: | |
| schedule: | |
| - cron: "0 7 * * *" | |
| workflow_dispatch: | |
| env: | |
| python-version: "3.9" | |
| poetry-version: "1.2.1" | |
| jobs: | |
| build: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout Our Code | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: '${{ github.ref }}' | |
| - name: Get GH Reference | |
| run: echo '${{ github.ref }}' >> ~/github.ref | |
| - name: Upload Ref | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: github_hash | |
| path: ~/github.ref | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.python-version }} | |
| - name: Install poetry | |
| run: python3 -m pip install poetry==${{ env.poetry-version }} | |
| - name: Install other python dependencies | |
| run: python3 -m pip install -r requirements.txt | |
| - name: build vcell-cli-utils package | |
| run: | | |
| cd vcell-cli-utils | |
| poetry install | |
| poetry run python -m pytest | |
| - name: build vcell-admin package | |
| run: | | |
| cd docker/swarm/vcell-admin | |
| poetry install | |
| poetry run python -m pytest | |
| - name: build pythonCopasiOpt package | |
| run: | | |
| cd pythonCopasiOpt/vcell-opt | |
| poetry install | |
| poetry run python -m pytest | |
| - name: build pythonVtk package | |
| run: | | |
| cd pythonVtk | |
| poetry install | |
| poetry run python -m pytest | |
| - name: build vcutils package | |
| run: | | |
| cd python-utils | |
| poetry install | |
| poetry run python -m pytest | |
| - name: build python-restclient package | |
| run: | | |
| cd python-restclient | |
| poetry install | |
| poetry run python -m pytest | |
| - name: build vcelldata package | |
| run: | | |
| cd pythonData | |
| poetry install | |
| poetry run python -m pytest | |
| - name: Setup Java for Build | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '17' | |
| distribution: 'temurin' | |
| cache: 'maven' | |
| - name: Build VCell Jars | |
| run: mvn clean install dependency:copy-dependencies -DskipTests=True | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build and Export | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| file: ./Dockerfile | |
| outputs: type=docker,dest=/tmp/image.tar | |
| build-args: | # Max ram for Ubuntu GitHub workflow is 7GB (as of 12/9/22), using MB for precision (0.8 of 7gb) | |
| MAX_JAVA_MEM=5734 | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: docker-image | |
| path: /tmp/image.tar | |
| # Report a Problem to Slack | |
| - name: Report Problem to Slack | |
| if: ${{ failure() }} | |
| run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Uh-Oh! The *Build Step* of Nightly Execution Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} )! This may need attention..." https://slack.com/api/chat.postMessage | |
| # Tmate just in case | |
| - name: Setup tmate session 3 | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ failure() }} | |
| published: | |
| runs-on: ubuntu-20.04 | |
| needs: build | |
| strategy: | |
| matrix: | |
| sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"] | |
| steps: | |
| - name: Create Docker Image Dir | |
| run: mkdir /tmp/docker | |
| - name: Download Docker Image from `build` Job | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: docker-image | |
| path: /tmp/docker | |
| - name: Load Downloaded Docker Image | |
| run: docker load --input /tmp/docker/$(ls /tmp/docker) | |
| # Get files from BMDB | |
| - name: Checkout Input Files | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: virtualcell/vcdb | |
| - name: Prepare sub directory | |
| run: | | |
| SOURCE_DIR="$GITHUB_WORKSPACE/published/biomodel/omex/sbml" | |
| TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}" | |
| MOD_VALUE=${{ matrix.sets }} | |
| file_index=0 | |
| # make target dir | |
| mkdir $TARGET_DIR | |
| # loop through files in directory, increment indexing and separating files | |
| for file in $SOURCE_DIR/*; do | |
| file_index=$((file_index+1)) | |
| if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then | |
| cp ${file} $TARGET_DIR/ | |
| fi | |
| done | |
| # Do the execution | |
| - name: Begin Execution | |
| #$(docker image ls | grep "<none>" | awk '{print $3;}') to get image id | |
| run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs | |
| - name: Check For Success | |
| run: | | |
| errorSearch=$(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) | |
| if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -eq 0 ] | |
| then | |
| exit 0 | |
| else | |
| # Copy error log to github workspace | |
| cp $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) ${{ github.workspace }}/errors.report | |
| fi | |
| - name: Upload failures | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: set_${{ matrix.sets }}.report | |
| path: ${{ github.workspace }}/errors.report | |
| # Report a Problem to Slack | |
| - name: Report Problem to Slack | |
| if: ${{ failure() }} | |
| run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Oops! The *Published Step* of set ${{ matrix.sets }} in Nightly **Execution** Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ). Go see what happened!" https://slack.com/api/chat.postMessage | |
| - name: Setup tmate session 3 | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ failure() }} | |
| bmdb: | |
| runs-on: ubuntu-20.04 | |
| needs: build | |
| strategy: | |
| matrix: | |
| sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"] | |
| steps: | |
| - name: Create Docker Image Dir | |
| run: mkdir /tmp/docker | |
| - name: Download Docker Image from `build` Job | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: docker-image | |
| path: /tmp/docker | |
| - name: Load Downloaded Docker Image | |
| run: docker load --input /tmp/docker/$(ls /tmp/docker) | |
| # Get files from BMDB | |
| - name: Checkout Input Files | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: sys-bio/temp-biomodels | |
| - name: Prepare sub directory | |
| run: | | |
| SOURCE_DIR="$GITHUB_WORKSPACE/omex_files" | |
| TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}" | |
| MOD_VALUE=${{ matrix.sets }} | |
| file_index=0 | |
| # make target dir | |
| mkdir $TARGET_DIR | |
| # loop through files in directory, increment indexing and separating files | |
| for file in $SOURCE_DIR/*; do | |
| file_index=$((file_index+1)) | |
| if [[ $file == *1008* ]]; then | |
| echo -n 'skipping ' | |
| echo $file | |
| continue | |
| fi | |
| if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then | |
| cp ${file} $TARGET_DIR/ | |
| fi | |
| done | |
| # Do the execution | |
| - name: Begin Execution | |
| #$(docker image ls | grep "<none>" | awk '{print $3;}') to get image id | |
| run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs | |
| - name: Check For Success | |
| run: | | |
| errorSearch=$(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) | |
| if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -eq 0 ] | |
| then | |
| exit 0 | |
| else | |
| # Copy error log to github workspace | |
| cp $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) ${{ github.workspace }}/errors.report | |
| fi | |
| - name: Upload failures | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: set_${{ matrix.sets }}.report | |
| path: ${{ github.workspace }}/errors.report | |
| # Report a Problem to Slack | |
| - name: Report Problem to Slack | |
| if: ${{ failure() }} | |
| run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Oops! The *BMDB Step* of Nightly **Execution** (set ${{ matrix.sets }}) Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ). Go see what happened!" https://slack.com/api/chat.postMessage | |
| - name: Setup tmate session 3 | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ failure() }} | |
| published_results: | |
| runs-on: ubuntu-20.04 | |
| needs: published | |
| steps: | |
| - name: Gather Artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| path: ${{ github.workspace }}/published-results | |
| - name: Combine .report artifacts into one file | |
| run: cat $(find $GITHUB_WORKSPACE/published-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt | |
| - name: Post results to slack part 1 | |
| run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) published models that didn't pass:" https://slack.com/api/chat.postMessage | |
| - name: Post results to slack part 2 | |
| run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(cat ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage | |
| bmdb_results: | |
| runs-on: ubuntu-20.04 | |
| needs: bmdb | |
| steps: | |
| - name: Gather Artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| path: ${{ github.workspace }}/bmdb-results | |
| - name: Combine .report artifacts into one file | |
| run: cat $(find $GITHUB_WORKSPACE/bmdb-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt | |
| - name: Post results to slack part 1 | |
| run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) BMDB models that didn't pass:" https://slack.com/api/chat.postMessage | |
| - name: Post results to slack part 2 | |
| run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(cat ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage |