ci.yml

name: CI

on: [push, pull_request]

env:
  CI: true
  PIPENV_VENV_IN_PROJECT: true
  AUTOTHROTTLE_MAX_DELAY: 30.0
  AUTOTHROTTLE_START_DELAY: 1.5
  AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0

jobs:
  check:
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        python-version: [3.9]

    steps:
      - uses: actions/checkout@v1

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Pipenv
        uses: dschep/install-pipenv-action@v1

      - name: Install dependencies
        run: pipenv sync --dev
        env:
          PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}

      - name: Check imports with isort
        run: pipenv run isort . --check-only

      - name: Lint with flake8
        run: pipenv run flake8 .

      - name: Test with pytest
        run: pipenv run pytest

      - name: Validate output with scrapy
        if: github.event_name == 'pull_request'
        run: |
          git checkout ${{ github.base_ref }}
          git checkout $(git show-ref | grep pull | awk '{ print $2 }')
          git diff-index --name-only --diff-filter=d $(git merge-base HEAD ${{ github.base_ref }}) | \
          grep -Pio '(?<=/spiders/).*(?=\.py)' | \
          xargs pipenv run scrapy validate