real-estate-scrape.yml

name: real-estate-scrape

on:
  workflow_dispatch:
  schedule:
    - cron:  '00 05 * * *'

jobs:
  real-estate-scrape:
    runs-on: ubuntu-latest
    steps:
    - name: Check out repo
      uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'
    - name: Install real-estate-scrape
      run: |
        pip install real-estate-scrape
    - name: Run scraper
      env:
        REDFIN_URL: ${{ secrets.REDFIN_URL }}
        ZILLOW_URL: ${{ secrets.ZILLOW_URL }}
        SCRAPERAPI_KEY: ${{ secrets.SCRAPERAPI_KEY }}
      run: real-estate-scrape
    - name: Commit and push if it changed
      run: |-
        git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
        git config user.name "github-actions[bot]"
        git add -A
        timestamp=$(date -u)
        git commit -m "Scraped at ${timestamp}" || exit 0
        git push
    - name: Set workflow status
      run: |
        # Each NaN in data.csv corresponds to a failure
        failures=$(tail -1 data.csv | grep -o NaN | wc -l)
        echo "Number of failures:" $failures
        # We don't pay for scraperapi so Zillow failure is expected. We only
        # want the workflow to fail if both fail. Change 2 to 1 if you want want
        # to fail if either Zillow or Redfin scraping fails.
        (( $failures < 2 ))