NOREF-Better error handling & logging (#365) #500

	name: ETL Pipeline Tests (Regression)

	on:
	push:
	branches: [ main ]
	schedule:
	- cron: '0 0 * * 1'

	jobs:
	test:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2
	- name: Set up Python 3.9
	uses: actions/setup-python@v2
	with:
	python-version: '3.9'
	- name: Cache dependencies
	uses: actions/cache@v2
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	${{ runner.os }}-
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip wheel
	pip install -r dev-requirements.txt
	pip install -r requirements.txt
	- name: Run test suite with Allure
	run: \|
	make allure-test
	- name: Get Allure history
	uses: actions/checkout@v2
	if: always()
	continue-on-error: true
	with:
	ref: gh-pages
	path: gh-pages
	- name: Generate allure report
	uses: simple-elf/allure-report-action@master
	if: always()
	id: allure-report
	with:
	allure_results: allure-results
	gh_pages: gh-pages
	allure_report: allure-report
	allure_history: allure-history
	- name: Deploy report to Github Pages
	if: always()
	uses: peaceiris/actions-gh-pages@v2
	env:
	PERSONAL_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	PUBLISH_BRANCH: gh-pages
	PUBLISH_DIR: allure-history
	- name: DRB Pipeline Slack Notification
	uses: rtCamp/action-slack-notify@v2
	env:
	SLACK_CHANNEL: test_reports
	#SLACK_MESSAGE: 'https://nypl.github.io/drb-etl-pipeline'
	SLACK_TITLE: DRB Pipeline Test Results
	SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
	SLACK_USERNAME: nyplorgBot

Provide feedback