-
Notifications
You must be signed in to change notification settings - Fork 50
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
8 changed files
with
29 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,11 +1,12 @@ | ||
# Scripts | ||
|
||
- activate.sh (call create_test_virtualenv.sh and install.sh) - activate virtual environment | ||
- activate.sh (call config.sh, create_test_virtualenv.sh and install.sh) - activate virtual environment | ||
- airflow.sh (call activate.sh) - prepare the virtual environment if required and start airflow | ||
- config.sh - set environment variables | ||
- coverage.sh (call activate.sh) - run tests coverage | ||
- create_test_virtualenv.sh (call prepare.sh) - create the virtual environment | ||
- destroy_test_virtualenv.sh - destroy the virtual environment | ||
- install.sh (called by github workflows) | ||
- tests.sh (call activate.sh) - run tests | ||
- prepare.sh (called by github workflows, call mo.sh) - prepare virtual environment | ||
- tests.sh (call activate.sh) - run tests | ||
- prepare.sh (called by github workflows, call config.sh) - prepare virtual environment | ||
- mo.sh - mustache template rendering library |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,38 +3,20 @@ set -e | |
|
||
export CURRENT_DIR=`realpath $(dirname "$0")` | ||
export PARENT_DIR=`dirname "$CURRENT_DIR"` | ||
source "${CURRENT_DIR}/config.sh" | ||
|
||
export AIRFLOW_HOME="${AIRFLOW_HOME:-${CURRENT_DIR}/venv}" | ||
|
||
source "${CURRENT_DIR}/mo.sh" | ||
|
||
export ENABLE_AIRFLOW_AUTH="${ENABLE_AIRFLOW_AUTH:-1}" | ||
# export AIRFLOW_VERSION=1.10.12 | ||
export AIRFLOW_VERSION="${AIRFLOW_VERSION:-2.3.1}" | ||
# export FORCE_PYTHON_VERSION=3.7 | ||
export CONSTRAINTS_VERSION="${CONSTRAINTS_VERSION:-${AIRFLOW_VERSION}}" | ||
|
||
export AIRFLOW_MAJOR_VERSION=${AIRFLOW_VERSION:0:1} | ||
if [[ "${AIRFLOW_MAJOR_VERSION}" == "2" ]]; then | ||
export EXTRAS="password" | ||
else | ||
export EXTRAS="crypto,password" | ||
fi | ||
export AIRFLOW_EXTRAS="password" | ||
export PYTHON_VERSION=$(python3 -c "import sys; print('%s.%s' % (sys.version_info.major, sys.version_info.minor))") | ||
|
||
function WEBSERVER_AUTH() { | ||
if [[ "${ENABLE_AIRFLOW_AUTH}" == "1" ]]; then | ||
echo "authenticate = True" | ||
echo "rbac = True" | ||
echo "auth_backend = airflow.contrib.auth.backends.password_auth" | ||
else | ||
echo "authenticate = False" | ||
echo "rbac = False" | ||
fi | ||
} | ||
echo "--------------------------------------------------------------------------------" | ||
env | ||
echo "--------------------------------------------------------------------------------" | ||
|
||
export PYTHON_VERSION=$(python3 -c "import sys; print('%s.%s' % (sys.version_info.major, sys.version_info.minor))") | ||
pip install \ | ||
"apache-airflow[${EXTRAS}]==${AIRFLOW_VERSION}" \ | ||
"apache-airflow[${AIRFLOW_EXTRAS}]==${AIRFLOW_VERSION}" \ | ||
--constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${CONSTRAINTS_VERSION}/constraints-${PYTHON_VERSION}.txt" \ | ||
-r "${PARENT_DIR}/requirements.txt" \ | ||
-r "${PARENT_DIR}/requirements-dev.txt" | ||
|
@@ -43,23 +25,9 @@ if [ ! -d "${AIRFLOW_HOME}" ]; then | |
mkdir -p "${AIRFLOW_HOME}" | ||
fi | ||
|
||
if [[ "${AIRFLOW_MAJOR_VERSION}" == "1" ]]; then | ||
# mo -u < "${CURRENT_DIR}/airflow1.cfg.tmpl" > "${AIRFLOW_HOME}/airflow.cfg" | ||
airflow initdb | ||
else | ||
# mo -u < "${CURRENT_DIR}/airflow2.cfg.tmpl" > "${AIRFLOW_HOME}/airflow.cfg" | ||
airflow db init | ||
fi | ||
|
||
source "${CURRENT_DIR}/config.sh" | ||
|
||
if [[ "${ENABLE_AIRFLOW_AUTH}" == "1" ]]; then | ||
# Create user 'admin' with password 'admin' | ||
if [[ "${AIRFLOW_MAJOR_VERSION}" == "1" ]]; then | ||
airflow create_user -r Admin -u admin -e [email protected] -f admin -l admin -p admin | ||
else | ||
airflow users create -r Admin -u admin -e [email protected] -f admin -l admin -p admin | ||
fi | ||
fi | ||
# Init airflow db | ||
airflow db init | ||
# Create user 'admin' with password 'admin' | ||
airflow users create -r Admin -u admin -e [email protected] -f admin -l admin -p admin | ||
|
||
ln -sf "${CURRENT_DIR}/dags" "${AIRFLOW_HOME}/dags" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters