Fix test and prepare scripts

.github/workflows/tests.yml

@@ -57,7 +57,9 @@ jobs:
         pip freeze
 
     - name: Run tests
-      run: bash ./scripts/tests.sh
+      run: |
+        source ./scripts/config.sh
+        pytest tests
 
   all_done:
     name: Tests done

scripts/README.md

@@ -1,11 +1,12 @@
 # Scripts
 
-- activate.sh (call create_test_virtualenv.sh and install.sh) - activate virtual environment
+- activate.sh (call config.sh, create_test_virtualenv.sh and install.sh) - activate virtual environment
 - airflow.sh (call activate.sh) - prepare the virtual environment if required and start airflow
+- config.sh - set environment variables
 - coverage.sh (call activate.sh) - run tests coverage
 - create_test_virtualenv.sh (call prepare.sh) - create the virtual environment
 - destroy_test_virtualenv.sh - destroy the virtual environment
 - install.sh (called by github workflows)
-- tests.sh (call activate.sh) - run tests 
-- prepare.sh (called by github workflows, call mo.sh) - prepare virtual environment
+- tests.sh (call activate.sh) - run tests
+- prepare.sh (called by github workflows, call config.sh) - prepare virtual environment
 - mo.sh - mustache template rendering library

scripts/activate.sh

@@ -1,14 +1,14 @@
 #!/usr/bin/env bash
 export CURRENT_DIR=`realpath $(dirname "$0")`
 export PARENT_DIR=`dirname "$CURRENT_DIR"`
+source "${CURRENT_DIR}/config.sh"
 
 if [[ ! -a "${CURRENT_DIR}/venv/bin/activate" ]]; then
     bash "${CURRENT_DIR}/create_test_virtualenv.sh"
 fi
 
 source "${CURRENT_DIR}/venv/bin/activate"
 bash "${CURRENT_DIR}/install.sh"
-source "${CURRENT_DIR}/config.sh"
 
 if ! compgen -G "${PARENT_DIR}/dist/*.whl" > /dev/null; then
     export PYTHONPATH="${PARENT_DIR}:${PYTHONPATH}"

scripts/airflow.sh

@@ -2,7 +2,6 @@
 set -e
 
 export CURRENT_DIR=`realpath $(dirname "$0")`
-
 source "${CURRENT_DIR}/activate.sh"
-source "${CURRENT_DIR}/config.sh"
+
 airflow "$@"

scripts/config.sh

@@ -1,10 +1,18 @@
+# Airflow home
+if [ -n "${AIRFLOW_HOME}" ]; then
+    export AIRFLOW_HOME=$(realpath "${AIRFLOW_HOME}")
+else
+    export AIRFLOW_HOME="$(realpath $(dirname "$0"))/venv"
+fi
+
 # Airflow config
 export AIRFLOW__CORE__DAGS_FOLDER="${AIRFLOW_HOME}/dags"
 export AIRFLOW__CORE__PLUGINS_FOLDER="${AIRFLOW_HOME}/plugins"
 export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="sqlite:///${AIRFLOW_HOME}/airflow.db"
 export AIRFLOW__LOGGING__BASE_LOG_FOLDER="${AIRFLOW_HOME}/logs"
 export AIRFLOW__LOGGING__DAG_PROCESSOR_MANAGER_LOG_LOCATION="${AIRFLOW_HOME}/logs/dag_processor_manager/dag_processor_manager.log"
 export AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY="${AIRFLOW_HOME}/logs"
+
 # Code Editor config
 export AIRFLOW__CODE_EDITOR__ENABLED="True"
 export AIRFLOW__CODE_EDITOR__MOUNT_NAME="airflow_home"

scripts/coverage.sh

@@ -2,7 +2,6 @@
 set -e
 
 export CURRENT_DIR=`realpath $(dirname "$0")`
-
 source "${CURRENT_DIR}/activate.sh"
 
 cd `dirname "$CURRENT_DIR"`

scripts/prepare.sh

@@ -3,38 +3,20 @@ set -e
 
 export CURRENT_DIR=`realpath $(dirname "$0")`
 export PARENT_DIR=`dirname "$CURRENT_DIR"`
+source "${CURRENT_DIR}/config.sh"
 
-export AIRFLOW_HOME="${AIRFLOW_HOME:-${CURRENT_DIR}/venv}"
-
-source "${CURRENT_DIR}/mo.sh"
-
-export ENABLE_AIRFLOW_AUTH="${ENABLE_AIRFLOW_AUTH:-1}"
-# export AIRFLOW_VERSION=1.10.12
 export AIRFLOW_VERSION="${AIRFLOW_VERSION:-2.3.1}"
-# export FORCE_PYTHON_VERSION=3.7
 export CONSTRAINTS_VERSION="${CONSTRAINTS_VERSION:-${AIRFLOW_VERSION}}"
-
 export AIRFLOW_MAJOR_VERSION=${AIRFLOW_VERSION:0:1}
-if [[ "${AIRFLOW_MAJOR_VERSION}" == "2" ]]; then
-    export EXTRAS="password"
-else
-    export EXTRAS="crypto,password"
-fi
+export AIRFLOW_EXTRAS="password"
+export PYTHON_VERSION=$(python3 -c "import sys; print('%s.%s' % (sys.version_info.major, sys.version_info.minor))")
 
-function WEBSERVER_AUTH() {
-    if [[ "${ENABLE_AIRFLOW_AUTH}" == "1" ]]; then
-        echo "authenticate = True"
-        echo "rbac = True"
-        echo "auth_backend = airflow.contrib.auth.backends.password_auth"
-    else
-        echo "authenticate = False"
-        echo "rbac = False"
-    fi
-}
+echo "--------------------------------------------------------------------------------"
+env
+echo "--------------------------------------------------------------------------------"
 
-export PYTHON_VERSION=$(python3 -c "import sys; print('%s.%s' % (sys.version_info.major, sys.version_info.minor))")
 pip install \
-      "apache-airflow[${EXTRAS}]==${AIRFLOW_VERSION}" \
+      "apache-airflow[${AIRFLOW_EXTRAS}]==${AIRFLOW_VERSION}" \
       --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${CONSTRAINTS_VERSION}/constraints-${PYTHON_VERSION}.txt" \
       -r "${PARENT_DIR}/requirements.txt" \
       -r "${PARENT_DIR}/requirements-dev.txt"
@@ -43,23 +25,9 @@ if [ ! -d "${AIRFLOW_HOME}" ]; then
   mkdir -p "${AIRFLOW_HOME}"
 fi
 
-if [[ "${AIRFLOW_MAJOR_VERSION}" == "1" ]]; then
-    # mo -u < "${CURRENT_DIR}/airflow1.cfg.tmpl" > "${AIRFLOW_HOME}/airflow.cfg"
-    airflow initdb
-else
-    # mo -u < "${CURRENT_DIR}/airflow2.cfg.tmpl" > "${AIRFLOW_HOME}/airflow.cfg"
-    airflow db init
-fi
-
-source "${CURRENT_DIR}/config.sh"
-
-if [[ "${ENABLE_AIRFLOW_AUTH}" == "1" ]]; then
-    # Create user 'admin' with password 'admin'
-    if [[ "${AIRFLOW_MAJOR_VERSION}" == "1" ]]; then
-        airflow create_user -r Admin -u admin -e [email protected] -f admin -l admin -p admin
-    else
-        airflow users create -r Admin -u admin -e [email protected] -f admin -l admin -p admin
-    fi
-fi
+# Init airflow db
+airflow db init
+# Create user 'admin' with password 'admin'
+airflow users create -r Admin -u admin -e [email protected] -f admin -l admin -p admin
 
 ln -sf "${CURRENT_DIR}/dags" "${AIRFLOW_HOME}/dags"

scripts/tests.sh

@@ -2,7 +2,6 @@
 set -e
 
 export CURRENT_DIR=`realpath $(dirname "$0")`
-
 source "${CURRENT_DIR}/activate.sh"
-source "${CURRENT_DIR}/config.sh"
+
 pytest tests