now running travis on all DB backends

- added Travis config to also run integration tests on sqlite
- added Travis config to also run integration test on postgre, and commented them out as they seem to fail
- added a bit more logs in bash scripts executed by Travis

.travis.yml

@@ -16,20 +16,44 @@ env:
   global:
     - TRAVIS_CACHE=$HOME/.travis_cache/
   matrix:
-    - TOX_ENV=py27-cdh
-    - TOX_ENV=py27-hdp
-    - TOX_ENV=py34-cdh
-    - TOX_ENV=py34-hdp
+    - TOX_ENV=py27-cdh-airflow_backend_mysql
+    - TOX_ENV=py27-cdh-airflow_backend_sqlite
+#    - TOX_ENV=py27-cdh-airflow_backend_postgres
+    - TOX_ENV=py27-hdp-airflow_backend_mysql
+    - TOX_ENV=py27-hdp-airflow_backend_sqlite
+#    - TOX_ENV=py27-hdp-airflow_backend_postgres
+    - TOX_ENV=py34-cdh-airflow_backend_mysql
+    - TOX_ENV=py34-cdh-airflow_backend_sqlite
+#    - TOX_ENV=py34-cdh-airflow_backend_postgres
+    - TOX_ENV=py34-hdp-airflow_backend_mysql
+    - TOX_ENV=py34-hdp-airflow_backend_sqlite
+#    - TOX_ENV=py34-hdp-airflow_backend_postgres
 matrix:
   exclude:
-    - python: "2.7"
-      env: TOX_ENV=py34-cdh
-    - python: "2.7"
-      env: TOX_ENV=py34-hdp
     - python: "3.4"
-      env: TOX_ENV=py27-cdh
+      env: TOX_ENV=py27-cdh-airflow_backend_mysql
+    - python: "3.4"
+      env: TOX_ENV=py27-cdh-airflow_backend_sqlite
+#    - python: "3.4"
+#      env: TOX_ENV=py27-cdh-airflow_backend_postgres
+    - python: "3.4"
+      env: TOX_ENV=py27-hdp-airflow_backend_mysql
     - python: "3.4"
-      env: TOX_ENV=py27-hdp
+      env: TOX_ENV=py27-hdp-airflow_backend_sqlite
+#    - python: "3.4"
+#      env: TOX_ENV=py27-hdp-airflow_backend_postgres
+    - python: "2.7"
+      env: TOX_ENV=py34-cdh-airflow_backend_mysql
+    - python: "2.7"
+      env: TOX_ENV=py34-cdh-airflow_backend_sqlite
+#    - python: "2.7"
+#      env: TOX_ENV=py34-cdh-airflow_backend_postgres
+    - python: "2.7"
+      env: TOX_ENV=py34-hdp-airflow_backend_mysql
+    - python: "2.7"
+      env: TOX_ENV=py34-hdp-airflow_backend_sqlite
+#    - python: "2.7"
+#      env: TOX_ENV=py34-hdp-airflow_backend_postgres
 cache:
   directories:
     - $HOME/.wheelhouse/

run_tox.sh

@@ -1 +1,3 @@
+set -o verbose
+
 python setup.py test --tox-args="-v -e $TOX_ENV"

scripts/ci/ldap.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -o verbose
 
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 LDAP_DB=/tmp/ldap_db
@@ -13,4 +14,4 @@ echo "Launching OpenLDAP ..."
 slapd -h "ldap://127.0.0.1:3890/" -f ${DIR}/slapd.conf
 
 # Wait for LDAP to start
-sleep 1
+sleep 1

scripts/ci/load_fixtures.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -o verbose
 
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 FIXTURES_DIR="$DIR/ldif"
@@ -10,4 +11,4 @@ load_fixture () {
 for FIXTURE in `ls ${FIXTURES_DIR}`
 do
   load_fixture "${FIXTURES_DIR}/${FIXTURE}"
-done;
+done;

scripts/ci/run_tests.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -o verbose
 
 if [ -z "$HADOOP_HOME" ]; then
     echo "HADOOP_HOME not set - abort" >&2
@@ -10,5 +11,6 @@ echo "Using ${HADOOP_DISTRO} distribution of Hadoop from ${HADOOP_HOME}"
 pwd
 
 mkdir ~/airflow/
-cp ${TRAVIS_BUILD_DIR}/scripts/ci/airflow_travis.cfg ~/airflow/unittests.cfg
+echo creating a unittest config with sql_alchemy_conn $BACKEND_SQL_ALCHEMY_CONN
+sed -e "s#sql_alchemy_conn.*#sql_alchemy_conn\ =\ $BACKEND_SQL_ALCHEMY_CONN#g" ${TRAVIS_BUILD_DIR}/scripts/ci/airflow_travis.cfg > ~/airflow/unittests.cfg
 ./run_unit_tests.sh

scripts/ci/setup_env.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+set -o verbose
+
 MINIKDC_VERSION=2.7.1
 
 HADOOP_DISTRO=${HADOOP_DISTRO:-"hdp"}

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = {py27,py34}-{cdh,hdp}
+envlist = {py27,py34}-{cdh,hdp}-airflow_backend_{mysql,sqlite,postgres}
 skipsdist=True
 
 [global]
@@ -25,6 +25,9 @@ setenv =
   hdp: HADOOP_HOME=/tmp/hadoop-hdp
   hdp: KRB5_CONFIG=/tmp/minikdc/work
   hdp: HADOOP_OPTS=-D/tmp/minikdc/work/krb5.conf
+  airflow_backend_mysql: BACKEND_SQL_ALCHEMY_CONN=mysql://root@localhost/airflow
+  airflow_backend_sqlite: BACKEND_SQL_ALCHEMY_CONN=sqlite:///{homedir}/airflow/airflow.db
+  airflow_backend_postgres: BACKEND_SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres@localhost/airflow
 passenv =
     HOME
     JAVA_HOME
@@ -42,4 +45,5 @@ commands =
   {toxinidir}/scripts/ci/ldap.sh
   {toxinidir}/scripts/ci/load_fixtures.sh
   {toxinidir}/scripts/ci/run_tests.sh []
+
   coveralls