Merge pull request #1302 from ESMValGroup/conda_build2

Various updates to dependencies and CircleCI configuration
ESMValGroup · Sep 6, 2019 · 6a8b50f · 6a8b50f
2 parents 2c66c08 + b4ccfb0
commit 6a8b50f
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 53 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -5,7 +5,7 @@ jobs:
     # Run Python 3 tests
     working_directory: /test
     docker:
-      - image: continuumio/anaconda3
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
@@ -19,28 +19,31 @@ jobs:
           # Update/Create Conda environment and run tests
           command: |
             . /opt/conda/etc/profile.d/conda.sh
-            conda update -y conda
-            conda env update
+            mkdir /logs
+            conda update -y conda >> /logs/conda.txt 2>&1
+            conda env update >> /logs/conda.txt 2>&1
             conda activate esmvaltool
-            # Install missing r-yaml and r-lintr
-            Rscript esmvaltool/install/R/setup_devutils.R
+            # Install r-lintr for r linter test
+            Rscript esmvaltool/install/R/setup_devutils.R > /logs/R_install.txt 2>&1
             # Run tests
             python setup.py test
       - save_cache:
           key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
           paths:
             - "/opt/conda/envs/esmvaltool"
             - ".eggs"
-      - store_test_results:
-          path: test-reports/
       - store_artifacts:
+          path: /logs
+      - store_artifacts:
+          path: test-reports/
+      - store_test_results:
           path: test-reports/
 
   test_installation:
     # Test Python 3 installation
     working_directory: /test_installation
     docker:
-      - image: continuumio/anaconda3
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - restore_cache:
@@ -64,8 +67,6 @@ jobs:
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml
             pip freeze > /logs/requirements.txt
-            # Install missing r-yaml
-            conda install -c conda-forge -yS r-yaml
             # Test installation
             python setup.py test --installation
             esmvaltool -h
@@ -89,14 +90,14 @@ jobs:
       - run:
           when: always
           command: |
-            /opt/conda/envs/esmvaltool/bin/pip install codacy-coverage
-            /opt/conda/envs/esmvaltool/bin/python-codacy-coverage -r test-reports/coverage.xml
+            pip install codacy-coverage
+            python-codacy-coverage -r test-reports/coverage.xml
 
   develop:
     # Test development installation
     working_directory: /develop
     docker:
-      - image: continuumio/anaconda3
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
@@ -130,7 +131,7 @@ jobs:
     # Test building documentation
     working_directory: /doc
     docker:
-      - image: continuumio/anaconda3
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - restore_cache:
@@ -139,15 +140,13 @@ jobs:
           command: |
             mkdir /logs
             # Install
-            . /opt/conda/etc/profile.d/conda.sh
-            conda install -c conda-forge -y pip > /logs/install.txt 2>&1
-            /opt/conda/bin/pip install -r doc/sphinx/source/requirements.txt > /logs/install.txt 2>&1
-            /opt/conda/bin/pip install sphinx sphinx_rtd_theme >> /logs/install.txt 2>&1
+            pip install -r doc/sphinx/source/requirements.txt > /logs/install.txt 2>&1
+            pip install sphinx sphinx_rtd_theme >> /logs/install.txt 2>&1
             # Log versions
             dpkg -l > /logs/versions.txt
-            /opt/conda/bin/pip freeze > /logs/requirements.txt
+            pip freeze > /logs/requirements.txt
             # Test building documentation
-            MPLBACKEND=Agg /opt/conda/bin/python setup.py build_sphinx --warning-is-error
+            MPLBACKEND=Agg python setup.py build_sphinx --warning-is-error
       - save_cache:
           key: documentation-{{ .Branch }}
           paths:
@@ -159,7 +158,7 @@ jobs:
     # Test conda build
     working_directory: /esmvaltool
     docker:
-      - image: continuumio/anaconda3
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
@@ -177,21 +176,16 @@ jobs:
             dpkg -l > /logs/versions.txt
             conda env export -n base > /logs/build_environment.yml
             # Build conda package
-            conda build . -c conda-forge -c esmvalgroup > /logs/build_log.txt
-            # Install Python 3 conda package
-            conda create -y --name esmvaltool3 > /logs/conda_esmvaltool3.txt 2>&1
-            set +x; conda activate esmvaltool3; set -x
-            conda install -y esmvaltool --use-local -c conda-forge -c esmvalgroup
-            conda env export > /logs/test_environment3.yml
-            esmvaltool -h
+            apt update && apt install time
+            \time -v conda build . -c conda-forge -c esmvalgroup > /logs/build_log.txt
       - store_artifacts:
           path: /logs
 
   conda_install:
     # Test conda package installation
     working_directory: /esmvaltool
     docker:
-      - image: continuumio/anaconda3
+      - image: continuumio/miniconda3
     steps:
       - run:
           command: |
@@ -219,7 +213,7 @@ jobs:
     # Test ncl and cdo conda packages
     working_directory: /ncl
     docker:
-      - image: continuumio/anaconda3
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
@@ -245,9 +239,11 @@ workflows:
   version: 2
   commit:
     jobs:
-      - test
-      - test_installation
       - documentation
+      - test
+      - test_installation:
+          requires:
+            - test
   nightly:
     triggers:
       - schedule:

diff --git a/conda_build_config.yaml b/conda_build_config.yaml
diff --git a/doc/sphinx/source/requirements.txt b/doc/sphinx/source/requirements.txt
@@ -9,7 +9,6 @@ numpy
 pillow
 prov[dot]
 pyyaml
-setuptools
 shapely
 six
 xarray

diff --git a/environment.yml b/environment.yml
@@ -6,12 +6,7 @@ channels:
 
 dependencies:
   # Python packages that cannot be installed from PyPI:
-  - esmpy
-  - esmvalcore>=2.0.0b0,<2.1
-  - iris>=2.2
-  - matplotlib<3  # Can be installed from PyPI, but is a dependency of iris and should be pinned.
-  - python-stratify
-  - xarray>=0.12.0  # Can be installed from PyPI, but here to get a consistent set of depencies with iris.
+  - esmvalcore>=2.0.0b1,<2.1
   # Non-Python dependencies
   - graphviz
   - cdo
@@ -22,7 +17,7 @@ dependencies:
   - python>=3.6
   - libunwind  # Needed for Python3.7+
   - tiledb=1.6.0  # Needed by the new ncl=6.6.2; evolved to 1.6.2 but ncl 6.6.2 still needs libtiledb1.6.0.so
-  - ncl
+  - ncl>=6.5.0
   - r-base
   - r-curl  # Dependency of lintr, but fails to compile because it cannot find libcurl installed from conda.
   - r-udunits2  # Fails to compile because it cannot find udunits2 installed from conda.

diff --git a/meta.yaml b/meta.yaml
@@ -22,6 +22,7 @@ build:
   # Increment the build number when building a new conda package of the same
   # esmvaltool version, reset to 0 when building a new version.
   number: 0
+  noarch: python
   script: |
       python setup.py install --single-version-externally-managed --record=/tmp/record.txt
       POST_LINK="${PREFIX}/bin/.esmvaltool-post-link.sh"
@@ -31,25 +32,25 @@ build:
 requirements:
   build:
     - git
-    - python {{ python }}
+    - python>=3.6
     # Normally installed via pip:
     - pytest-runner
     - setuptools_scm
   run:
     # esmvaltool
-    - python
+    - python>=3.6
     - libunwind  #  specifically for Python3.7+
     - graphviz
-    - iris>=2.2
+    - iris
     # Normally installed via pip:
     - cartopy
     - cf-units
     - cython
     - eofs
     - esmpy
-    - esmvalcore>=2.0.0a1,<2.1
+    - esmvalcore>=2.0.0b1,<2.1
     - jinja2
-    - matplotlib<3
+    - matplotlib
     - nc-time-axis
     - netCDF4
     - numpy
@@ -60,7 +61,7 @@ requirements:
     - scikit-learn
     - shapely
     - xarray>=0.12.0
-    - yamale  # in birdhouse channel
+    - yamale  # in esmvalgroup channel
     - fiona
     - xlsxwriter
     # Command line tools used by diagnostic scripts