Merge pull request #427 from ESMValGroup/version2_development

Version 2: Merge development branch

.circleci/config.yml

@@ -287,7 +287,7 @@ workflows:
             branches:
               only:
                 - version2_development
-		- version2_master
+                - version2_master
     jobs:
       - python3_test
       - python2_test

NOTICE

@@ -1,6 +1,6 @@
 
 
-Earth System Model Evaluation Tool VERSION 1.1.0 (the Software) License
+Earth System Model Evaluation Tool VERSION 2 (the Software) License
 
 
 PLEASE READ THIS SOFTWARE LICENSE ("LICENSE") CAREFULLY. USE
@@ -14,7 +14,7 @@ DATE: 26 January 2017
 ========================================================================
 
 
-THIS LICENSE APPLIES TO THE ESMValTool Version 1.1.0 VERSION THAT IS RELEASED AS
+THIS LICENSE APPLIES TO THE ESMValTool Version 2 VERSION THAT IS RELEASED AS
 OPEN-SOURCE SOFTWARE UNDER the Apache License, Version 2.0
 
 ========================================================================

README.md

@@ -1,5 +1,5 @@
 # ESMValTool
-[![DOIBadge](https://img.shields.io/badge/DOI-10.17874%2Fac8548f0315-blue.svg)](http://dx.doi.org/10.17874/ac8548f0315)
+[![DOIBadge](https://img.shields.io/badge/DOI-10.17874%2Fac8548f0315-blue.svg)](https://doi.org/10.17874/ac8548f0315)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/79bf6932c2e844eea15d0fb1ed7e415c)](https://www.codacy.com/app/ESMValGroup/ESMValTool?utm_source=github.com&utm_medium=referral&utm_content=ESMValGroup/ESMValTool&utm_campaign=Badge_Grade)
 [![Codacy Coverage Badge](https://api.codacy.com/project/badge/Coverage/79bf6932c2e844eea15d0fb1ed7e415c)](https://www.codacy.com/app/ESMValGroup/ESMValTool?utm_source=github.com&utm_medium=referral&utm_content=ESMValGroup/ESMValTool&utm_campaign=Badge_Coverage)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ESMValGroup?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)

doc/RELEASE_NOTES → RELEASE_NOTES

@@ -2,6 +2,12 @@
 ####### ESMValTool - Earth System Model Evaluation Tool - RELEASE NOTES #######
 ###############################################################################
 
+===============================================================================
+Version 2.0aX - June 2018
+===============================================================================
+Completely revised version of ESMValTool. This are alpha releases, intended
+to be used solely as a preview of the upcoming version 2 release.
+
 ===============================================================================
 Version 1.1.0 - January 2017
 ===============================================================================

doc/MASTER_authors-refs-acknow.txt

@@ -127,13 +127,13 @@ A_tilm_si = Tilmes, Simone (NCAR, US - tilmes 'at' ucar.edu)
 D_0000 = Please acknowledge the author(s).
 D_0001 = Please contact the author(s) to discuss acknowledgment or co-authorship.
 D_0002 = Please acknowledge the project(s).
-D_anav13jclim = Anav et al., J. Clim., 26, 6801-6843, doi: http://dx.doi.org/10.1175/JCLI-D-12-00417.1, 2013.
+D_anav13jclim = Anav et al., J. Clim., 26, 6801-6843, doi: https://doi.org/10.1175/JCLI-D-12-00417.1, 2013.
 D_antonov10usgov = Antonov, J. I. et al., World Ocean Atlas 2009, Volume 2: Salinity. S. Levitus, Ed. NOAA Atlas NESDIS 69, U.S. Government Printing Office, Washington, D.C., 184 pp. (2010)
 D_aquila11gmd = Aquila et al., Geosci. Model Dev. 4, 325-355, doi:10.5194/gmd-4-325-2011, 2011.
 D_bakker14essd, Bakker, D. C. E. et al., Earth Syst. Sci. Data, 6, 69-90, doi:10.5194/essd-6-69-2014, 2014.
 D_bianchi12gbc, Bianchi, D. et al., Global Biogeochem. Cy., 26, GB2009, doi:10.1029/2011GB004209, 2012.
 D_cionni11acp = Cionni et al., Atmos. Chem. Phys., 11, 11267-11292, doi:10.5194/acp-11-11267-2011, 2011.
-D_clivar09jclim = CLIVAR Madden-Julian Oscillation Working Group, J. Climate, 22, 3006–3030, doi: http://dx.doi.org/10.1175/2008JCLI2731.1 (2009)
+D_clivar09jclim = CLIVAR Madden-Julian Oscillation Working Group, J. Climate, 22, 3006–3030, doi: https://doi.org/10.1175/2008JCLI2731.1 (2009)
 D_dong08grl = Dong, S. et al., J. Geophys. Res., 113, C06013, doi:10.1029/2006JC004051 (2008)
 D_duemenil00mpimr: Duemenil Gates et al., Observed historical discharge data from major rivers for climate model validation. Max Planck Institute for Meteorology Report 307, Hamburg, Germany (2000)
 D_emmons00jgr = Emmons et al., J. Geophys. Res., 105, D16, 20497-20538, 2000.
@@ -152,7 +152,7 @@ D_key04gbc = Key, R. M. et al., Global Biogeochem. Cy., 18, GB4031, doi:10.109/2
 D_lauer05acp = Lauer et al., Atmos. Chem. Phys., 5, 3251-3276, doi:10.5194/acp-5-3251-2005, 2005.
 D_lauer13jclim = Lauer and Hamilton, J. Climate, 26, 3823-3845, doi:10.1175/JCLI-D-12-00451.1, 2013.
 D_lauer17rse = Lauer et al., Remote Sens. Environ., in press, 2017.
-D_li14jclim = Li and Xie, J. Climate, 27, 1765-1780, doi:http://dx.doi.org/10.1175/JCLI-D-13-00337.1, 2014.
+D_li14jclim = Li and Xie, J. Climate, 27, 1765-1780, doi:https://doi.org/10.1175/JCLI-D-13-00337.1, 2014.
 D_lin08jclim = Lin, J-L. et al., J. Climate, 21, 4541-4567, doi: 10.1175/2008JCLI1816.1, 2008.
 D_lloyd-hughes02jclim = Lloyd-Hughes, B. and Saunders, M. A., Int. J. Climatol., 22, 1571-1592, doi:10.1002/joc.846 (2002)
 D_locarini10usgov = Locarnini, R. A. et al., World Ocean Atlas 2009, Volume 1: Temperature. S. Levitus, Ed. NOAA Atlas NESDIS 68, U.S. Government Printing Office, Washington, D.C., 184 pp. (2010)

doc/sphinx/source/annex_c.rst

@@ -11,12 +11,12 @@ information on the observational data used, references, and example plots.
 For further information on the diagnostics and performance metrics of
 ESMValTool v1.0, see also
 
-* Eyring et al., ESMValTool (v1.0) – a community diagnostic and performance metrics tool for routine evaluation of Earth System Models in CMIP, Geosci.Model Dev., 9, 1747-1802, 2016. `doi:10.5194/gmd-9-1747-2016 <http://dx.doi.org/10.5194/gmd-9-1747-2016>`_ . 
+* Eyring et al., ESMValTool (v1.0) – a community diagnostic and performance metrics tool for routine evaluation of Earth System Models in CMIP, Geosci.Model Dev., 9, 1747-1802, 2016. `doi:10.5194/gmd-9-1747-2016 <https://doi.org/10.5194/gmd-9-1747-2016>`_ . 
 
 Diagnostics and observational data sets newly implemented into the ESMValTool
 v1.1 are described in
 
-* Lauer et al., Benchmarking CMIP5 models with a subset of ESA CCI Phase 2 data using the ESMValTool, Remote Sens. Environ. (2017), `doi:10.1016/j.rse.2017.01.007 <http://dx.doi.org/10.1016/j.rse.2017.01.007>`_
+* Lauer et al., Benchmarking CMIP5 models with a subset of ESA CCI Phase 2 data using the ESMValTool, Remote Sens. Environ. (2017), `doi:10.1016/j.rse.2017.01.007 <https://doi.org/10.1016/j.rse.2017.01.007>`_
 
 
 

doc/sphinx/source/developer_guide2/porting.rst

@@ -84,7 +84,7 @@ The new ESMValTool version includes a completely revised interface, handling the
 +-------------------------------------------------+-----------------------------------------------------+------------------+
 | ``info_output(arg1, arg2, arg3)``               | ``log_info(arg1)`` if ``arg3=1``                    | all .ncl scripts |
 +-------------------------------------------------+-----------------------------------------------------+------------------+
-| ``info_output(arg1, arg2, arg3)``               | ``debug_info(arg1)`` if ``arg3>1``                  | all .ncl scripts |
+| ``info_output(arg1, arg2, arg3)``               | ``log_debug(arg1)`` if ``arg3>1``                   | all .ncl scripts |
 +-------------------------------------------------+-----------------------------------------------------+------------------+
 | ``verbosity = config_user_info@verbosity``      | remove this statement                               | all .ncl scripts |
 +-------------------------------------------------+-----------------------------------------------------+------------------+
@@ -121,6 +121,8 @@ The new ESMValTool version includes a completely revised interface, handling the
 +-------------------------------------------------+-----------------------------------------------------+------------------+
 | ``load diag_scripts/lib/ncl/misc_function.ncl`` | ``load diag_scripts/shared/plot/misc_function.ncl`` | all .ncl scripts |
 +-------------------------------------------------+-----------------------------------------------------+------------------+
+| ``LW_CRE``, ``SW_CRE``                          | ``lwcre``, ``swcre``                                | some yml nmls    |
++-------------------------------------------------+-----------------------------------------------------+------------------+
 
 The following changes shall also be considered:
 

doc/sphinx/source/namelists/namelist_asian_monsoon.rst

@@ -176,7 +176,7 @@ References
 ----------
 
 * Sperber, K. R., et al., The Asian summer monsoon: an intercomparison of CMIP5 vs. CMIP3 simulations of the late 20th century, Clim Dyn (2013) 41:2711--2744, doi: 10.1007/s00382-012-1607-6, 2012.
-* Lin, Jia-Lin, Klaus M. Weickman, George N. Kiladis, Brian E. Mapes, Siegfried D. Schubert, Max J. Suarez, Julio T. Bacmeister, Myong-In Lee, 2008: Subseasonal Variability Associated with Asian Summer Monsoon Simulated by 14 IPCC AR4 Coupled GCMs. J. Climate, 21, 4541-4567. doi: http://dx.doi.org/10.1175/2008JCLI1816.1.
+* Lin, Jia-Lin, Klaus M. Weickman, George N. Kiladis, Brian E. Mapes, Siegfried D. Schubert, Max J. Suarez, Julio T. Bacmeister, Myong-In Lee, 2008: Subseasonal Variability Associated with Asian Summer Monsoon Simulated by 14 IPCC AR4 Coupled GCMs. J. Climate, 21, 4541-4567. doi: https://doi.org/10.1175/2008JCLI1816.1.
 * Webster, P. J., and S.Yang, 1992: Monsoon and ENSO: Selectively interactive systems. Quart. J. Roy. Meteor. Soc., 118, 877-926. (Webster-Yang dynamical monsoon index)
 * Goswami, B. N., B. Krishnamurthy, and H. Annamalai, 1999: A broad-scale circulation index for interannual variability of the Indian summer monsoon. Quart. J. Roy. Meteor. Soc., 125, 611-633. (Goswami dynamical monsoon index)
 * Wang, B., and Z. Fan, 1999: Choice of south Asian summer monsoon indices. Bull. Amer. Meteor. Soc., 80, 629-638. (Wang-Fan dynamical monsoon index)

doc/sphinx/source/namelists/namelist_esacci.rst

@@ -142,7 +142,7 @@ Observations and reformat scripts
 References
 ----------
 
-* Lauer, A., V. Eyring, M. Righi, M. Buchwitz, P. Defourny, M. Evaldsson, P. Friedlingstein, R. de Jeuf, G. de Leeuw, A. Loew, C. J. Merchant, B. Müller, T. Popp, M. Reuter, S. Sandven, D. Senftleben, M. Stengel, M. Van Roozendael, S. Wenzel, and U. Willén: Benchmarking CMIP5 models with a subset of ESA CCI Phase 2 data using the ESMValTool, Remote Sensing of Environment, http://dx.doi.org/10.1016/j.rse.2017.01.007, 2017.
+* Lauer, A., V. Eyring, M. Righi, M. Buchwitz, P. Defourny, M. Evaldsson, P. Friedlingstein, R. de Jeuf, G. de Leeuw, A. Loew, C. J. Merchant, B. Müller, T. Popp, M. Reuter, S. Sandven, D. Senftleben, M. Stengel, M. Van Roozendael, S. Wenzel, and U. Willén: Benchmarking CMIP5 models with a subset of ESA CCI Phase 2 data using the ESMValTool, Remote Sensing of Environment, https://doi.org/10.1016/j.rse.2017.01.007, 2017.
 
 
 Example plots

doc/sphinx/source/namelists/namelist_tropical_variability.rst

@@ -120,7 +120,7 @@ Observations and reformat scripts
 References
 ----------
 
-* Li, G. and S.-P. Xie (2014), Tropical Biases in CMIP5 Multimodel Ensemble: The Excessive Equatorial Pacific Cold Tongue and Double ITCZ Problems. J. Climate, 27, 1765-1780. doi: http://dx.doi.org/10.1175/JCLI-D-13-00337.1.
+* Li, G. and S.-P. Xie (2014), Tropical Biases in CMIP5 Multimodel Ensemble: The Excessive Equatorial Pacific Cold Tongue and Double ITCZ Problems. J. Climate, 27, 1765-1780. doi: https://doi.org/10.1175/JCLI-D-13-00337.1.
 
 
 Example plots

doc/sphinx/source/namelists/namelist_wam.rst

@@ -203,7 +203,7 @@ References
 ----------
 
 * Cook, K. H. and E. K. Vizy, 2006: Coupled model simulations of the West African monsoon system: Twentieth- and twenty-first-century simulations. J. Climate, 19, 3681-3703.
-* Roehrig, R., D. Bouniol, F. Guichard, F. Hourdin, and J.-L. Redelsperger, 2013: The Present and Future of the West African Monsoon: A Process-Oriented Assessment of CMIP5 Simulations along the AMMA Transect. J. Climate, 26, 6471-6505. doi: http://dx.doi.org/10.1175/JCLI-D-12-00505.1.
+* Roehrig, R., D. Bouniol, F. Guichard, F. Hourdin, and J.-L. Redelsperger, 2013: The Present and Future of the West African Monsoon: A Process-Oriented Assessment of CMIP5 Simulations along the AMMA Transect. J. Climate, 26, 6471-6505. doi: https://doi.org/10.1175/JCLI-D-12-00505.1.
 
 
 Example plots

doc/sphinx/source/references.rst

@@ -4,7 +4,7 @@ References
 **********
 
     * Flato, G., Marotzke, J., Abiodun, B., Braconnot, P., Chou, S. C., Collins, W., Cox, P., Driouech, F., Emori, S., Eyring, V., Forest, C., Gleckler, P., Guilyardi, E., Jakob, C., Kattsov, V., Reason, C., and Rummukainen, M.: Evaluation of Climate Models. In: Climate Change 2013: The Physical Science Basis. Contribution of Working Group I to the Fifth Assessment Report of the Intergovernmental Panel on Climate Change, Stocker, T. F., D. Qin, G.-K. Plattner, M. Tignor, S.K. Allen, J. Boschung, A. Nauels, Y. Xia, V. Bex and P.M. Midgley (Ed.), Cambridge University Press, Cambridge, United Kingdom and New York, NY, USA, 2013.
-    * NCL (2014) The NCAR Command Language (Version 6.2.1) [Software]. (2014). Boulder, Colorado: UCAR/NCAR/CISL/VETS. http://dx.doi.org/10.5065/D6WD3XH5
+    * NCL (2014) The NCAR Command Language (Version 6.2.1) [Software]. (2014). Boulder, Colorado: UCAR/NCAR/CISL/VETS. https://doi.org/10.5065/D6WD3XH5
     * PEP8 (2001) https://www.python.org/dev/peps/pep-0008/
     * Taylor, K. E., Stouffer, R. J., and Meehl, G. A.: An Overview of Cmip5 and the Experiment Design, B Am Meteorol Soc, 93, 485-498, 2012.
     * XML http://www.w3.org/TR/xml/www.gmail.com

environment.yml

@@ -6,12 +6,12 @@ channels:
 
 dependencies:
   # Python packages that cannot be installed from PyPI:
+  # TODO when switching to iris 2.x, switch straight to 2.1+
   - basemap
   - iris=1.13
   - python-stratify
   # Multi language support:
   - ncl
-  - libgdal=2.2.3  # Temporarily pin libgdal to get NCL working
   # TODO: add R, julia
 
   - pip:

esmvaltool/_config.py

@@ -23,6 +23,7 @@ def read_config_user_file(config_file, namelist_name):
         'output_file_type': 'ps',
         'output_dir': './output_dir',
         'save_intermediary_cubes': False,
+        'remove_preproc_dir': False,
         'max_parallel_tasks': 1,
         'run_diagnostic': True,
         'drs': {},

esmvaltool/_main.py

@@ -84,6 +84,10 @@ def get_args():
         action='store_true',
         help='Download input data using synda. This requires a working '
         'synda installation.')
+    parser.add_argument(
+        '--max-models',
+        type=int,
+        help='Try to limit the number of models used to MAX_MODELS.')
     args = parser.parse_args()
     return args
 
@@ -119,10 +123,12 @@ def main(args):
     logger.info("Writing program log files to:\n%s", "\n".join(log_files))
 
     cfg['synda_download'] = args.synda_download
+    cfg['max_models'] = args.max_models
 
     resource_log = os.path.join(cfg['run_dir'], 'resource_usage.txt')
     with resource_usage_logger(pid=os.getpid(), filename=resource_log):
         process_namelist(namelist_file=namelist_file, config_user=cfg)
+    return cfg
 
 
 def process_namelist(namelist_file, config_user):
@@ -183,12 +189,17 @@ def run():
     """Run the `esmvaltool` program, logging any exceptions."""
     args = get_args()
     try:
-        main(args)
+        conf = main(args)
     except:  # noqa
         logger.exception(
             "Program terminated abnormally, see stack trace "
             "below for more information",
             exc_info=True)
         sys.exit(1)
     else:
+        if conf["remove_preproc_dir"]:
+            logger.info("Removing preproc containing preprocessed data")
+            logger.info("If this data is further needed, then")
+            logger.info("set remove_preproc_dir to false in config")
+            shutil.rmtree(conf["preproc_dir"])
         logger.info("Run was succesful")