Update intercomparison with cmip models workflow (#786)

examples/e3sm_diags_for_cmip/e3sm_diags.bash

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+#SBATCH  --job-name=e3sm_diags_{{ model }}
+#SBATCH  --nodes=1
+#SBATCH  --output=e3sm_diags_{{ model }}.o%j
+#SBATCH  --exclusive
+#SBATCH  --time=02:00:00
+
+# Load environment
+#source /export/golaz1/conda/etc/profile.d/conda.sh
+#conda activate e3sm_diags_env_dev
+source /p/user_pub/e3sm_unified/envs/load_latest_e3sm_unified_acme1.sh
+
+# Make sure UVCDAT doesn't prompt us about anonymous logging
+export UVCDAT_ANONYMOUS_LOG=False
+
+# Run E3SM Diags
+time python << EOF
+
+import os
+from e3sm_diags.parameter.core_parameter import CoreParameter
+from e3sm_diags.run import runner
+
+param = CoreParameter()
+
+param.test_data_path = '{{ simulation }}'
+
+param.short_test_name = '{{ institution }} {{ model }} {{ experiment }} ({{ realization }})'
+param.test_timeseries_input = True
+param.test_start_yr = '1985'
+param.test_end_yr = '2014'
+
+#param.reference_data_path = '/p/user_pub/e3sm/e3sm_diags_data/obs_for_e3sm_diags/climatology/'
+param.reference_data_path = '/p/user_pub/e3sm/diagnostics/observations/Atm/climatology_1985-2014'
+
+
+param.results_dir = '/var/www/acme/acme-diags/zhang40/CMIP6_20240109_1985-2014/{{ model }}/{{ experiment }}/{{ realization }}'
+param.multiprocessing = True
+param.num_workers = 16
+#param.output_format_subplot = ["pdf", "png"]
+param.diff_title = '{{ model }} {{ experiment }} ({{ realization }}) vs Obs'
+
+# Use below to run all core sets of diags:
+#runner.sets_to_run = ['lat_lon','zonal_mean_xy', 'zonal_mean_2d', 'polar', 'cosp_histogram', 'meridional_mean_2d']
+# Use below to run only a subset:
+runner.sets_to_run = ['lat_lon']#, 'zonal_mean_xy', 'zonal_mean_2d']
+runner.run_diags([param])
+
+EOF
+
+echo All done...
+
+

examples/e3sm_diags_for_cmip/generate_page.py

@@ -20,8 +20,9 @@ def table_elements(search):
        "RESTOM global ceres_ebaf_toa_v4.1",
        "FSNTOA global ceres_ebaf_toa_v4.1", "FLUT global ceres_ebaf_toa_v4.1",
        "SWCF global ceres_ebaf_toa_v4.1", "LWCF global ceres_ebaf_toa_v4.1",
-       "U-850mb global ERA-Interim", "U-200mb global ERA-Interim",
-       "Z3-500mb global ERA-Interim"
+       "U-850mb global ERA5", "U-200mb global ERA5",
+       "T-850mb global ERA5", "T-200mb global ERA5",
+       "Z3-500mb global ERA5"
     ]
     metrics = ["Unit", "RMSE", "Mean_Bias", "Correlation"]
     seasons = ["ANN", "DJF", "MAM", "JJA", "SON"]
@@ -34,9 +35,11 @@ def table_elements(search):
     "FLUT global ceres_ebaf_toa_v4.1":"TOA LW (vs CERES-EBAAF Ed4.1)",
     "SWCF global ceres_ebaf_toa_v4.1":"TOA SWCRE (vs CERES-EBAAF Ed4.1)",
     "LWCF global ceres_ebaf_toa_v4.1":"TOA LWCRE (vs CERES-EBAAF Ed4.1)",
-    "U-850mb global ERA-Interim":"u 850 hPa (vs ERA-Interim)",
-    "U-200mb global ERA-Interim":"u 200 hPa (vs ERA-Interim)",
-    "Z3-500mb global ERA-Interim":"Geo-Z 500 hPA (vs ERA-Interim)",
+    "U-850mb global ERA5":"u 850 hPa (vs ERA5)",
+    "U-200mb global ERA5":"u 200 hPa (vs ERA5)",
+    "T-850mb global ERA5":"t 850 hPa (vs ERA5)",
+    "T-200mb global ERA5":"t 200 hPa (vs ERA5)",
+    "Z3-500mb global ERA5":"Geo-Z 500 hPA (vs ERA5)",
     "Unit":"unit",
     "RMSE":"rmse",
     "Mean_Bias":"bias",
@@ -51,9 +54,11 @@ def table_elements(search):
     "FLUT global ceres_ebaf_toa_v4.1":"lat_lon/ceres-ebaf-toa-v41/flut-global-ceres_ebaf_toa_v41",
     "SWCF global ceres_ebaf_toa_v4.1":"lat_lon/ceres-ebaf-toa-v41/swcf-global-ceres_ebaf_toa_v41",
     "LWCF global ceres_ebaf_toa_v4.1":"lat_lon/ceres-ebaf-toa-v41/lwcf-global-ceres_ebaf_toa_v41",
-    "U-850mb global ERA-Interim":"lat_lon/era-interim/u-850mb-global-era-interim",
-    "U-200mb global ERA-Interim":"lat_lon/era-interim/u-200mb-global-era-interim",
-    "Z3-500mb global ERA-Interim":"lat_lon/era-interim/z3-500mb-global-era-interim",
+    "U-850mb global ERA5":"lat_lon/era5/u-850mb-global-era5",
+    "U-200mb global ERA5":"lat_lon/era5/u-200mb-global-era5",
+    "T-850mb global ERA5":"lat_lon/era5/t-850mb-global-era5",
+    "T-200mb global ERA5":"lat_lon/era5/t-200mb-global-era5",
+    "Z3-500mb global ERA5":"lat_lon/era5/z3-500mb-global-era5",
     }
 
     # Loop over all simulations to gather data
@@ -74,7 +79,8 @@ def table_elements(search):
         c['model'] = p[-3]
         c['institution'] = p[-4]
         #c['www'] = "/var/www/acme/acme-diags/zhang40/CMIP6/%s/%s/%s" \
-        c['www'] = "/var/www/acme/acme-diags/e3sm_diags_for_cmip/%s/%s/%s" \
+        #c['www'] = "/var/www/acme/acme-diags/e3sm_diags_for_cmip/%s/%s/%s" \
+        c['www'] = "/var/www/acme/acme-diags/zhang40/CMIP6_20240109_1985-2014/%s/%s/%s" \
                    % (c['model'],c['experiment'],c['realization'])
 
         print(c['www'])
@@ -173,14 +179,16 @@ def table_elements(search):
 
 # amip simulations
 #fields, header, content = table_elements('/var/www/acme/acme-diags/zhang40/CMIP6/*/amip/r1i1p1f1/')
-fields, header, content = table_elements('/var/www/acme/acme-diags/e3sm_diags_for_cmip/*/amip/*/')
+#fields, header, content = table_elements('/var/www/acme/acme-diags/e3sm_diags_for_cmip/*/amip/*/')
+fields, header, content = table_elements('/var/www/acme/acme-diags/zhang40/CMIP6_20240109_1985-2014/*/amip/*/')
 c['fields'].append(fields)
 c['header'].append(header)
 c['content'].append(content)
 
 # historical simulations
 #fields, header, content = table_elements('/var/www/acme/acme-diags/zhang40/CMIP6/*/historical/r1i1p1f1/')
-fields, header, content = table_elements('/var/www/acme/acme-diags/e3sm_diags_for_cmip/*/historical/*/')
+#fields, header, content = table_elements('/var/www/acme/acme-diags/e3sm_diags_for_cmip/*/historical/*/')
+fields, header, content = table_elements('/var/www/acme/acme-diags/zhang40/CMIP6_20240109_1985-2014/*/historical/*/')
 c['fields'].append(fields)
 c['header'].append(header)
 c['content'].append(content)
@@ -191,7 +199,7 @@ def table_elements(search):
 template = templateEnv.get_template( 'cmip6_template.html' )
 
 # Instantiate page
-path = os.path.join('index.html')
+path = os.path.join('/var/www/acme/acme-diags/zhang40/CMIP6_20240109_1985-2014','index.html')
 with open(path, 'w') as f:
     f.write(template.render( **c ))
 

examples/e3sm_diags_for_cmip/generate_xmls.py

@@ -25,13 +25,15 @@ def run_command(command):
 dry_run = False
 
 # Output directory for xml files
-destination = '/home/zhang40/e3sm_diags_for_CMIP6/CMIP6_20211206'
+destination = '/home/zhang40/e3sm_diags_for_CMIP6/CMIP6_20240111'
 
 # Input search paths
-paths = ('/p/css03/esgf_publish/CMIP6/CMIP', '/p/user_pub/work/CMIP6/CMIP')
+paths = ('/p/user_pub/work/CMIP6/CMIP','')  # for E3SM CMIP archive only
+#paths = ('/p/css03/esgf_publish/CMIP6/CMIP', '/p/user_pub/work/CMIP6/CMIP')
 
 # Search patterm
-patterns = ('*/*/%s/r*i1p1f1/Amon/' % (experiment),)
+patterns = ('*/*/%s/r1i1p1f1/Amon/' % (experiment),)
+#patterns = ('UCSB/*/%s/r*i*p*f*/Amon/' % (experiment),)
 
 # Output file to log included input netCDF files
 name = "%s_%s.log" % (experiment,date.today().strftime("%y%m%d"))
@@ -65,6 +67,7 @@ def run_command(command):
         versions = sorted(versions)
         mostRecent = versions[-1]
         # Now, extract first and last date
+        print('most recent version', mostRecent)
         files = sorted(os.listdir(mostRecent))
         # First time stamp
         first = files[0].split('_')[-1]

examples/e3sm_diags_for_cmip/run_e3sm_diags.py

@@ -8,7 +8,8 @@
 import utils
 
 # Location of xml files
-input = '/home/zhang40/e3sm_diags_for_CMIP6/CMIP6_20211206/CMIP/*/*/amip/*/'
+input = '/home/zhang40/e3sm_diags_for_CMIP6/CMIP6_20240109_wE3SM/CMIP/*/*/amip/r1i1p1f1/'
+#input = '/home/zhang40/e3sm_diags_for_CMIP6/CMIP6_20240109/CMIP/*/*/historical/r1i1p1f1/'
 print(input)
 
 # Initialize jinja2 template engine

examples/e3sm_diags_for_cmip/templates/cmip6_template.html

@@ -21,12 +21,17 @@ <h1>E3SM Diags for CMIP6</h1>
     </div>
 
     <div class="container alert alert-warning" role="alert">
-      Warning: Data and plots presented on this page should be considered experimental and not used for publications.
+      Warning: Data and plots presented on this page should be considered experimental. The "ripf" identifier uniquely distinguishes each member of an ensemble: "r" for realization, "i" for initialization, "p" for physics, and "f" for forcing. Only "r1i1p1f1" is included here. (CMIP archive access date: Jan 2024).
+
+      Missing values (999.999) are resulted from missing data, incompatible metadata or unsupported grids or units with E3SM Diags.
+
+      Contact: [email protected]
+
     </div>
 
     <div class="container">
 
-    <h1>AMIP simulations (1980-2014)</h1>
+    <h1>AMIP simulations (1985-2014)</h1>
 
     <label for="select">Show RMSE for</label>
     <select id="amipSelect">
@@ -63,7 +68,7 @@ <h1>AMIP simulations (1980-2014)</h1>
     <div class="col-xs-12" style="height:70px;"></div>
     <div class="container">
 
-    <h1>Historical simulations (1980-2014)</h1>
+    <h1>Historical simulations (1985-2014)</h1>
 
     <label for="select">Show RMSE for</label>
     <select id="histSelect">