Add input_files_ref parameter

tests/integration/generated/test_bundles_chrysalis.cfg

@@ -7,11 +7,11 @@ input = "/lcrc/group/e3sm/ac.forsyth2/E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-607v6/v2.LR.historical_0201"
 partition = "compute"
 qos = "regular"
 walltime = "07:00:00"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-607v6"
 
 [bundle]
 

tests/integration/generated/test_complete_run_chrysalis.cfg

@@ -7,10 +7,10 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-607v6/v2.LR.historical_0201"
 partition = "debug"
 qos = "regular"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-607v6"
 
 [climo]
 active = True
@@ -134,7 +134,7 @@ years = "1850:1854:2", "1850:1854:4",
   ref_name = "v2.LR.historical_0201"
   ref_start_yr = 1850
   ref_years = "1850-1851",
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-607v6/v2.LR.historical_0201/post/atm/180x360_aave/clim"
   run_type = "model_vs_model"
   short_ref_name = "v2.LR.historical_0201"
   swap_test_ref = False
@@ -155,7 +155,7 @@ years = "1850:1854:2", "1850:1854:4",
   ref_name = "v2.LR.historical_0201"
   ref_start_yr = 1850
   ref_years = "1850-1851",
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/unique_id/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/test-607v6/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
   run_type = "model_vs_model"
   sets = "lat_lon_land",
   short_ref_name = "same simulation"

tests/integration/generated/test_debug_chrysalis.cfg

@@ -13,45 +13,53 @@ input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/unique_id/v2.LR.historical_0201"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_debug_output/test-607v6/v2.LR.historical_0201"
 partition = "debug"
 qos = "regular"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_debug_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_debug_www/test-607v6"
 
-[ts]
+[climo]
 active = True
 walltime = "00:30:00"
-years = "1850:1854:2",
+years = "1850:1854:2", "1850:1854:4",
 
-  [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
-  input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
-  ts_fmt = "cmip"
-
-  [[ atm_daily_180x360_aave ]]
-  frequency = "daily"
-  input_files = "eam.h1"
+  [[ atm_monthly_diurnal_8xdaily_180x360_aave ]]
+  frequency = "diurnal_8xdaily"
+  input_files = "eam.h3"
   input_subdir = "archive/atm/hist"
   vars = "PRECT"
 
-  [[ land_monthly ]]
-  e3sm_to_cmip_environment_commands = ""
-  extra_vars = "landfrac"
-  frequency = "monthly"
-  input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  vars = "LAISHA,LAISUN"
-  #vars = "FSH,RH2M"
-  ts_fmt = "cmip"
-
-[ilamb]
+[e3sm_diags]
 active = True
 grid = '180x360_aave'
-nodes = 8
-partition = "compute"
-short_name = 'v2.LR.historical_0201'
+ref_final_yr = 2014
+ref_start_yr = 1985
+# TODO: this directory is missing OMI-MLS
+sets = "diurnal_cycle",
 ts_num_years = 2
-years = "1850:1854:2",
+walltime = "00:30:00"
 
-  [[ land_monthly ]]
+  [[ atm_monthly_180x360_aave_mvm ]]
+  # Test model-vs-model using the same files as the reference
+  climo_diurnal_frequency = "diurnal_8xdaily"
+  climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
+  climo_subsection = "atm_monthly_180x360_aave"
+  diff_title = "Difference"
+  input_files_ref = "eam.h4"
+  partition = "compute"
+  qos = "regular"
+  ref_final_yr = 2014
+  ref_name = "v2.LR.amip_0101"
+  ref_start_yr = 1985
+  ref_years = "1985-2014",
+  reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.sfeng1/E3SM_output/v2.LR.amip_0101/post/atm/180x360_aave/clim_atm_monthly_diurnal_8xdaily_180x360_aave"
+  reference_data_path = "/lcrc/group/e3sm/ac.sfeng1/E3SM_output/v2.LR.amip_0101/post/"
+  run_type = "model_vs_model"
+  short_name = "BGCv2LNDATM"
+  short_ref_name = "v2.LR.amip"
+  swap_test_ref = False
+  tag = "model_vs_model"
+  ts_num_years_ref = 5
+  ts_subsection = "atm_monthly_180x360_aave"
+  walltime = "5:00:00"
+  years = "1852-1853",

tests/integration/generated/update_bundles_expected_files_chrysalis.sh

@@ -6,9 +6,9 @@
 rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 # Your output will now become the new expectation.
 # Copy output so you don't have to rerun zppy to generate the output.
-cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/test-607v6/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
-cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/unique_id/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
+cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/test-607v6/v2.LR.historical_0201/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
 zppy_top_level=$(pwd)
 cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles
 # Remove the image check failures, so they don't end up in the expected files.

tests/integration/generated/update_complete_run_expected_files_chrysalis.sh

@@ -6,7 +6,7 @@
 rm -rf /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 # Your output will now become the new expectation.
 # Copy output so you don't have to rerun zppy to generate the output.
-cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/unique_id/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
+cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/test-607v6/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 zppy_top_level=$(pwd)
 cd /lcrc/group/e3sm/public_html/zppy_test_resources/expected_complete_run
 # Remove the image check failures, so they don't end up in the expected files.

tests/integration/template_debug.cfg

@@ -16,42 +16,50 @@ mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 output = "#expand user_output#zppy_test_debug_output/#expand unique_id#/v2.LR.historical_0201"
 partition = "#expand partition_short#"
 qos = "#expand qos_short#"
-www = "#expand user_www#zppy_debug_www/#expand unique_id#"
+www = "#expand user_www#zppy_test_debug_www/#expand unique_id#"
 
-[ts]
+[climo]
 active = True
 walltime = "00:30:00"
-years = "1850:1854:2",
+years = "1850:1854:2", "1850:1854:4",
 
-  [[ atm_monthly_180x360_aave ]]
-  frequency = "monthly"
-  input_files = "eam.h0"
-  input_subdir = "archive/atm/hist"
-  ts_fmt = "cmip"
-
-  [[ atm_daily_180x360_aave ]]
-  frequency = "daily"
-  input_files = "eam.h1"
+  [[ atm_monthly_diurnal_8xdaily_180x360_aave ]]
+  frequency = "diurnal_8xdaily"
+  input_files = "eam.h3"
   input_subdir = "archive/atm/hist"
   vars = "PRECT"
 
-  [[ land_monthly ]]
-  e3sm_to_cmip_environment_commands = "#expand e3sm_to_cmip_environment_commands#"
-  extra_vars = "landfrac"
-  frequency = "monthly"
-  input_files = "elm.h0"
-  input_subdir = "archive/lnd/hist"
-  vars = "LAISHA,LAISUN"
-  #vars = "FSH,RH2M"
-  ts_fmt = "cmip"
-
-[ilamb]
+[e3sm_diags]
 active = True
 grid = '180x360_aave'
-nodes = 8
-partition = "#expand partition_long#"
-short_name = 'v2.LR.historical_0201'
+ref_final_yr = 2014
+ref_start_yr = 1985
+# TODO: this directory is missing OMI-MLS
+sets = "diurnal_cycle",
 ts_num_years = 2
-years = "1850:1854:2",
+walltime = "00:30:00"
 
-  [[ land_monthly ]]
+  [[ atm_monthly_180x360_aave_mvm ]]
+  # Test model-vs-model using the same files as the reference
+  climo_diurnal_frequency = "diurnal_8xdaily"
+  climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
+  climo_subsection = "atm_monthly_180x360_aave"
+  diff_title = "Difference"
+  input_files_ref = "eam.h4"
+  partition = "#expand partition_long#"
+  qos = "#expand qos_long#"
+  ref_final_yr = 2014
+  ref_name = "v2.LR.amip_0101"
+  ref_start_yr = 1985
+  ref_years = "1985-2014",
+  reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.sfeng1/E3SM_output/v2.LR.amip_0101/post/atm/180x360_aave/clim_atm_monthly_diurnal_8xdaily_180x360_aave"
+  reference_data_path = "/lcrc/group/e3sm/ac.sfeng1/E3SM_output/v2.LR.amip_0101/post/"
+  run_type = "model_vs_model"
+  short_name = "BGCv2LNDATM"
+  short_ref_name = "v2.LR.amip"
+  swap_test_ref = False
+  tag = "model_vs_model"
+  ts_num_years_ref = 5
+  ts_subsection = "atm_monthly_180x360_aave"
+  walltime = "#expand diags_walltime#"
+  years = "1852-1853",

tests/integration/utils.py

@@ -7,7 +7,7 @@
 from mache import MachineInfo
 from PIL import Image, ImageChops, ImageDraw
 
-UNIQUE_ID = "unique_id"
+UNIQUE_ID = "test-607v6"
 
 # Image checking ##########################################################
 

zppy/e3sm_diags.py

@@ -89,12 +89,13 @@ def e3sm_diags(config, scriptDir, existing_bundles, job_ids_file):  # noqa: C901
                 reference_data_path = (
                     c["reference_data_path"].split("/post")[0] + "/post"
                 )
-                if ("diurnal_cycle" in c["sets"]) and (
-                    c["reference_data_path_climo_diurnal"] == ""
-                ):
-                    c[
-                        "reference_data_path_climo_diurnal"
-                    ] = f"{reference_data_path}/atm/{c['grid']}/clim_diurnal_8xdaily"
+                if "diurnal_cycle" in c["sets"]:
+                    if c["reference_data_path_climo_diurnal"] == "":
+                        c[
+                            "reference_data_path_climo_diurnal"
+                        ] = f"{reference_data_path}/atm/{c['grid']}/clim_diurnal_8xdaily"
+                    if c["climo_diurnal_input_files_ref"] == "":
+                        c["climo_diurnal_input_files_ref"] = c["input_files_ref"]
                 if ("tc_analysis" in c["sets"]) and (c["reference_data_path_tc"] == ""):
                     c[
                         "reference_data_path_tc"

zppy/templates/default.ini

@@ -128,6 +128,9 @@ cfg = string(default="")
 # Name of the frequency from `[climo]` to use for "diurnal_cycle" runs
 climo_diurnal_frequency = string(default="")
 # climo_diurnal_input_files -- NOTE: this parameter is created internally
+# Below is the `input_files_ref` parameter specifically for climo_diurnal.
+# If it's not defined, `input_files_ref` will be used instead.
+climo_diurnal_input_files_ref = string(default="")
 # Name of the subsection of `[climo]` to use for "diurnal_cycle" runs
 climo_diurnal_subsection = string(default="")
 # Name of the `[climo]` subtask
@@ -142,6 +145,9 @@ keep_mvm_case_name_in_fig = boolean(default=True)
 # See https://e3sm-project.github.io/e3sm_diags/_build/html/master/available-parameters.html
 # Required for run_type="model_vs_model" "streamflow" runs
 gauges_path = string(default="")
+# Which files to use as input
+# Required for run_type="model_vs_model" if the required h# is different for the ref
+input_files_ref = string(default="eam.h0")
 # See https://e3sm-project.github.io/e3sm_diags/_build/html/master/available-parameters.html
 multiprocessing = boolean(default=True)
 # See https://e3sm-project.github.io/e3sm_diags/_build/html/master/available-parameters.html
@@ -238,11 +244,13 @@ ts_daily_subsection = string(default="")
   backend = string(default=None)
   cfg = string(default=None)
   climo_diurnal_frequency = string(default=None)
+  climo_diurnal_input_files_ref = string(default=None)
   climo_diurnal_subsection = string(default=None)
   climo_subsection = string(default=None)
   dc_obs_climo = string(default=None)
   diff_title = string(default=None)
   gauges_path = string(default=None)
+  input_files_ref = string(default=None)
   multiprocessing = boolean(default=None)
   num_workers = integer(default=None)
   obs_ts = string(default=None)

zppy/templates/e3sm_diags.bash

@@ -70,9 +70,15 @@ create_links_climo_diurnal()
   begin_year=$4
   end_year=$5
   error_num=$6
+  use_ref=$7
   mkdir -p ${climo_diurnal_dir_destination}
   cd ${climo_diurnal_dir_destination}
-  cp -s ${climo_diurnal_dir_source}/${nc_prefix}.{{ climo_diurnal_input_files }}_*_${begin_year}??_${end_year}??_climo.nc .
+  # TODO: for climo_diurnal, can we just use * and not worry about _ref at all?
+  if [[ "${use_ref,,}" == "true" ]]; then
+    cp -s ${climo_diurnal_dir_source}/${nc_prefix}.{{ climo_diurnal_input_files_ref }}_*_${begin_year}??_${end_year}??_climo.nc .
+  else
+    cp -s ${climo_diurnal_dir_source}/${nc_prefix}.{{ climo_diurnal_input_files }}_*_${begin_year}??_${end_year}??_climo.nc .
+  fi
   if [ $? != 0 ]; then
     cd {{ scriptDir }}
     echo "ERROR (${error_num})" > {{ prefix }}.status
@@ -182,12 +188,12 @@ climo_diurnal_dir_primary=climo_{{ climo_diurnal_frequency }}_test
 {%- endif %}
 # Create local links to input diurnal cycle climo files
 climo_diurnal_dir_source={{ output }}/post/atm/{{ grid }}/clim_{{ climo_diurnal_frequency }}/{{ '%dyr' % (year2-year1+1) }}
-create_links_climo_diurnal ${climo_diurnal_dir_source} ${climo_diurnal_dir_primary} ${case} ${Y1} ${Y2} 3
+create_links_climo_diurnal ${climo_diurnal_dir_source} ${climo_diurnal_dir_primary} ${case} ${Y1} ${Y2} 3 false
 {% if run_type == "model_vs_model" %}
 # Create local links to input climo files (ref model)
 climo_diurnal_dir_source={{ reference_data_path_climo_diurnal }}/{{ '%dyr' % (ref_year2-ref_year1+1) }}
 climo_diurnal_dir_ref=climo_diurnal_ref
-create_links_climo_diurnal ${climo_diurnal_dir_source} ${climo_diurnal_dir_ref} {{ ref_name }} ${ref_Y1} ${ref_Y2} 4
+create_links_climo_diurnal ${climo_diurnal_dir_source} ${climo_diurnal_dir_ref} {{ ref_name }} ${ref_Y1} ${ref_Y2} 4 true
 {%- endif %}
 {%- endif %}